From 174aa67cc3c07ff35ec0cb9a99c0d04a4806710e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:49:00 +0000 Subject: [PATCH 01/56] unbounded-storage: remove unused crypto dependencies --- cmd/unbounded-storage/Cargo.lock | 91 +------------------------------- cmd/unbounded-storage/Cargo.toml | 4 +- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/cmd/unbounded-storage/Cargo.lock b/cmd/unbounded-storage/Cargo.lock index 5ca541ff8..07a551f33 100644 --- a/cmd/unbounded-storage/Cargo.lock +++ b/cmd/unbounded-storage/Cargo.lock @@ -180,16 +180,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", + "cpufeatures", ] [[package]] @@ -272,15 +263,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.0" @@ -305,16 +287,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "debugid" version = "0.8.0" @@ -324,17 +296,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - [[package]] name = "either" version = "1.16.0" @@ -456,16 +417,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -518,15 +469,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "home" version = "0.5.12" @@ -1318,17 +1260,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - [[package]] name = "shlex" version = "2.0.1" @@ -1374,12 +1305,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "symbolic-common" version = "12.18.3" @@ -1517,12 +1442,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unarray" version = "0.1.4" @@ -1538,7 +1457,6 @@ dependencies = [ "blake3", "cc", "clap", - "hmac", "http", "httparse", "io-uring", @@ -1554,7 +1472,6 @@ dependencies = [ "rand 0.8.6", "rand_chacha 0.3.1", "serde", - "sha2", "tempfile", "toml", "twox-hash", @@ -1582,12 +1499,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/cmd/unbounded-storage/Cargo.toml b/cmd/unbounded-storage/Cargo.toml index 47bcd3891..cf6424513 100644 --- a/cmd/unbounded-storage/Cargo.toml +++ b/cmd/unbounded-storage/Cargo.toml @@ -18,11 +18,9 @@ bincode = "1.3" httparse = "1" http = "1" blake3 = "1.5" -sha2 = "0.10" -hmac = "0.12" arc-swap = "1.7" twox-hash = { version = "1.6", default-features = false } -clap = { version = "4", features = ["derive", "env"] } +clap = { version = "4", features = ["derive"] } rand = "0.8" rand_chacha = "0.3" toml = "0.8" From 1b31cfad6eb08ca2ae804ec39ac73202fee06c93 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:49:59 +0000 Subject: [PATCH 02/56] unbounded-storage: avoid boxing eviction checks --- cmd/unbounded-storage/src/storage/engine.rs | 152 ++++++++++---------- 1 file changed, 73 insertions(+), 79 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index 93cf80412..d180374e4 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -25,8 +25,6 @@ //! NVMe-backed engine. use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; use std::sync::{Arc, Mutex}; use crate::bufferpool::{self, PageRef, StripeKey}; @@ -422,89 +420,85 @@ impl StorageEngine { } } - fn evict_if_over_watermark<'a>(&'a self) -> Pin + 'a>> { - // Boxed because called from inside an async fn that's - // already 'a; eviction may itself await on the mutator. - Box::pin(async move { - if !self - .lru - .watermark_exceeded(self.cfg.eviction_watermark as f64) - { - return; - } - let candidates = self.lru.sweep(8); - if candidates.is_empty() { - return; - } - // Skip pinned LBAs - an in-flight reader still - // references them. Re-admit so they remain tracked - // and are reconsidered on a later sweep. - let mut victims: Vec = Vec::with_capacity(candidates.len()); - for lba in candidates { - if self.refcount.pin_count(lba.0).unwrap_or(0) == 0 { - victims.push(lba); - } else { - self.lru.admit(lba); - } - } - if victims.is_empty() { - return; + async fn evict_if_over_watermark(&self) { + if !self + .lru + .watermark_exceeded(self.cfg.eviction_watermark as f64) + { + return; + } + let candidates = self.lru.sweep(8); + if candidates.is_empty() { + return; + } + // Skip pinned LBAs - an in-flight reader still + // references them. Re-admit so they remain tracked + // and are reconsidered on a later sweep. + let mut victims: Vec = Vec::with_capacity(candidates.len()); + for lba in candidates { + if self.refcount.pin_count(lba.0).unwrap_or(0) == 0 { + victims.push(lba); + } else { + self.lru.admit(lba); } - // Resolve victim start-LBAs to (key, byte_len) via - // the reverse map before submitting: the mutator only - // needs the keys to commit the delete, and we still - // own the LBA runs here for the post-commit free - // below. - let mut keys: Vec = Vec::with_capacity(victims.len()); - let mut victim_runs: Vec<(Lba, u64)> = Vec::with_capacity(victims.len()); - { - let rev = self.reverse.lock().unwrap(); - for lba in &victims { - if let Some((k, byte_len)) = rev.get(&lba.0).copied() { - keys.push(k); - victim_runs.push((*lba, self.n_pages(byte_len))); - } + } + if victims.is_empty() { + return; + } + // Resolve victim start-LBAs to (key, byte_len) via + // the reverse map before submitting: the mutator only + // needs the keys to commit the delete, and we still + // own the LBA runs here for the post-commit free + // below. + let mut keys: Vec = Vec::with_capacity(victims.len()); + let mut victim_runs: Vec<(Lba, u64)> = Vec::with_capacity(victims.len()); + { + let rev = self.reverse.lock().unwrap(); + for lba in &victims { + if let Some((k, byte_len)) = rev.get(&lba.0).copied() { + keys.push(k); + victim_runs.push((*lba, self.n_pages(byte_len))); } } - if keys.is_empty() { + } + if keys.is_empty() { + return; + } + // The btree mutation MUST commit before the allocator + // slot is released: a concurrent writer that observes + // the stale btree entry would otherwise call + // retire_lba and double-free the slot the allocator + // already handed back. The mutator is the single + // committer, so this ordering is preserved by + // construction once `done.wait()` returns. + let done = MutatorReply::new(); + self.mutator_queue.push(MutatorReq::Delete { + keys, + done: done.clone(), + }); + match done.wait().await { + MutatorOutcome::DeleteCommitted => {} + _ => { + // Apply failed or queue closed: leave the + // LBAs allocated and tracked. A later sweep + // will retry. return; } - // The btree mutation MUST commit before the allocator - // slot is released: a concurrent writer that observes - // the stale btree entry would otherwise call - // retire_lba and double-free the slot the allocator - // already handed back. The mutator is the single - // committer, so this ordering is preserved by - // construction once `done.wait()` returns. - let done = MutatorReply::new(); - self.mutator_queue.push(MutatorReq::Delete { - keys, - done: done.clone(), - }); - match done.wait().await { - MutatorOutcome::DeleteCommitted => {} - _ => { - // Apply failed or queue closed: leave the - // LBAs allocated and tracked. A later sweep - // will retry. - return; - } - } - { - let mut rev = self.reverse.lock().unwrap(); - for (lba, _) in &victim_runs { - rev.remove(&lba.0); - } - } - for (lba, n) in &victim_runs { - let _ = self.allocator.free_range(*lba, *n); - let _ = self.refcount.reset(lba.0); - } - self.metric(|m| m.evictions += victim_runs.len() as u64); - for _ in &victim_runs { - crate::metrics::storage_eviction(self.disk()); + } + { + let mut rev = self.reverse.lock().unwrap(); + for (lba, _) in &victim_runs { + rev.remove(&lba.0); } - }) + } + for (lba, n) in &victim_runs { + let _ = self.allocator.free_range(*lba, *n); + let _ = self.refcount.reset(lba.0); + } + self.metric(|m| m.evictions += victim_runs.len() as u64); + for _ in &victim_runs { + crate::metrics::storage_eviction(self.disk()); + } } } From 57fed295a678419dcadab5420f98e7c1501c8233 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:52:15 +0000 Subject: [PATCH 03/56] unbounded-storage: remove unused admission sketch --- cmd/unbounded-storage/ARCHITECTURE.md | 2 +- .../src/storage/admission/mod.rs | 144 +++--------------- .../src/storage/admission/stripe.rs | 5 +- .../src/storage/admission/tests.rs | 29 ++-- cmd/unbounded-storage/src/storage/engine.rs | 8 +- cmd/unbounded-storage/src/storage/types.rs | 4 +- .../tests/storage/recovery.rs | 2 +- 7 files changed, 39 insertions(+), 155 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 97e12ab61..628b6fbd5 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -480,7 +480,7 @@ touches the kernel device) -> `alloc` + `refcount` (pure LBA tables) -> `btree` `btree_page_bytes`), `btree_page_bytes` 4096 (the device atomic write unit), `commit_batch_max` 1024, `commit_batch_deadline_us` 200, `eviction_watermark` 0.9, `probationary_fraction` 0.1, -`admission_sketch_multiplier` 2, `singleflight_shards` 64, +`singleflight_shards` 64, `restart_scan_queue_depth` 256, `bypass_admission` false (bench/tooling only), and `skip_recovery_scan_if_no_meta` false (set from the public `skip_recovery_scan` config flag; production keeps it false so partial recovery diff --git a/cmd/unbounded-storage/src/storage/admission/mod.rs b/cmd/unbounded-storage/src/storage/admission/mod.rs index 327e9d91b..c98635a8e 100644 --- a/cmd/unbounded-storage/src/storage/admission/mod.rs +++ b/cmd/unbounded-storage/src/storage/admission/mod.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! TinyLFU-style admission filter. +//! Second-touch admission filter. //! //! Implements the design's "admit on second touch" policy via a //! doorkeeper bloom filter. The first time a [`PageKey`] is @@ -10,25 +10,14 @@ //! within the same epoch return `true`. This filters one-hit //! wonders from polluting the resident set. //! -//! The full TinyLFU adds a count-min sketch for frequency -//! tracking. We keep a sketch as a feature-flag stub -//! ([`record_frequency`](AdmissionFilter::record_frequency) / -//! [`frequency`](AdmissionFilter::frequency)) so the engine can be -//! extended later; the admission decision today only consults -//! the doorkeeper. Views that only need the doorkeeper construct -//! via [`AdmissionFilter::new_doorkeeper_only`] to skip the -//! sketch allocation and per-admit bumps entirely. -//! //! All state is in memory and is intentionally rebuilt empty //! on restart - bloom mistakes are bounded and the cache is //! warm-up tolerant. //! //! ## Concurrency //! -//! The hot path ([`should_admit`](AdmissionFilter::should_admit), -//! [`record_frequency`](AdmissionFilter::record_frequency), -//! [`frequency`](AdmissionFilter::frequency)) is lock-free. The -//! doorkeeper, sketch, and aging counter are all atomics accessed +//! The hot path ([`should_admit`](AdmissionFilter::should_admit)) +//! is lock-free. The doorkeeper and aging counter are atomics accessed //! with [`Ordering::Relaxed`]: this is a probabilistic structure //! and we do not build a happens-before relationship through it. //! The only mutex is a small one guarding the rare aging step @@ -41,7 +30,7 @@ //! eventual-admit semantics are preserved. use std::sync::Mutex; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use crate::storage::types::{GOLDEN_RATIO_64, PageKey}; @@ -54,24 +43,18 @@ pub struct AdmissionFilter { // tells us whether the bit was already set. doorkeeper: Box<[AtomicU64]>, capacity_bits: u64, - // Count-min sketch, one row per hash. `None` when the filter - // is in doorkeeper-only mode (see `new_doorkeeper_only`). - sketch: Option<[Box<[AtomicU8]>; SKETCH_ROWS]>, - sketch_width: u32, - // Aging is driven by a counter of admits since the last age - // step. When it crosses `reset_threshold`, exactly one writer - // takes `aging` and halves the sketch counters. + // Resetting is driven by a counter of admits since the last + // reset. When it crosses `reset_threshold`, exactly one writer + // takes `aging` and clears the doorkeeper. inserts_since_reset: AtomicU64, reset_threshold: u64, aging: Mutex<()>, } const NUM_HASHES: u32 = 3; -const SKETCH_ROWS: usize = 4; // Domain tag for the doorkeeper bloom filter probes. Not a -// secret; see PageKey::mix. Sketch rows use domains 1..=SKETCH_ROWS via -// sketch_index, so 0 is reserved for the doorkeeper. +// secret; see PageKey::mix. const DOORKEEPER_DOMAIN: u32 = 0; // Bloom sizing multiplier. With k = 3 hashes the FPR is @@ -80,34 +63,24 @@ const DOORKEEPER_DOMAIN: u32 = 0; const BITS_PER_ENTRY: u64 = 12; impl AdmissionFilter { - /// Full filter with both doorkeeper and count-min sketch. - pub fn new(capacity_pages: u64, sketch_multiplier: u32) -> Self { + pub fn new(capacity_pages: u64) -> Self { let (doorkeeper, capacity_bits) = make_doorkeeper(capacity_pages); - let sketch_width = (capacity_pages.max(1) as u32).saturating_mul(sketch_multiplier.max(1)); - let sketch = make_sketch(sketch_width); Self { doorkeeper, capacity_bits, - sketch: Some(sketch), - sketch_width, inserts_since_reset: AtomicU64::new(0), - // Halve the sketch (aging) every ~10 * capacity admits. + // Reset the doorkeeper every ~10 * capacity admits. reset_threshold: capacity_pages.max(1).saturating_mul(10), aging: Mutex::new(()), } } - /// Doorkeeper-only filter: no count-min sketch is allocated, - /// `record_frequency` is a no-op, and `frequency` returns 0. - /// Used by views (see [`StripeAdmission`]) that only need the - /// second-sight admission decision. + /// Non-aging filter used by views whose lifetime defines the epoch. pub fn new_doorkeeper_only(capacity_pages: u64) -> Self { let (doorkeeper, capacity_bits) = make_doorkeeper(capacity_pages); Self { doorkeeper, capacity_bits, - sketch: None, - sketch_width: 0, inserts_since_reset: AtomicU64::new(0), reset_threshold: u64::MAX, aging: Mutex::new(()), @@ -125,76 +98,38 @@ impl AdmissionFilter { pub fn should_admit(&self, key: &PageKey) -> bool { let admit = doorkeeper_probe_and_set(&self.doorkeeper, self.capacity_bits, key); if admit { - if let Some(sketch) = &self.sketch { - sketch_bump(sketch, self.sketch_width, key); - let prev = self.inserts_since_reset.fetch_add(1, Ordering::Relaxed); - if prev + 1 >= self.reset_threshold { - self.maybe_age(sketch); - } + let prev = self.inserts_since_reset.fetch_add(1, Ordering::Relaxed); + if prev + 1 >= self.reset_threshold { + self.maybe_reset(); } } admit } - /// Increment the sketch without consulting the doorkeeper. - /// Exposed so the engine can record hits as well as admits. - /// No-op in doorkeeper-only mode. - pub fn record_frequency(&self, key: &PageKey) { - if let Some(sketch) = &self.sketch { - sketch_bump(sketch, self.sketch_width, key); - } - } - /// Reset all state. Tests and explicit warmup boundaries /// call this. Restart is handled at engine construction by /// simply not persisting any of this state. pub fn reset(&self) { - // Hold the aging mutex so a concurrent aging step cannot + // Hold the aging mutex so a concurrent reset cannot // race the wipe. let _g = self.aging.lock().unwrap(); clear_doorkeeper(&self.doorkeeper); - if let Some(sketch) = &self.sketch { - for row in sketch.iter() { - for c in row.iter() { - c.store(0, Ordering::Relaxed); - } - } - } self.inserts_since_reset.store(0, Ordering::Relaxed); } - /// Approximate frequency estimate from the count-min - /// sketch. Used by callers that want to make their own - /// decision (e.g. compare against a victim's estimate). - /// Returns 0 in doorkeeper-only mode. - pub fn frequency(&self, key: &PageKey) -> u8 { - let Some(sketch) = &self.sketch else { - return 0; - }; - let mut min = u8::MAX; - for (i, row) in sketch.iter().enumerate() { - let idx = sketch_index(key, i as u32, self.sketch_width); - min = min.min(row[idx].load(Ordering::Relaxed)); - } - min - } - - fn maybe_age(&self, sketch: &[Box<[AtomicU8]>; SKETCH_ROWS]) { - // Serialize aging on its own mutex. Multiple writers that + fn maybe_reset(&self) { + // Serialize resetting on its own mutex. Multiple writers that // also observed threshold-crossed will block briefly, then // double-check the counter and exit if another writer - // already aged. + // already reset it. let _g = self.aging.lock().unwrap(); if self.inserts_since_reset.load(Ordering::Relaxed) < self.reset_threshold { return; } - age(sketch); - // Clear the doorkeeper in lockstep with the sketch reset. - // Without this the bloom fills monotonically over the + // Without resetting, the bloom fills monotonically over the // process lifetime, its FPR climbs toward 1.0, and // "admit on second touch" degrades to "admit on first - // touch", defeating scan resistance (canonical W-TinyLFU - // resets the doorkeeper alongside each aging period). + // touch", defeating scan resistance. clear_doorkeeper(&self.doorkeeper); self.inserts_since_reset.store(0, Ordering::Relaxed); } @@ -208,17 +143,9 @@ fn make_doorkeeper(capacity_pages: u64) -> (Box<[AtomicU64]>, u64) { (v.into_boxed_slice(), capacity_bits) } -fn make_sketch(width: u32) -> [Box<[AtomicU8]>; SKETCH_ROWS] { - std::array::from_fn(|_| { - let mut row = Vec::with_capacity(width as usize); - row.resize_with(width as usize, || AtomicU8::new(0)); - row.into_boxed_slice() - }) -} - -// Zero every doorkeeper word. Called under the `aging` mutex -// from both `reset` and the aging step so a wipe cannot race a -// concurrent age. Relaxed matches the rest of this probabilistic +// Zero every doorkeeper word. Called under the `aging` mutex so +// explicit and threshold-driven resets cannot race. Relaxed matches +// the rest of this probabilistic // structure (see the module-level concurrency note). fn clear_doorkeeper(words: &[AtomicU64]) { for w in words.iter() { @@ -241,30 +168,5 @@ fn doorkeeper_probe_and_set(words: &[AtomicU64], bits: u64, key: &PageKey) -> bo all_set } -fn sketch_bump(rows: &[Box<[AtomicU8]>; SKETCH_ROWS], width: u32, key: &PageKey) { - for i in 0..SKETCH_ROWS { - let idx = sketch_index(key, i as u32, width); - let _ = rows[i][idx].fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { - if c == u8::MAX { None } else { Some(c + 1) } - }); - } -} - -fn sketch_index(key: &PageKey, row: u32, width: u32) -> usize { - let h = key.mix(row.wrapping_add(1)); - (h % (width as u64)) as usize -} - -fn age(rows: &[Box<[AtomicU8]>; SKETCH_ROWS]) { - for row in rows.iter() { - for c in row.iter() { - // Halve. Relaxed is fine: aging is approximate and - // serialized via the aging mutex against other agers. - let v = c.load(Ordering::Relaxed); - c.store(v >> 1, Ordering::Relaxed); - } - } -} - #[cfg(test)] mod tests; diff --git a/cmd/unbounded-storage/src/storage/admission/stripe.rs b/cmd/unbounded-storage/src/storage/admission/stripe.rs index 10f938f39..1ce8355ab 100644 --- a/cmd/unbounded-storage/src/storage/admission/stripe.rs +++ b/cmd/unbounded-storage/src/storage/admission/stripe.rs @@ -41,8 +41,7 @@ pub struct StripeAdmission { impl StripeAdmission { /// Construct with the same capacity semantics as /// [`AdmissionFilter`]. Sized for a few million entries at - /// roughly 1% FPR. The view uses doorkeeper-only mode; the - /// count-min sketch is not allocated. + /// roughly 1% FPR. The view's lifetime defines its admission epoch. pub fn new(capacity_stripes: u64) -> Self { Self { inner: Arc::new(AdmissionFilter::new_doorkeeper_only(capacity_stripes)), @@ -62,7 +61,7 @@ impl StripeAdmission { } } - /// Reset the underlying doorkeeper and sketch. Used by tests + /// Reset the underlying doorkeeper. Used by tests /// and future warmup boundaries. pub fn reset(&self) { self.inner.reset(); diff --git a/cmd/unbounded-storage/src/storage/admission/tests.rs b/cmd/unbounded-storage/src/storage/admission/tests.rs index cbaf15344..4f1bb5bcc 100644 --- a/cmd/unbounded-storage/src/storage/admission/tests.rs +++ b/cmd/unbounded-storage/src/storage/admission/tests.rs @@ -9,14 +9,14 @@ fn key(i: u32) -> PageKey { #[test] fn first_touch_rejects_second_admits() { - let f = AdmissionFilter::new(1024, 2); + let f = AdmissionFilter::new(1024); assert!(!f.should_admit(&key(1))); assert!(f.should_admit(&key(1))); } #[test] fn distinct_keys_each_need_two_touches() { - let f = AdmissionFilter::new(1024, 2); + let f = AdmissionFilter::new(1024); for i in 0..32u32 { assert!(!f.should_admit(&key(i))); } @@ -25,19 +25,9 @@ fn distinct_keys_each_need_two_touches() { } } -#[test] -fn record_frequency_bumps_counter() { - let f = AdmissionFilter::new(64, 4); - let k = key(7); - assert_eq!(f.frequency(&k), 0); - f.record_frequency(&k); - f.record_frequency(&k); - assert!(f.frequency(&k) >= 2); -} - #[test] fn reset_clears_state() { - let f = AdmissionFilter::new(256, 2); + let f = AdmissionFilter::new(256); assert!(!f.should_admit(&key(1))); f.reset(); // After reset, the doorkeeper is empty again so the next @@ -47,13 +37,12 @@ fn reset_clears_state() { #[test] fn aging_clears_doorkeeper_restoring_first_sight() { - // Aging halves the sketch every `10 * capacity` admits and - // must clear the doorkeeper in lockstep. Otherwise the bloom - // fills forever and "admit on second touch" silently decays - // into "admit on first touch". Use a tiny capacity so the + // Aging clears the doorkeeper every `10 * capacity` admits. + // Otherwise the bloom fills forever and "admit on second touch" + // silently decays into "admit on first touch". Use a tiny capacity so the // threshold (10 * 4 = 40 admits) is cheap to cross. let cap = 4u64; - let f = AdmissionFilter::new(cap, 2); + let f = AdmissionFilter::new(cap); let victim = key(1); let threshold = (cap * 10) as usize; @@ -71,8 +60,8 @@ fn aging_clears_doorkeeper_restoring_first_sight() { assert!(f.should_admit(&victim)); } - // This admit crosses the threshold and triggers the aging - // step, which clears the doorkeeper. The call itself still + // This admit crosses the threshold and triggers the reset, + // which clears the doorkeeper. The call itself still // observes the bit as set (clearing happens after the probe), // so it returns true. assert!(f.should_admit(&victim)); diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index d180374e4..a0249d494 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -73,7 +73,6 @@ pub struct EngineConfig { pub commit_batch_ticks: u32, pub eviction_watermark: f32, pub probationary_fraction: f32, - pub admission_sketch_multiplier: usize, pub singleflight_shards: usize, pub restart_scan_queue_depth: u32, /// Number of 4 KiB registered scratch buffers reserved for @@ -131,7 +130,6 @@ impl Default for EngineConfig { commit_batch_ticks: 8, eviction_watermark: 0.9, probationary_fraction: 0.1, - admission_sketch_multiplier: 2, singleflight_shards: 64, restart_scan_queue_depth: 256, btree_scratch_pages: 64, @@ -285,10 +283,7 @@ impl StorageEngine { .await?, ); let lru = Arc::new(SieveLru::new(capacity, refcount.clone())); - let admission = Arc::new(AdmissionFilter::new( - capacity, - cfg.admission_sketch_multiplier.max(1) as u32, - )); + let admission = Arc::new(AdmissionFilter::new(capacity)); let singleflight = Arc::new(Singleflight::new(cfg.singleflight_shards.max(1))); let engine = Self { device, @@ -628,7 +623,6 @@ impl StorageEngine { } self.lru.touch(entry.lba); - self.admission.record_frequency(&pk); self.metric(|m| m.hits += 1); crate::metrics::storage_lookup(crate::metrics::Lookup::Hit); Ok(true) diff --git a/cmd/unbounded-storage/src/storage/types.rs b/cmd/unbounded-storage/src/storage/types.rs index 3beef3860..05d85feda 100644 --- a/cmd/unbounded-storage/src/storage/types.rs +++ b/cmd/unbounded-storage/src/storage/types.rs @@ -81,8 +81,8 @@ impl PageKey { } /// FNV-1a-style mixer over the key's bytes, tagged with a - /// per-caller domain separator. Used by admission's count-min - /// sketch and the singleflight shard selector. Not cryptographic; + /// per-caller domain separator. Used by the admission doorkeeper + /// and the singleflight shard selector. Not cryptographic; /// the `domain` tag is not a secret and only exists to give /// different callers near-independent 64-bit lanes. pub fn mix(&self, domain: u32) -> u64 { diff --git a/cmd/unbounded-storage/tests/storage/recovery.rs b/cmd/unbounded-storage/tests/storage/recovery.rs index 112684e9e..a2dc293d9 100644 --- a/cmd/unbounded-storage/tests/storage/recovery.rs +++ b/cmd/unbounded-storage/tests/storage/recovery.rs @@ -832,7 +832,7 @@ fn concurrent_writes_same_key_collapse_to_one_entry() { // Eight concurrent writers, all aimed at the same key/offset. // Each writer issues two `write_page` calls so the admission // filter promotes the key past its "second touch" gate: the - // first call from each writer primes the sketch, the second + // first call from each writer primes the doorkeeper, the second // becomes a singleflight contender. Without this, eight // single-shot writers all get filter-rejected and the engine // never reaches the device path the test exists to exercise. From 3c31427eff418b181cae6ae0132776a81b9c65aa Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:53:59 +0000 Subject: [PATCH 04/56] unbounded-storage: simplify write singleflight --- cmd/unbounded-storage/src/storage/engine.rs | 20 +- .../src/storage/singleflight.rs | 518 +++--------------- 2 files changed, 76 insertions(+), 462 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index a0249d494..b1e9624c5 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -35,7 +35,7 @@ use crate::storage::btree::{BTreeIndex, LeafEntry, Mutation}; use crate::storage::lru::SieveLru; use crate::storage::mutator::{MutatorOutcome, MutatorQueue, MutatorReply, MutatorReq}; use crate::storage::refcount::RefcountTable; -use crate::storage::singleflight::{Acquire, Singleflight}; +use crate::storage::singleflight::Singleflight; use crate::storage::traits::{PageChecksum, Xxh3Checksum}; use crate::storage::types::{Lba, PageKey}; @@ -165,7 +165,7 @@ pub struct StorageEngine { btree: Arc>, lru: Arc, admission: Arc, - singleflight: Arc, + singleflight: Singleflight, cfg: EngineConfig, bufferpool: Mutex, /// Reverse map start-LBA -> (PageKey, byte_len). Eviction @@ -284,7 +284,7 @@ impl StorageEngine { ); let lru = Arc::new(SieveLru::new(capacity, refcount.clone())); let admission = Arc::new(AdmissionFilter::new(capacity)); - let singleflight = Arc::new(Singleflight::new(cfg.singleflight_shards.max(1))); + let singleflight = Singleflight::new(cfg.singleflight_shards.max(1)); let engine = Self { device, allocator, @@ -660,12 +660,8 @@ impl StorageEngine { return Ok(()); } - let acquire = self.singleflight.acquire(pk); - let leader = match acquire { - Acquire::Leader(g) => g, - Acquire::Follower(_w) => { - return Ok(()); - } + let Some(_leader) = self.singleflight.try_acquire(pk) else { + return Ok(()); }; // SAFETY: caller-provided contract on `src`. @@ -685,7 +681,6 @@ impl StorageEngine { src_buf.len(), self.cfg.btree_page_bytes, ); - leader.abandon(); return Err(bufferpool::Error::Io(libc::EINVAL)); } @@ -693,7 +688,6 @@ impl StorageEngine { let lba = match self.allocator.alloc_contig(n_pages) { Ok(l) => l, Err(_) => { - leader.abandon(); return Ok(()); } }; @@ -714,7 +708,6 @@ impl StorageEngine { crate::metrics::Outcome::Err, ); let _ = self.allocator.free_range(lba, n_pages); - leader.abandon(); return Err(bufferpool::Error::Io(io_errno(&e))); } crate::metrics::storage_disk_op( @@ -751,7 +744,6 @@ impl StorageEngine { // the device-write branch above. self.metric(|m| m.write_io_errors += 1); let _ = self.allocator.free_range(lba, n_pages); - leader.abandon(); return Err(bufferpool::Error::Io(libc::EIO)); } MutatorOutcome::DeleteCommitted => { @@ -769,8 +761,6 @@ impl StorageEngine { } self.lru.admit(lba); self.metric(|m| m.admitted += 1); - leader.publish(entry); - self.evict_if_over_watermark().await; Ok(()) diff --git a/cmd/unbounded-storage/src/storage/singleflight.rs b/cmd/unbounded-storage/src/storage/singleflight.rs index 9d8913438..0a8717ace 100644 --- a/cmd/unbounded-storage/src/storage/singleflight.rs +++ b/cmd/unbounded-storage/src/storage/singleflight.rs @@ -3,98 +3,45 @@ //! Per-key singleflight gate. //! -//! Concurrent fills for the same [`PageKey`] collapse onto a -//! single in-flight operation. The leader holds a -//! [`LeaseGuard`] and is responsible for either publishing a -//! result or letting the guard drop (which wakes followers with -//! a `None` so they fall back to their normal "treat as miss" -//! path). -//! -//! Internally the table is sharded by key hash so contention -//! between unrelated keys is bounded. Sharding is independent -//! of the singleflight semantics for any one key - all callers -//! for the same key always land on the same shard. +//! Concurrent fills for the same [`PageKey`] collapse onto one in-flight +//! operation. The leader holds a [`LeaseGuard`]; followers return immediately +//! without performing duplicate I/O. Dropping the guard releases the key on +//! both success and failure paths. -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll, Waker}; +use std::collections::HashSet; +use std::sync::Mutex; -use crate::storage::btree::LeafEntry; use crate::storage::types::PageKey; -/// Outcome of a fill: `Some(entry)` if the leader published a -/// successful result, `None` if the leader dropped without -/// publishing (treated as a miss by callers). -pub type FillResult = Option; - -/// Result of [`Singleflight::acquire`]: either you're the leader -/// and must do the work, or you're a follower waiting for -/// someone else's result. -pub enum Acquire { - Leader(LeaseGuard), - Follower(LeaseWait), -} - -struct Lease { - result: Mutex, -} - -struct LeaseInner { - result: Option, - waiters: Vec, - // Once true the lease is "settled" - either published or - // abandoned - and the table entry should be cleared. - settled: bool, -} - pub struct Singleflight { - shards: Vec>>>, + shards: Vec>>, } impl Singleflight { pub fn new(shards: usize) -> Self { let shards = shards.max(1); Self { - shards: (0..shards).map(|_| Mutex::new(HashMap::new())).collect(), + shards: (0..shards).map(|_| Mutex::new(HashSet::new())).collect(), } } - fn shard(&self, key: &PageKey) -> &Mutex>> { - // Domain tag for the singleflight shard selector. Not a - // secret; see PageKey::mix. + fn shard(&self, key: &PageKey) -> &Mutex> { const SHARD_DOMAIN: u32 = 0; let h = key.mix(SHARD_DOMAIN); &self.shards[(h as usize) % self.shards.len()] } - pub fn acquire(self: &Arc, key: PageKey) -> Acquire { - let shard = self.shard(&key); - let mut map = shard.lock().unwrap(); - if let Some(existing) = map.get(&key).cloned() { - return Acquire::Follower(LeaseWait { lease: existing }); + pub fn try_acquire(&self, key: PageKey) -> Option> { + if !self.shard(&key).lock().unwrap().insert(key) { + return None; } - let lease = Arc::new(Lease { - result: Mutex::new(LeaseInner { - result: None, - waiters: Vec::new(), - settled: false, - }), - }); - map.insert(key, lease.clone()); - Acquire::Leader(LeaseGuard { - owner: self.clone(), - key, - lease, - armed: true, - }) + + Some(LeaseGuard { owner: self, key }) } - fn remove(&self, key: &PageKey) { - let shard = self.shard(key); - let mut map = shard.lock().unwrap(); - map.remove(key); + fn release(&self, key: &PageKey) { + let removed = self.shard(key).lock().unwrap().remove(key); + debug_assert!(removed, "singleflight lease released more than once"); } pub fn in_flight(&self) -> usize { @@ -102,299 +49,63 @@ impl Singleflight { } } -/// Held by the leader. Call [`publish`] to hand a result to -/// followers; dropping without publishing signals "give up" and -/// followers receive `None`. -pub struct LeaseGuard { - owner: Arc, +pub struct LeaseGuard<'a> { + owner: &'a Singleflight, key: PageKey, - lease: Arc, - armed: bool, -} - -impl LeaseGuard { - pub fn publish(mut self, value: LeafEntry) { - self.settle(Some(value)); - } - - pub fn abandon(mut self) { - self.settle(None); - } - - fn settle(&mut self, value: FillResult) { - let wakers = { - let mut inner = self.lease.result.lock().unwrap(); - inner.result = Some(value); - inner.settled = true; - std::mem::take(&mut inner.waiters) - }; - self.owner.remove(&self.key); - self.armed = false; - for w in wakers { - w.wake(); - } - } } -impl Drop for LeaseGuard { +impl Drop for LeaseGuard<'_> { fn drop(&mut self) { - if self.armed { - self.settle(None); - } - } -} - -/// Future returned to followers. Resolves once the leader -/// settles the lease. -pub struct LeaseWait { - lease: Arc, -} - -impl Future for LeaseWait { - type Output = FillResult; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut inner = self.lease.result.lock().unwrap(); - if inner.settled { - return Poll::Ready(inner.result.clone().unwrap_or(None)); - } - // Stash our waker if we don't already have one. - if !inner.waiters.iter().any(|w| w.will_wake(cx.waker())) { - inner.waiters.push(cx.waker().clone()); - } - Poll::Pending + self.owner.release(&self.key); } } #[cfg(test)] mod tests { - use super::*; - use crate::runtime::noop_waker; - use crate::storage::types::{Checksum, Lba}; + use std::collections::HashMap; + use proptest::collection::vec; use proptest::prelude::*; - use std::collections::HashMap; - use std::future::Future; - use std::pin::pin; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::task::Wake; + + use super::*; fn key(i: u32) -> PageKey { PageKey::new([0u8; 32], i) } - fn entry(lba: u64) -> LeafEntry { - LeafEntry { - lba: Lba(lba), - data_checksum: Checksum(0), - byte_len: 0, - } - } - - #[derive(Clone, Debug, PartialEq, Eq)] - enum ExpectedFill { - Published(u64), - Abandoned, - } - - enum Role { - Leader { - key: PageKey, - epoch: u64, - guard: LeaseGuard, - }, - Follower { - key: PageKey, - epoch: u64, - wait: Pin>, - expected: Option, - polled: bool, - wakes: Arc, - }, - } - - struct CountWaker { - count: Arc, - } - - impl Wake for CountWaker { - fn wake(self: Arc) { - self.wake_by_ref(); - } - - fn wake_by_ref(self: &Arc) { - self.count.fetch_add(1, Ordering::SeqCst); - } - } - - fn assert_fill_matches(got: FillResult, expected: &ExpectedFill) { - match (got, expected) { - (Some(e), ExpectedFill::Published(lba)) => assert_eq!(e.lba.0, *lba), - (None, ExpectedFill::Abandoned) => {} - (got, expected) => panic!("fill result {got:?} did not match {expected:?}"), - } - } - - fn follower_wake_counts( - roles: &[Option], - key: PageKey, - epoch: u64, - ) -> Vec<(usize, usize)> { - roles - .iter() - .enumerate() - .filter_map(|(idx, role)| match role { - Some(Role::Follower { - key: follower_key, - epoch: follower_epoch, - polled: true, - wakes, - .. - }) if *follower_key == key && *follower_epoch == epoch => { - Some((idx, wakes.load(Ordering::SeqCst))) - } - _ => None, - }) - .collect() - } - - fn mark_followers_settled( - roles: &mut [Option], - key: PageKey, - epoch: u64, - expected: ExpectedFill, - wake_counts: &[(usize, usize)], - ) { - for (idx, before) in wake_counts { - let Some(Role::Follower { wakes, .. }) = roles[*idx].as_ref() else { - continue; - }; - assert!( - wakes.load(Ordering::SeqCst) > *before, - "settling leader for key {:?} did not wake polled follower {}", - key, - idx, - ); - } - - for role in roles.iter_mut().flatten() { - if let Role::Follower { - key: follower_key, - epoch: follower_epoch, - expected: follower_expected, - .. - } = role - { - if *follower_key == key && *follower_epoch == epoch { - assert!( - follower_expected.replace(expected.clone()).is_none(), - "follower for key {:?} settled twice", - key, - ); - } - } - } - } - - fn poll_follower(role: &mut Option) { - let Some(Role::Follower { - wait, - expected, - polled, - wakes, - .. - }) = role - else { - return; - }; - - let waker: Waker = Arc::new(CountWaker { - count: wakes.clone(), - }) - .into(); - let mut cx = Context::from_waker(&waker); - match wait.as_mut().poll(&mut cx) { - Poll::Pending => { - assert!( - expected.is_none(), - "settled follower returned Pending instead of Ready", - ); - *polled = true; - } - Poll::Ready(got) => { - let expected = expected - .as_ref() - .expect("follower resolved before its leader settled"); - assert_fill_matches(got, expected); - *role = None; - } - } - } - #[test] - fn leader_followers_get_published_value() { - let sf = Arc::new(Singleflight::new(4)); - let leader = match sf.acquire(key(1)) { - Acquire::Leader(g) => g, - Acquire::Follower(_) => panic!("expected leader"), - }; - let follower = match sf.acquire(key(1)) { - Acquire::Follower(f) => f, - Acquire::Leader(_) => panic!("expected follower"), - }; - leader.publish(entry(42)); - let w = noop_waker(); - let mut cx = Context::from_waker(&w); - let mut f = pin!(follower); - match f.as_mut().poll(&mut cx) { - Poll::Ready(Some(e)) => assert_eq!(e.lba.0, 42), - other => panic!("got {other:?}"), - } - assert_eq!(sf.in_flight(), 0); - } + fn follower_is_rejected_until_guard_drops() { + let sf = Singleflight::new(4); + let leader = sf.try_acquire(key(1)).expect("first caller is leader"); + + assert!(sf.try_acquire(key(1)).is_none()); + assert_eq!(sf.in_flight(), 1); - #[test] - fn leader_drop_signals_none() { - let sf = Arc::new(Singleflight::new(4)); - let leader = match sf.acquire(key(7)) { - Acquire::Leader(g) => g, - _ => unreachable!(), - }; - let follower = match sf.acquire(key(7)) { - Acquire::Follower(f) => f, - _ => unreachable!(), - }; drop(leader); - let w = noop_waker(); - let mut cx = Context::from_waker(&w); - let mut f = pin!(follower); - match f.as_mut().poll(&mut cx) { - Poll::Ready(None) => {} - other => panic!("got {other:?}"), - } + assert!(sf.try_acquire(key(1)).is_some()); } #[test] - fn different_keys_dont_collide() { - let sf = Arc::new(Singleflight::new(4)); - let a = sf.acquire(key(1)); - let b = sf.acquire(key(2)); - assert!(matches!(a, Acquire::Leader(_))); - assert!(matches!(b, Acquire::Leader(_))); + fn different_keys_do_not_collide() { + let sf = Singleflight::new(1); + let first = sf.try_acquire(key(1)).expect("first key is available"); + let second = sf.try_acquire(key(2)).expect("second key is available"); + assert_eq!(sf.in_flight(), 2); + drop((first, second)); + assert_eq!(sf.in_flight(), 0); } #[test] - fn second_acquire_after_settle_is_leader_again() { - let sf = Arc::new(Singleflight::new(4)); - let l1 = match sf.acquire(key(3)) { - Acquire::Leader(g) => g, - _ => unreachable!(), - }; - l1.publish(entry(1)); - // After publish, the table entry is gone and a new - // acquire becomes the leader. - let l2 = sf.acquire(key(3)); - assert!(matches!(l2, Acquire::Leader(_))); + fn failure_scope_releases_key() { + let sf = Singleflight::new(4); + let result: Result<(), ()> = (|| { + let _guard = sf.try_acquire(key(7)).expect("first caller is leader"); + Err(()) + })(); + + assert!(result.is_err()); + assert!(sf.try_acquire(key(7)).is_some()); } proptest! { @@ -404,126 +115,39 @@ mod tests { })] #[test] - fn randomized_leader_follower_interleavings( - ops in vec((0usize..8, 0u32..4, 0u8..5), 1..200), + fn randomized_acquire_release_interleavings( + ops in vec((0usize..8, 0u32..4, any::()), 1..200), ) { - let sf = Arc::new(Singleflight::new(4)); - let mut roles: Vec> = (0..8).map(|_| None).collect(); - let mut active: HashMap = HashMap::new(); - let mut next_epoch: HashMap = HashMap::new(); - - for (actor, key_idx, op) in ops { - let page_key = key(key_idx); - match op { - 0 => { - if roles[actor].is_some() { - continue; - } - match sf.acquire(page_key) { - Acquire::Leader(guard) => { - let epoch = next_epoch.entry(page_key).or_insert(0); - let leader_epoch = *epoch; - *epoch += 1; - assert!( - active.insert(page_key, leader_epoch).is_none(), - "created a second leader for key {:?}", - page_key, - ); - roles[actor] = Some(Role::Leader { - key: page_key, - epoch: leader_epoch, - guard, - }); - } - Acquire::Follower(wait) => { - let follower_epoch = *active.get(&page_key).unwrap_or_else(|| { - panic!( - "created a follower for key {:?} without an active leader", - page_key, - ) - }); - roles[actor] = Some(Role::Follower { - key: page_key, - epoch: follower_epoch, - wait: Box::pin(wait), - expected: None, - polled: false, - wakes: Arc::new(AtomicUsize::new(0)), - }); - } - } + let sf = Singleflight::new(4); + let mut guards: Vec>> = + (0..8).map(|_| None).collect(); + let mut active: HashMap = HashMap::new(); + + for (actor, key_idx, acquire) in ops { + if acquire { + if guards[actor].is_some() { + continue; } - 1 => poll_follower(&mut roles[actor]), - 2 | 3 | 4 => { - let Some(role) = roles[actor].take() else { - continue; - }; - match role { - Role::Leader { - key: leader_key, - epoch, - guard, - } => { - let expected = if op == 2 { - ExpectedFill::Published(actor as u64 + 1000) - } else { - ExpectedFill::Abandoned - }; - let wake_counts = follower_wake_counts(&roles, leader_key, epoch); - match expected { - ExpectedFill::Published(lba) => guard.publish(entry(lba)), - ExpectedFill::Abandoned => { - if op == 3 { - guard.abandon(); - } else { - drop(guard); - } - } - } - active.remove(&leader_key); - mark_followers_settled( - &mut roles, - leader_key, - epoch, - expected, - &wake_counts, - ); - } - Role::Follower { .. } => { - // Dropping a follower before settlement must not affect the leader - // or any other follower. The remaining end-of-test checks catch leaks. - } + + let page_key = key(key_idx); + match sf.try_acquire(page_key) { + Some(guard) => { + prop_assert!(active.insert(page_key, actor).is_none()); + guards[actor] = Some(guard); } + None => prop_assert!(active.contains_key(&page_key)), } - _ => unreachable!(), + } else if let Some(guard) = guards[actor].take() { + active.remove(&guard.key); + drop(guard); } prop_assert_eq!(sf.in_flight(), active.len()); } - for actor in 0..roles.len() { - let Some(Role::Leader { - key: leader_key, - epoch, - guard, - }) = roles[actor].take() else { - continue; - }; - let wake_counts = follower_wake_counts(&roles, leader_key, epoch); + for guard in guards.into_iter().flatten() { + active.remove(&guard.key); drop(guard); - active.remove(&leader_key); - mark_followers_settled( - &mut roles, - leader_key, - epoch, - ExpectedFill::Abandoned, - &wake_counts, - ); - } - - for role in &mut roles { - poll_follower(role); - prop_assert!(role.is_none(), "follower did not resolve after all leaders settled"); } prop_assert!(active.is_empty()); prop_assert_eq!(sf.in_flight(), 0); From 374c49a3cb3fa784e632a550ff22442eb92f46be Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:55:35 +0000 Subject: [PATCH 05/56] unbounded-storage: avoid allocating for btree checksums --- .../src/storage/btree/cow.rs | 10 ++-- .../src/storage/btree/meta.rs | 4 +- .../src/storage/btree/page.rs | 51 +++++++++++-------- .../src/storage/btree/rebuild.rs | 2 +- .../src/storage/btree/tests.rs | 2 +- 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/btree/cow.rs b/cmd/unbounded-storage/src/storage/btree/cow.rs index 0b1bd63af..119d83554 100644 --- a/cmd/unbounded-storage/src/storage/btree/cow.rs +++ b/cmd/unbounded-storage/src/storage/btree/cow.rs @@ -114,7 +114,7 @@ pub async fn build_internal_cache( Err(_) => continue, } - match page::decode(buf.as_slice()) { + match page::decode(buf.as_mut_slice()) { Decoded::Internal { keys, children, .. } if keys.len() == children.len() => { if remaining_depth > 1 { for &child in &children { @@ -161,7 +161,7 @@ async fn internal_depth( Err(e) if strict => return Err(e), Err(_) => return Ok(depth), } - match page::decode(buf.as_slice()) { + match page::decode(buf.as_mut_slice()) { Decoded::Internal { children, .. } if !children.is_empty() => { depth += 1; cur = children[0]; @@ -208,7 +208,7 @@ pub async fn lookup( Ok(()) => {} Err(_) => return Ok(None), } - match page::decode(buf.as_slice()) { + match page::decode(buf.as_mut_slice()) { Decoded::Empty => return Ok(None), Decoded::Leaf { entries, .. } => { for (k, v) in entries { @@ -297,7 +297,7 @@ where if device.read(node, buf.as_mut_slice()).await.is_err() { continue; } - let decoded = page::decode(buf.as_slice()); + let decoded = page::decode(buf.as_mut_slice()); if let Decoded::Internal { ref children, .. } = decoded { for &c in children { stack.push(c); @@ -544,7 +544,7 @@ impl<'a, B: BlockDevice> PathCopyCtx<'a, B> { let decoded = { let mut buf = self.scratch.acquire().await; self.device.read(node_lba, buf.as_mut_slice()).await?; - page::decode(buf.as_slice()) + page::decode(buf.as_mut_slice()) }; match decoded { diff --git a/cmd/unbounded-storage/src/storage/btree/meta.rs b/cmd/unbounded-storage/src/storage/btree/meta.rs index 2d1ddf25e..9efb38412 100644 --- a/cmd/unbounded-storage/src/storage/btree/meta.rs +++ b/cmd/unbounded-storage/src/storage/btree/meta.rs @@ -60,11 +60,11 @@ pub async fn load_meta( // We tolerate I/O errors here: the design wants us to treat a // partially-failing disk as "no meta", not propagate the error. let a = match device.read(page::META_SLOT_A, buf_a.as_mut_slice()).await { - Ok(()) => page::decode(buf_a.as_slice()), + Ok(()) => page::decode(buf_a.as_mut_slice()), Err(_) => Decoded::Empty, }; let b = match device.read(page::META_SLOT_B, buf_b.as_mut_slice()).await { - Ok(()) => page::decode(buf_b.as_slice()), + Ok(()) => page::decode(buf_b.as_mut_slice()), Err(_) => Decoded::Empty, }; let a_meta = as_meta(a, MetaSlot::A); diff --git a/cmd/unbounded-storage/src/storage/btree/page.rs b/cmd/unbounded-storage/src/storage/btree/page.rs index 68573b252..4ecebfd79 100644 --- a/cmd/unbounded-storage/src/storage/btree/page.rs +++ b/cmd/unbounded-storage/src/storage/btree/page.rs @@ -107,7 +107,7 @@ pub fn max_internal_keys(page_size: usize) -> usize { /// Decode a page slice. The slice must be exactly `page_size` /// bytes (the caller has already loaded it from disk). -pub fn decode(page: &[u8]) -> Decoded { +pub fn decode(page: &mut [u8]) -> Decoded { if page.len() < HEADER_LEN { return Decoded::Empty; } @@ -116,7 +116,6 @@ pub fn decode(page: &[u8]) -> Decoded { let txn_id = u64::from_le_bytes(page[8..16].try_into().unwrap()); let stored_checksum = u64::from_le_bytes(page[HDR_CSUM_OFF..HDR_CSUM_END].try_into().unwrap()); - // Validate checksum: clone the page with checksum field zeroed. if !verify_checksum(page, stored_checksum) { return Decoded::Empty; } @@ -129,16 +128,13 @@ pub fn decode(page: &[u8]) -> Decoded { } } -fn verify_checksum(page: &[u8], stored: u64) -> bool { - // xxh3 in this crate only has a one-shot helper, so we hash a - // copy of the page with the checksum field zeroed. One Vec - // allocation; the body bytes copy is unavoidable without a - // streaming API. - let mut buf = page.to_vec(); - for b in &mut buf[HDR_CSUM_OFF..HDR_CSUM_END] { - *b = 0; - } - Xxh3Checksum::checksum_of(&buf).0 == stored +fn verify_checksum(page: &mut [u8], stored: u64) -> bool { + let saved: [u8; HDR_CSUM_END - HDR_CSUM_OFF] = + page[HDR_CSUM_OFF..HDR_CSUM_END].try_into().unwrap(); + page[HDR_CSUM_OFF..HDR_CSUM_END].fill(0); + let actual = Xxh3Checksum::checksum_of(page).0; + page[HDR_CSUM_OFF..HDR_CSUM_END].copy_from_slice(&saved); + actual == stored } fn decode_leaf(page: &[u8], nentries: usize, txn_id: u64) -> Decoded { @@ -330,8 +326,8 @@ mod tests { }, ), ]; - let p = encode_leaf(4096, 42, &entries).unwrap(); - match decode(&p) { + let mut p = encode_leaf(4096, 42, &entries).unwrap(); + match decode(&mut p) { Decoded::Leaf { txn_id, entries: got, @@ -347,8 +343,8 @@ mod tests { fn internal_roundtrip() { let keys = vec![key(1), key(5), key(9)]; let children = vec![Lba(10), Lba(20), Lba(30)]; - let p = encode_internal(4096, 7, &keys, &children).unwrap(); - match decode(&p) { + let mut p = encode_internal(4096, 7, &keys, &children).unwrap(); + match decode(&mut p) { Decoded::Internal { txn_id, keys: gk, @@ -364,8 +360,8 @@ mod tests { #[test] fn meta_roundtrip() { - let p = encode_meta(4096, 99, Lba(12345), 7777); - match decode(&p) { + let mut p = encode_meta(4096, 99, Lba(12345), 7777); + match decode(&mut p) { Decoded::Meta { txn_id, root_lba, @@ -385,8 +381,8 @@ mod tests { // meta page written with hwm = 0 decodes back as hwm = 0, // matching what an old encoder that left the trailing // reserved bytes zeroed would produce. - let p = encode_meta(4096, 1, Lba(42), 0); - match decode(&p) { + let mut p = encode_meta(4096, 1, Lba(42), 0); + match decode(&mut p) { Decoded::Meta { hwm, .. } => assert_eq!(hwm, 0), other => panic!("expected meta, got {other:?}"), } @@ -397,7 +393,18 @@ mod tests { let p = encode_meta(4096, 1, Lba(1), 0); let mut bad = p.clone(); bad[HEADER_LEN] ^= 0x01; // flip a byte in the body - assert!(matches!(decode(&bad), Decoded::Empty)); + let before = bad.clone(); + assert!(matches!(decode(&mut bad), Decoded::Empty)); + assert_eq!(bad, before); + } + + #[test] + fn decode_restores_checksum_bytes() { + let mut page = encode_meta(4096, 1, Lba(1), 0); + let before = page.clone(); + + assert!(matches!(decode(&mut page), Decoded::Meta { .. })); + assert_eq!(page, before); } #[test] @@ -406,7 +413,7 @@ mod tests { p[0] = 99; // reseal so checksum is valid but type is unknown seal_checksum(&mut p); - assert!(matches!(decode(&p), Decoded::Empty)); + assert!(matches!(decode(&mut p), Decoded::Empty)); } #[test] diff --git a/cmd/unbounded-storage/src/storage/btree/rebuild.rs b/cmd/unbounded-storage/src/storage/btree/rebuild.rs index 90b75ab7a..c3603d1bc 100644 --- a/cmd/unbounded-storage/src/storage/btree/rebuild.rs +++ b/cmd/unbounded-storage/src/storage/btree/rebuild.rs @@ -69,7 +69,7 @@ pub async fn scan_for_leaves( if device.read(Lba(lba), buf.as_mut_slice()).await.is_err() { continue; } - if let Decoded::Leaf { txn_id, entries } = page::decode(buf.as_slice()) { + if let Decoded::Leaf { txn_id, entries } = page::decode(buf.as_mut_slice()) { match txn_id.cmp(&best_txn) { Ordering::Greater => { best_txn = txn_id; diff --git a/cmd/unbounded-storage/src/storage/btree/tests.rs b/cmd/unbounded-storage/src/storage/btree/tests.rs index 0ac61cd06..2445fcbdc 100644 --- a/cmd/unbounded-storage/src/storage/btree/tests.rs +++ b/cmd/unbounded-storage/src/storage/btree/tests.rs @@ -361,7 +361,7 @@ fn snapshot_drop_frees_old_pages() { fn read_meta_slot(dev: &MockDevice, lba: Lba) -> Option<(u64, Lba, u64)> { let mut buf = vec![0u8; dev.page_size()]; dev.peek(lba, &mut buf); - match page::decode(&buf) { + match page::decode(&mut buf) { Decoded::Meta { txn_id, root_lba, From d863effd631a4e4c5c652a369c0c99def6b7e081 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:56:51 +0000 Subject: [PATCH 06/56] unbounded-storage: update btree mirror in place --- .../src/storage/btree/mod.rs | 30 +++++++++++-------- .../src/storage/btree/tests.rs | 9 ++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/btree/mod.rs b/cmd/unbounded-storage/src/storage/btree/mod.rs index aadba9e39..ee0160b2f 100644 --- a/cmd/unbounded-storage/src/storage/btree/mod.rs +++ b/cmd/unbounded-storage/src/storage/btree/mod.rs @@ -457,30 +457,24 @@ impl BTreeIndex { /// at most one task at a time (the engine's mutator). pub async fn apply_batch(&self, mutations: Vec) -> Result<(), Error> { // Coalesce the input mutations into a sorted (key, op) - // list with at most one entry per key, while also building - // the next-state mirror. We don't touch the live mirror - // yet: if path-copy or the meta-page write fails we leave - // the in-memory state exactly as it was so the caller's - // recovery (freeing the new data LBA, etc.) matches what - // is on disk. - let (sorted_ops, txn_id, next_entries) = { + // list with at most one entry per key. The live mirror is + // not touched until the inactive meta write makes the new + // root durable. + let (sorted_ops, txn_id) = { let state = self.mutator.borrow(); - let mut next = state.entries.clone(); let mut ops: BTreeMap> = BTreeMap::new(); for m in mutations { match m { Mutation::Insert { key, value } => { ops.insert(key, Some(value)); - next.insert(key, value); } Mutation::Delete { key } => { ops.insert(key, None); - next.remove(&key); } } } let sorted_ops: Vec<(PageKey, Option)> = ops.into_iter().collect(); - (sorted_ops, state.next_txn_id, next) + (sorted_ops, state.next_txn_id) }; if sorted_ops.is_empty() { @@ -490,6 +484,9 @@ impl BTreeIndex { return Ok(()); } + // Path-copy consumes the operations, so retain only the bounded + // batch needed to update the mirror after the durable commit. + let mirror_ops = sorted_ops.clone(); let parent_root = self.root.load().root_lba; let result = cow::apply_path_copy( &*self.device, @@ -541,7 +538,16 @@ impl BTreeIndex { // the published snapshot all advance together. { let mut state = self.mutator.borrow_mut(); - state.entries = next_entries; + for (key, value) in mirror_ops { + match value { + Some(value) => { + state.entries.insert(key, value); + } + None => { + state.entries.remove(&key); + } + } + } state.next_txn_id = txn_id + 1; } self.active_meta.set(new_active); diff --git a/cmd/unbounded-storage/src/storage/btree/tests.rs b/cmd/unbounded-storage/src/storage/btree/tests.rs index 2445fcbdc..c2ee959c6 100644 --- a/cmd/unbounded-storage/src/storage/btree/tests.rs +++ b/cmd/unbounded-storage/src/storage/btree/tests.rs @@ -1007,11 +1007,20 @@ fn failed_write_mid_commit_unwinds_and_preserves_tree() { Some(entry(i as u64)), "pre-commit key {i} lost", ); + assert_eq!( + idx.lookup_committed_mirror(&key(i)), + Some(entry(i as u64)), + "pre-commit mirror entry {i} lost", + ); } assert!( block_on(idx.lookup(&key(250))).unwrap().is_none(), "key from the failed commit must not be visible", ); + assert!( + idx.lookup_committed_mirror(&key(250)).is_none(), + "key from the failed commit must not enter the mirror", + ); // The tree must still accept new commits after the failure. dev.fail_at.set(None); From 0705a99182ca5664450d5c2ead0c40704bd6674f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:33:20 +0000 Subject: [PATCH 07/56] unbounded-storage: simplify checksum implementation --- .../src/storage/btree/page.rs | 6 +- cmd/unbounded-storage/src/storage/checksum.rs | 28 +++++++++ cmd/unbounded-storage/src/storage/engine.rs | 8 +-- cmd/unbounded-storage/src/storage/local.rs | 4 +- cmd/unbounded-storage/src/storage/mod.rs | 3 +- cmd/unbounded-storage/src/storage/traits.rs | 62 ------------------- 6 files changed, 37 insertions(+), 74 deletions(-) create mode 100644 cmd/unbounded-storage/src/storage/checksum.rs delete mode 100644 cmd/unbounded-storage/src/storage/traits.rs diff --git a/cmd/unbounded-storage/src/storage/btree/page.rs b/cmd/unbounded-storage/src/storage/btree/page.rs index 4ecebfd79..87f131483 100644 --- a/cmd/unbounded-storage/src/storage/btree/page.rs +++ b/cmd/unbounded-storage/src/storage/btree/page.rs @@ -32,7 +32,7 @@ //! Internal entries store, in their `key` slot, the *smallest* key //! reachable through their child. -use crate::storage::traits::{PageChecksum, Xxh3Checksum}; +use crate::storage::checksum::checksum; use crate::storage::types::{Checksum, Error, Lba, PageKey}; pub const HEADER_LEN: usize = 32; @@ -132,7 +132,7 @@ fn verify_checksum(page: &mut [u8], stored: u64) -> bool { let saved: [u8; HDR_CSUM_END - HDR_CSUM_OFF] = page[HDR_CSUM_OFF..HDR_CSUM_END].try_into().unwrap(); page[HDR_CSUM_OFF..HDR_CSUM_END].fill(0); - let actual = Xxh3Checksum::checksum_of(page).0; + let actual = checksum(page).0; page[HDR_CSUM_OFF..HDR_CSUM_END].copy_from_slice(&saved); actual == stored } @@ -294,7 +294,7 @@ fn seal_checksum(page: &mut [u8]) { for b in &mut page[HDR_CSUM_OFF..HDR_CSUM_END] { *b = 0; } - let cs = Xxh3Checksum::checksum_of(page).0; + let cs = checksum(page).0; page[HDR_CSUM_OFF..HDR_CSUM_END].copy_from_slice(&cs.to_le_bytes()); } diff --git a/cmd/unbounded-storage/src/storage/checksum.rs b/cmd/unbounded-storage/src/storage/checksum.rs new file mode 100644 index 000000000..9eb101809 --- /dev/null +++ b/cmd/unbounded-storage/src/storage/checksum.rs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::storage::types::Checksum; + +pub(crate) fn checksum(bytes: &[u8]) -> Checksum { + Checksum(twox_hash::xxh3::hash64(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn checksum_is_deterministic() { + assert_eq!(checksum(b"hello world"), checksum(b"hello world")); + } + + #[test] + fn checksum_differs_for_different_inputs() { + assert_ne!(checksum(b"hello world"), checksum(b"hello worle")); + } + + #[test] + fn checksum_empty_is_well_defined() { + assert_eq!(checksum(&[]), checksum(&[])); + } +} diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index b1e9624c5..5a3d6629e 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -32,11 +32,11 @@ use crate::storage::admission::AdmissionFilter; use crate::storage::alloc::Allocator; use crate::storage::blockdev::{BlockDevice, ScratchPool}; use crate::storage::btree::{BTreeIndex, LeafEntry, Mutation}; +use crate::storage::checksum::checksum; use crate::storage::lru::SieveLru; use crate::storage::mutator::{MutatorOutcome, MutatorQueue, MutatorReply, MutatorReq}; use crate::storage::refcount::RefcountTable; use crate::storage::singleflight::Singleflight; -use crate::storage::traits::{PageChecksum, Xxh3Checksum}; use crate::storage::types::{Lba, PageKey}; #[derive(Clone, Debug)] @@ -611,9 +611,7 @@ impl StorageEngine { crate::metrics::Outcome::Ok, ); - if !self.cfg.bypass_checksum - && Xxh3Checksum::checksum_of(dst_buf).0 != entry.data_checksum.0 - { + if !self.cfg.bypass_checksum && checksum(dst_buf).0 != entry.data_checksum.0 { self.metric(|m| { m.misses += 1; m.checksum_misses += 1; @@ -716,7 +714,7 @@ impl StorageEngine { crate::metrics::Outcome::Ok, ); - let cs = Xxh3Checksum::checksum_of(src_buf); + let cs = checksum(src_buf); let entry = LeafEntry { lba, data_checksum: cs, diff --git a/cmd/unbounded-storage/src/storage/local.rs b/cmd/unbounded-storage/src/storage/local.rs index 296a8802a..9a58b1908 100644 --- a/cmd/unbounded-storage/src/storage/local.rs +++ b/cmd/unbounded-storage/src/storage/local.rs @@ -20,8 +20,8 @@ use std::sync::Arc; use crate::bufferpool; use crate::bufferpool::{Error, PageRef, StripeKey}; use crate::storage::blockdev::BlockDevice; +use crate::storage::checksum::checksum; use crate::storage::engine::StorageEngine; -use crate::storage::traits::{PageChecksum, Xxh3Checksum}; /// Fans the [`bufferpool::BlockStore`] surface across `N` per-disk /// engines on a single node. Construction takes ownership of the @@ -137,7 +137,7 @@ pub fn disk_for(key: &StripeKey, stripe_off: u64, num_disks: usize) -> usize { let mut buf = [0u8; 40]; buf[..32].copy_from_slice(&key.0); buf[32..].copy_from_slice(&stripe_off.to_le_bytes()); - let h = Xxh3Checksum::checksum_of(&buf).0; + let h = checksum(&buf).0; (h % num_disks as u64) as usize } diff --git a/cmd/unbounded-storage/src/storage/mod.rs b/cmd/unbounded-storage/src/storage/mod.rs index 1e3eadb9e..4d10ceae7 100644 --- a/cmd/unbounded-storage/src/storage/mod.rs +++ b/cmd/unbounded-storage/src/storage/mod.rs @@ -21,6 +21,7 @@ pub mod types; mod alloc; mod btree; +mod checksum; mod engine; mod local; mod lru; @@ -31,7 +32,6 @@ mod page_channel; mod refcount; mod singleflight; mod synthetic; -mod traits; pub use admission::{AdmissionFilter, AdmitDecision, StripeAdmission}; pub use engine::{EngineConfig, StorageEngine}; @@ -44,5 +44,4 @@ pub use synthetic::{ matches_bytes as synthetic_matches_bytes, object_hash, object_id as synthetic_object_id, parse_object_id as parse_synthetic_object_id, splitmix64, }; -pub use traits::PageChecksum; pub use types::{Checksum, DiskId, Error, Lba, PageKey}; diff --git a/cmd/unbounded-storage/src/storage/traits.rs b/cmd/unbounded-storage/src/storage/traits.rs deleted file mode 100644 index 7ad3f91cb..000000000 --- a/cmd/unbounded-storage/src/storage/traits.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! Cross-module trait surface. -//! -//! This file is intentionally small: the meaningful traits -//! ([`BlockDevice`](crate::storage::blockdev::BlockDevice)) live in -//! the module that owns their concrete implementations. What lives -//! here are tiny helper traits that more than one storage submodule -//! needs to agree on - currently just the checksum routine used by -//! both the btree and the engine. - -use crate::storage::types::Checksum; - -/// Single source of truth for "how do we compute a 64-bit checksum -/// over a byte slice". Implemented as a trait so the btree and the -/// engine can share it without anyone reaching into the other. -/// -/// The default impl uses xxh3 with the storage subsystem's fixed -/// seed (`0`); callers should treat `Checksum::ZERO` as "no -/// checksum recorded yet" but verify-time `0` is a real value too. -pub trait PageChecksum { - fn checksum_of(bytes: &[u8]) -> Checksum; -} - -/// Default checksum impl over [xxh3](twox_hash::xxh3). Lives as a -/// zero-sized type so callers can write -/// `::checksum_of(...)` without -/// having to construct anything. -pub struct Xxh3Checksum; - -impl PageChecksum for Xxh3Checksum { - fn checksum_of(bytes: &[u8]) -> Checksum { - Checksum(twox_hash::xxh3::hash64(bytes)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn checksum_is_deterministic() { - let a = Xxh3Checksum::checksum_of(b"hello world"); - let b = Xxh3Checksum::checksum_of(b"hello world"); - assert_eq!(a, b); - } - - #[test] - fn checksum_differs_for_different_inputs() { - let a = Xxh3Checksum::checksum_of(b"hello world"); - let b = Xxh3Checksum::checksum_of(b"hello worle"); - assert_ne!(a, b); - } - - #[test] - fn checksum_empty_is_well_defined() { - let a = Xxh3Checksum::checksum_of(&[]); - let b = Xxh3Checksum::checksum_of(&[]); - assert_eq!(a, b); - } -} From 3c6c2efd8603a16985346575195b77032e184a45 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:34:19 +0000 Subject: [PATCH 08/56] unbounded-storage: remove unused engine settings --- cmd/unbounded-storage/ARCHITECTURE.md | 6 ++--- cmd/unbounded-storage/src/storage/engine.rs | 25 +++----------------- cmd/unbounded-storage/src/storage/mutator.rs | 10 ++++---- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 628b6fbd5..b7519e97d 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -478,10 +478,10 @@ touches the kernel device) -> `alloc` + `refcount` (pure LBA tables) -> `btree` `EngineConfig` defaults: `page_size_bytes` 2 MiB (a multiple of `btree_page_bytes`), `btree_page_bytes` 4096 (the device atomic write unit), -`commit_batch_max` 1024, `commit_batch_deadline_us` 200, -`eviction_watermark` 0.9, `probationary_fraction` 0.1, +`commit_batch_max` 1024, `commit_batch_ticks` 8, +`eviction_watermark` 0.9, `singleflight_shards` 64, -`restart_scan_queue_depth` 256, `bypass_admission` false (bench/tooling only), +`bypass_admission` false (bench/tooling only), and `skip_recovery_scan_if_no_meta` false (set from the public `skip_recovery_scan` config flag; production keeps it false so partial recovery runs). diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index 5a3d6629e..5f675da27 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -52,29 +52,15 @@ pub struct EngineConfig { /// the CoW B+tree commit cost; the mutator never exceeds this /// per commit. pub commit_batch_max: usize, - /// Target batch-coalescing latency, in microseconds. This is a - /// logical hint, not a value the engine reads as elapsed time: - /// the deterministic-simulation harness forbids reading the - /// wall clock, so the mutator realizes this budget as - /// [`Self::commit_batch_ticks`] cooperative yields rather than - /// a real deadline. Retained for documentation and future - /// production tuning. - pub commit_batch_deadline_us: u64, /// Number of cooperative yields the mutator may spend waiting /// for more requests to coalesce into a batch that has not yet - /// reached [`Self::commit_batch_max`]. This is the - /// wall-clock-free realization of - /// [`Self::commit_batch_deadline_us`]: each yield lets the + /// reached [`Self::commit_batch_max`]. Each yield lets the /// executor interleave producers that are about to enqueue, so /// the batch grows under load while a drained or closed queue - /// commits immediately. Default 8 (roughly - /// `commit_batch_deadline_us / 25us` per yield); 0 disables - /// coalescing. + /// commits immediately. Default 8; 0 disables coalescing. pub commit_batch_ticks: u32, pub eviction_watermark: f32, - pub probationary_fraction: f32, pub singleflight_shards: usize, - pub restart_scan_queue_depth: u32, /// Number of 4 KiB registered scratch buffers reserved for /// btree / meta I/O. This is the hard ceiling on how many /// btree page writes a single path-copy commit can keep in @@ -126,12 +112,9 @@ impl Default for EngineConfig { page_size_bytes: 2 * 1024 * 1024, btree_page_bytes: 4096, commit_batch_max: 1024, - commit_batch_deadline_us: 200, commit_batch_ticks: 8, eviction_watermark: 0.9, - probationary_fraction: 0.1, singleflight_shards: 64, - restart_scan_queue_depth: 256, btree_scratch_pages: 64, bypass_admission: false, bypass_index_read: false, @@ -781,9 +764,7 @@ impl StorageEngine { // Coalesce up to `commit_batch_max`, spending a bounded // budget of cooperative yields (`commit_batch_ticks`) // so producers that are about to enqueue join this - // commit. Wall-clock-free stand-in for - // `commit_batch_deadline_us`; see - // `MutatorQueue::drain_batch`. The drain stops early if + // commit. The drain stops early if // the queue closes, so a draining shutdown still // terminates. let batch = self diff --git a/cmd/unbounded-storage/src/storage/mutator.rs b/cmd/unbounded-storage/src/storage/mutator.rs index 66415088a..ed22a6ed0 100644 --- a/cmd/unbounded-storage/src/storage/mutator.rs +++ b/cmd/unbounded-storage/src/storage/mutator.rs @@ -227,12 +227,10 @@ impl MutatorQueue { /// as the batch reaches `max`, the queue is closed, or the /// yield budget is exhausted. /// - /// This is the wall-clock-free realization of - /// [`crate::storage::EngineConfig::commit_batch_deadline_us`]: - /// the DST harness forbids reading elapsed time, so the - /// deadline is expressed as a budget of `ticks` logical yields - /// (see [`crate::storage::EngineConfig::commit_batch_ticks`]). - /// Each [`yield_once`] gives the executor a chance to + /// The DST harness forbids reading elapsed time, so coalescing is + /// expressed as a budget of `ticks` logical yields (see + /// [`crate::storage::EngineConfig::commit_batch_ticks`]). Each + /// [`yield_once`] gives the executor a chance to /// interleave producers that are about to push, so batches /// grow under load while a drained or closed queue never /// stalls. `ticks == 0` disables coalescing (a single drain). From 9deea9dac730c11145f113aa086233344a642865 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:35:42 +0000 Subject: [PATCH 09/56] unbounded-storage: remove unused write depth hint --- .../src/storage/blockdev/core_local.rs | 24 +++++-------------- .../src/storage/blockdev/mock.rs | 6 ----- .../src/storage/blockdev/mod.rs | 5 ---- .../src/storage/blockdev/tests.rs | 1 - .../src/storage/blockdev/uring.rs | 3 +-- .../src/storage/btree/tests.rs | 4 ---- .../src/storage/disks/shard_view.rs | 3 --- cmd/unbounded-storage/tests/storage/mocks.rs | 4 ---- 8 files changed, 7 insertions(+), 43 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/blockdev/core_local.rs b/cmd/unbounded-storage/src/storage/blockdev/core_local.rs index c4fe119eb..ef19bdc02 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/core_local.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/core_local.rs @@ -35,24 +35,17 @@ pub struct CoreLocalDevice { file_index: u32, page_size: usize, capacity_pages: u64, - write_queue_depth: u32, } impl CoreLocalDevice { /// Build a device bound to `file_index` on the storage core's /// ring, carrying the disk geometry the engine needs without /// touching the ring. - pub fn new( - file_index: u32, - page_size: usize, - capacity_pages: u64, - write_queue_depth: u32, - ) -> Self { + pub fn new(file_index: u32, page_size: usize, capacity_pages: u64) -> Self { Self { file_index, page_size, capacity_pages, - write_queue_depth, } } @@ -106,10 +99,6 @@ impl BlockDevice for CoreLocalDevice { ring.write(self.file_index, offset, src).await } - fn write_queue_depth(&self) -> u32 { - self.write_queue_depth - } - fn progress(&self) -> Result<(), Error> { with_current_storage_ring(|ring| ring.progress()).unwrap_or_else(off_core) } @@ -169,7 +158,7 @@ mod tests { #[test] fn io_offset_validates_geometry() { - let dev = CoreLocalDevice::new(0, 4096, 16, 8); + let dev = CoreLocalDevice::new(0, 4096, 16); // Happy path: page-aligned, in range. assert_eq!(dev.io_offset(Lba(2), 4096).unwrap(), 8192); // Non-multiple of page size. @@ -198,7 +187,7 @@ mod tests { fn read_off_core_returns_enxio() { // Ensure no ring is installed on this test thread. clear_current_storage_ring(); - let dev = CoreLocalDevice::new(0, 4096, 16, 8); + let dev = CoreLocalDevice::new(0, 4096, 16); let mut buf = vec![0u8; 4096]; let res = block_on(dev.read(Lba(0), &mut buf)); assert!( @@ -210,7 +199,7 @@ mod tests { #[test] fn write_off_core_returns_enxio() { clear_current_storage_ring(); - let dev = CoreLocalDevice::new(0, 4096, 16, 8); + let dev = CoreLocalDevice::new(0, 4096, 16); let buf = vec![0u8; 4096]; let res = block_on(dev.write(Lba(0), &buf)); assert!( @@ -222,7 +211,7 @@ mod tests { #[test] fn register_and_progress_off_core_return_enxio() { clear_current_storage_ring(); - let dev = CoreLocalDevice::new(0, 4096, 16, 8); + let dev = CoreLocalDevice::new(0, 4096, 16); let mut buf = vec![0u8; 4096]; assert!(matches!( dev.register_buffers(buf.as_mut_ptr(), buf.len()), @@ -237,11 +226,10 @@ mod tests { #[test] fn metadata_accessors_do_not_touch_the_ring() { clear_current_storage_ring(); - let dev = CoreLocalDevice::new(3, 4096, 128, 32); + let dev = CoreLocalDevice::new(3, 4096, 128); assert_eq!(dev.file_index(), 3); assert_eq!(dev.page_size(), 4096); assert_eq!(dev.capacity_pages(), 128); - assert_eq!(dev.write_queue_depth(), 32); } #[test] diff --git a/cmd/unbounded-storage/src/storage/blockdev/mock.rs b/cmd/unbounded-storage/src/storage/blockdev/mock.rs index 6d73c480d..caa0ec151 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/mock.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/mock.rs @@ -34,7 +34,6 @@ pub enum MockFaultMode { pub struct MockDeviceConfig { pub page_size: usize, pub capacity_pages: u64, - pub write_queue_depth: u32, pub fault_mode: MockFaultMode, pub fault_errno: i32, } @@ -44,7 +43,6 @@ impl Default for MockDeviceConfig { Self { page_size: 4096, capacity_pages: 1024, - write_queue_depth: 32, fault_mode: MockFaultMode::None, fault_errno: libc_eio(), } @@ -151,10 +149,6 @@ impl BlockDevice for MockDevice { Ok(()) } - fn write_queue_depth(&self) -> u32 { - self.cfg.get().write_queue_depth - } - async fn read(&self, lba: Lba, dst: &mut [u8]) -> Result<(), Error> { self.read_count.set(self.read_count.get() + 1); let cfg = self.cfg.get(); diff --git a/cmd/unbounded-storage/src/storage/blockdev/mod.rs b/cmd/unbounded-storage/src/storage/blockdev/mod.rs index 29cbf8534..6ed403141 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/mod.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/mod.rs @@ -73,11 +73,6 @@ pub trait BlockDevice { /// semantics as [`BlockDevice::read`]. async fn write(&self, lba: Lba, src: &[u8]) -> Result<(), Error>; - /// Hint for how many concurrent writes the device can absorb - /// efficiently. The mutator uses this to size its commit - /// batches. - fn write_queue_depth(&self) -> u32; - /// Drive any submitted-but-not-yet-completed I/O forward. The /// io_uring backend pushes queued SQEs to the kernel and reaps /// available CQEs, waking the tasks awaiting them. Backends diff --git a/cmd/unbounded-storage/src/storage/blockdev/tests.rs b/cmd/unbounded-storage/src/storage/blockdev/tests.rs index 54f76807b..48e0d9357 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/tests.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/tests.rs @@ -234,7 +234,6 @@ mod uring_tests { let (device, _installed) = open_installed(&path.0, page_size); assert_eq!(device.page_size(), page_size); assert_eq!(device.capacity_pages(), pages); - assert_eq!(device.write_queue_depth(), 8); } #[test] diff --git a/cmd/unbounded-storage/src/storage/blockdev/uring.rs b/cmd/unbounded-storage/src/storage/blockdev/uring.rs index bdad8b6f7..8ed49986b 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/uring.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/uring.rs @@ -70,8 +70,7 @@ impl UringDevice { let file_index = ring .register_file(file.as_raw_fd()) .map_err(OpenError::RegisterFile)?; - let device = - CoreLocalDevice::new(file_index, page_size, capacity_pages, ring_cfg.queue_depth); + let device = CoreLocalDevice::new(file_index, page_size, capacity_pages); Ok(OpenDisk { device, ring, file }) } } diff --git a/cmd/unbounded-storage/src/storage/btree/tests.rs b/cmd/unbounded-storage/src/storage/btree/tests.rs index c2ee959c6..a0082b53c 100644 --- a/cmd/unbounded-storage/src/storage/btree/tests.rs +++ b/cmd/unbounded-storage/src/storage/btree/tests.rs @@ -859,10 +859,6 @@ impl BlockDevice for GateDevice { self.inner.register_buffers(base, len) } - fn write_queue_depth(&self) -> u32 { - self.inner.write_queue_depth() - } - async fn read(&self, lba: Lba, dst: &mut [u8]) -> Result<(), crate::storage::types::Error> { self.inner.read(lba, dst).await } diff --git a/cmd/unbounded-storage/src/storage/disks/shard_view.rs b/cmd/unbounded-storage/src/storage/disks/shard_view.rs index 52d82322a..f4d9d23d9 100644 --- a/cmd/unbounded-storage/src/storage/disks/shard_view.rs +++ b/cmd/unbounded-storage/src/storage/disks/shard_view.rs @@ -495,9 +495,6 @@ mod tests { self.registers.fetch_add(1, Ordering::Relaxed); self.inner.register_buffers(base, len) } - fn write_queue_depth(&self) -> u32 { - self.inner.write_queue_depth() - } async fn read(&self, lba: Lba, dst: &mut [u8]) -> Result<(), DevError> { self.inner.read(lba, dst).await } diff --git a/cmd/unbounded-storage/tests/storage/mocks.rs b/cmd/unbounded-storage/tests/storage/mocks.rs index 5c2d64e7a..ef42f6642 100644 --- a/cmd/unbounded-storage/tests/storage/mocks.rs +++ b/cmd/unbounded-storage/tests/storage/mocks.rs @@ -183,10 +183,6 @@ impl BlockDevice for SimBlockDevice { self.inner.register_buffers(base, len) } - fn write_queue_depth(&self) -> u32 { - self.inner.write_queue_depth() - } - async fn read(&self, lba: Lba, dst: &mut [u8]) -> Result<(), Error> { let delay = draw_delay(&self.cfg); let fault = draw_fault(&self.cfg); From 0fdfe22525752704a3aa2b3345fcb9c95a7aacb1 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 05:32:49 +0000 Subject: [PATCH 10/56] unbounded-storage: fold eviction metadata into SIEVE --- cmd/unbounded-storage/src/storage/engine.rs | 79 ++++++-------------- cmd/unbounded-storage/src/storage/lru/mod.rs | 67 +++++++++++------ 2 files changed, 67 insertions(+), 79 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index 5f675da27..ffc0c7310 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -24,7 +24,6 @@ //! [`crate::bufferpool::Pool`] can be configured against an //! NVMe-backed engine. -use std::collections::HashMap; use std::sync::{Arc, Mutex}; use crate::bufferpool::{self, PageRef, StripeKey}; @@ -33,7 +32,7 @@ use crate::storage::alloc::Allocator; use crate::storage::blockdev::{BlockDevice, ScratchPool}; use crate::storage::btree::{BTreeIndex, LeafEntry, Mutation}; use crate::storage::checksum::checksum; -use crate::storage::lru::SieveLru; +use crate::storage::lru::{Resident, SieveLru}; use crate::storage::mutator::{MutatorOutcome, MutatorQueue, MutatorReply, MutatorReq}; use crate::storage::refcount::RefcountTable; use crate::storage::singleflight::Singleflight; @@ -151,11 +150,6 @@ pub struct StorageEngine { singleflight: Singleflight, cfg: EngineConfig, bufferpool: Mutex, - /// Reverse map start-LBA -> (PageKey, byte_len). Eviction - /// targets pick a start LBA from the LRU; the entry's - /// byte_len tells the engine how many contiguous LBAs the - /// run covers so the whole run is freed atomically. - reverse: Mutex>, /// Runs that have been logically orphaned (no btree entry, /// removed from the LRU) but cannot be freed yet because a /// reader still holds a pin on the start LBA. Drained by the @@ -282,7 +276,6 @@ impl StorageEngine { page_size: 0, page_count: 0, }), - reverse: Mutex::new(HashMap::new()), pending_free: Mutex::new(Vec::new()), mutator_queue: Arc::new(MutatorQueue::new()), metrics: EngineMetrics::new(), @@ -385,10 +378,6 @@ impl StorageEngine { /// bookkeeping and either free it immediately or queue it for /// later reclamation when its readers are done. fn retire_range(&self, old: Lba, n: u64) { - { - let mut rev = self.reverse.lock().unwrap(); - rev.remove(&old.0); - } self.lru.forget(old); if self.refcount.pin_count(old.0).unwrap_or(0) == 0 { let _ = self.allocator.free_range(old, n); @@ -412,36 +401,18 @@ impl StorageEngine { // Skip pinned LBAs - an in-flight reader still // references them. Re-admit so they remain tracked // and are reconsidered on a later sweep. - let mut victims: Vec = Vec::with_capacity(candidates.len()); - for lba in candidates { - if self.refcount.pin_count(lba.0).unwrap_or(0) == 0 { - victims.push(lba); + let mut victims: Vec = Vec::with_capacity(candidates.len()); + for resident in candidates { + if self.refcount.pin_count(resident.lba.0).unwrap_or(0) == 0 { + victims.push(resident); } else { - self.lru.admit(lba); + self.lru.admit(resident); } } if victims.is_empty() { return; } - // Resolve victim start-LBAs to (key, byte_len) via - // the reverse map before submitting: the mutator only - // needs the keys to commit the delete, and we still - // own the LBA runs here for the post-commit free - // below. - let mut keys: Vec = Vec::with_capacity(victims.len()); - let mut victim_runs: Vec<(Lba, u64)> = Vec::with_capacity(victims.len()); - { - let rev = self.reverse.lock().unwrap(); - for lba in &victims { - if let Some((k, byte_len)) = rev.get(&lba.0).copied() { - keys.push(k); - victim_runs.push((*lba, self.n_pages(byte_len))); - } - } - } - if keys.is_empty() { - return; - } + let keys = victims.iter().map(|resident| resident.key).collect(); // The btree mutation MUST commit before the allocator // slot is released: a concurrent writer that observes // the stale btree entry would otherwise call @@ -457,24 +428,22 @@ impl StorageEngine { match done.wait().await { MutatorOutcome::DeleteCommitted => {} _ => { - // Apply failed or queue closed: leave the - // LBAs allocated and tracked. A later sweep - // will retry. + // Apply failed or queue closed: restore the + // resident records so a later sweep retries. + for resident in victims { + self.lru.admit(resident); + } return; } } - { - let mut rev = self.reverse.lock().unwrap(); - for (lba, _) in &victim_runs { - rev.remove(&lba.0); - } - } - for (lba, n) in &victim_runs { - let _ = self.allocator.free_range(*lba, *n); - let _ = self.refcount.reset(lba.0); + for resident in &victims { + let _ = self + .allocator + .free_range(resident.lba, self.n_pages(resident.byte_len)); + let _ = self.refcount.reset(resident.lba.0); } - self.metric(|m| m.evictions += victim_runs.len() as u64); - for _ in &victim_runs { + self.metric(|m| m.evictions += victims.len() as u64); + for _ in &victims { crate::metrics::storage_eviction(self.disk()); } } @@ -736,11 +705,11 @@ impl StorageEngine { self.retire_range(old.lba, self.n_pages(old.byte_len)); } - { - let mut rev = self.reverse.lock().unwrap(); - rev.insert(lba.0, (pk, src_buf.len() as u32)); - } - self.lru.admit(lba); + self.lru.admit(Resident { + lba, + key: pk, + byte_len: src_buf.len() as u32, + }); self.metric(|m| m.admitted += 1); self.evict_if_over_watermark().await; diff --git a/cmd/unbounded-storage/src/storage/lru/mod.rs b/cmd/unbounded-storage/src/storage/lru/mod.rs index 86a63020c..af1a95d05 100644 --- a/cmd/unbounded-storage/src/storage/lru/mod.rs +++ b/cmd/unbounded-storage/src/storage/lru/mod.rs @@ -23,11 +23,20 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use crate::storage::refcount::RefcountTable; -use crate::storage::types::Lba; +use crate::storage::types::{Lba, PageKey}; + +/// Metadata needed to evict one resident data run without a +/// separate reverse-LBA index. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Resident { + pub lba: Lba, + pub key: PageKey, + pub byte_len: u32, +} pub struct SieveLru { capacity: u64, - queue: Mutex>, + queue: Mutex>, refcount: Arc, } @@ -61,8 +70,8 @@ impl SieveLru { len / cap >= fraction } - pub fn admit(&self, lba: Lba) { - self.queue.lock().unwrap().push_front(lba); + pub fn admit(&self, resident: Resident) { + self.queue.lock().unwrap().push_front(resident); } pub fn touch(&self, lba: Lba) { @@ -76,23 +85,25 @@ impl SieveLru { /// victims. The sweep gives up after seeing twice the /// queue's length to avoid an unbounded loop when every /// resident page is pinned. - pub fn sweep(&self, target: usize) -> Vec { + pub fn sweep(&self, target: usize) -> Vec { let mut victims = Vec::with_capacity(target.min(64)); let mut q = self.queue.lock().unwrap(); let mut budget = q.len() * 2; while victims.len() < target && budget > 0 { budget -= 1; - let Some(lba) = q.pop_back() else { break }; - if self.refcount.is_pinned(lba.0).unwrap_or(false) { - q.push_front(lba); + let Some(resident) = q.pop_back() else { + break; + }; + if self.refcount.is_pinned(resident.lba.0).unwrap_or(false) { + q.push_front(resident); continue; } - if self.refcount.is_referenced(lba.0).unwrap_or(false) { - let _ = self.refcount.clear_referenced(lba.0); - q.push_front(lba); + if self.refcount.is_referenced(resident.lba.0).unwrap_or(false) { + let _ = self.refcount.clear_referenced(resident.lba.0); + q.push_front(resident); continue; } - victims.push(lba); + victims.push(resident); } victims } @@ -102,7 +113,7 @@ impl SieveLru { /// observed bitrot). pub fn forget(&self, lba: Lba) { let mut q = self.queue.lock().unwrap(); - if let Some(pos) = q.iter().position(|&x| x == lba) { + if let Some(pos) = q.iter().position(|resident| resident.lba == lba) { q.remove(pos); } } @@ -112,6 +123,14 @@ impl SieveLru { mod tests { use super::*; + fn resident(i: u64) -> Resident { + Resident { + lba: Lba(i), + key: PageKey::new([i as u8; 32], i as u32), + byte_len: 4096, + } + } + fn lru(capacity: u64) -> SieveLru { let rc = Arc::new(RefcountTable::new(capacity)); SieveLru::new(capacity, rc) @@ -121,10 +140,10 @@ mod tests { fn admit_then_sweep_returns_cold_pages_in_lru_order() { let l = lru(16); for i in 2..6 { - l.admit(Lba(i)); + l.admit(resident(i)); } let v = l.sweep(2); - assert_eq!(v, vec![Lba(2), Lba(3)]); + assert_eq!(v, vec![resident(2), resident(3)]); assert_eq!(l.len(), 2); } @@ -132,15 +151,15 @@ mod tests { fn touched_pages_survive_first_sweep() { let l = lru(16); for i in 2..6 { - l.admit(Lba(i)); + l.admit(resident(i)); } l.touch(Lba(2)); // Hand reaches Lba(2) first; bit set => clear+rotate. let v = l.sweep(1); - assert_eq!(v, vec![Lba(3)]); + assert_eq!(v, vec![resident(3)]); // Sweep again: now Lba(2) has bit cleared and gets evicted next. let v = l.sweep(1); - assert_eq!(v, vec![Lba(4)]); + assert_eq!(v, vec![resident(4)]); } #[test] @@ -148,11 +167,11 @@ mod tests { let l = lru(16); let rc = l.refcount.clone(); for i in 2..6 { - l.admit(Lba(i)); + l.admit(resident(i)); } let _g = rc.pin(2).unwrap(); let v = l.sweep(3); - assert_eq!(v, vec![Lba(3), Lba(4), Lba(5)]); + assert_eq!(v, vec![resident(3), resident(4), resident(5)]); // Lba(2) is still in the queue (skipped, re-enqueued). assert_eq!(l.len(), 1); } @@ -162,7 +181,7 @@ mod tests { let l = lru(10); assert!(!l.watermark_exceeded(0.9)); for i in 2..11 { - l.admit(Lba(i)); + l.admit(resident(i)); } assert!(l.watermark_exceeded(0.9)); } @@ -170,10 +189,10 @@ mod tests { #[test] fn forget_removes_lba() { let l = lru(8); - l.admit(Lba(2)); - l.admit(Lba(3)); + l.admit(resident(2)); + l.admit(resident(3)); l.forget(Lba(2)); assert_eq!(l.len(), 1); - assert_eq!(l.sweep(2), vec![Lba(3)]); + assert_eq!(l.sweep(2), vec![resident(3)]); } } From 8dade4260f739d04496a566cc9e8c2dced6a8a01 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:52:13 +0000 Subject: [PATCH 11/56] unbounded-storage: remove worker-local origin rings --- cmd/unbounded-storage/src/backend/azure.rs | 37 +--- cmd/unbounded-storage/src/backend/http.rs | 37 +--- cmd/unbounded-storage/src/backend/mod.rs | 6 - .../src/backend/origin_ring.rs | 182 ------------------ cmd/unbounded-storage/src/backend/registry.rs | 48 ++--- cmd/unbounded-storage/src/backend/s3.rs | 37 +--- cmd/unbounded-storage/src/main.rs | 4 +- 7 files changed, 42 insertions(+), 309 deletions(-) delete mode 100644 cmd/unbounded-storage/src/backend/origin_ring.rs diff --git a/cmd/unbounded-storage/src/backend/azure.rs b/cmd/unbounded-storage/src/backend/azure.rs index a544d1124..159fa2b2a 100644 --- a/cmd/unbounded-storage/src/backend/azure.rs +++ b/cmd/unbounded-storage/src/backend/azure.rs @@ -40,7 +40,6 @@ use super::conn::{ OriginConnPool, body_response_reusable, head_response_reusable, send_request_read_head, }; use super::limiter::FetchLimiter; -use super::origin_ring::OriginRing; /// The pinned `x-ms-version` REST API version sent on every Azure Blob /// request. Azure requires this header to select the wire semantics of @@ -51,14 +50,10 @@ const AZURE_MS_VERSION: &str = "2021-08-06"; /// Origin backend that fetches stripe byte ranges from an Azure Blob /// origin into bufferpool pages over plaintext HTTP/1.1. /// -/// Shard-pinned: the [`OriginRing`] and the raw `backing_base` pointer -/// are only ever touched on the owning shard thread that built this -/// backend, OR (for the RPC-handler instance) on the ephemeral -/// `fabric-rpc-worker` thread that uses an [`OriginRing::WorkerLocal`] -/// ring private to that thread. See the `unsafe impl Send + Sync` -/// below. +/// Shard-pinned: the [`NetHandle`] and raw `backing_base` pointer are +/// only ever touched on the owning shard thread that built this backend. pub struct AzureBackend { - ring: OriginRing, + handle: NetHandle, origin: SockAddr, /// The origin host used for the `Host:` header. The TCP connect /// uses `origin` (the resolved IPv4), but the storage account's @@ -79,7 +74,7 @@ pub struct AzureBackend { // SAFETY: mirrors `S3Backend`'s justification. `AzureBackend` is // shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `OriginRing`, any `Rc`/ +// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ // `RefCell` it holds, and the raw `backing_base` pointer are never // shared across threads at runtime. The `Send + Sync` marker exists // solely to satisfy the `Backend: Send + Sync` bound the embedder @@ -91,7 +86,7 @@ unsafe impl Sync for AzureBackend {} impl AzureBackend { #[allow(clippy::too_many_arguments)] pub fn new( - ring: OriginRing, + handle: NetHandle, origin: SockAddr, host: String, sni_host: String, @@ -103,7 +98,7 @@ impl AzureBackend { http_concurrency: usize, ) -> Self { Self { - ring, + handle, origin, host, sni_host, @@ -151,10 +146,7 @@ impl AzureBackend { let path = origin.origin_object_id.clone(); let dsts_owned = dsts.to_vec(); - let handle = match self.ring.handle() { - Ok(h) => h, - Err(e) => return AzureFetchStream::immediate_err(io_to_err(e)), - }; + let handle = self.handle.clone(); let origin_addr = self.origin; let backing_base = self.backing_base; let page_size = self.page_size; @@ -264,14 +256,6 @@ impl<'a> AzureFetchStream<'a> { next: 0, } } - - fn immediate_err(err: Error) -> Self { - Self { - state: FetchState::Failed(Some(err)), - delivered: Vec::new(), - next: 0, - } - } } impl PageStream for AzureFetchStream<'_> { @@ -784,13 +768,6 @@ fn format_head_request(path: &str, host: &str) -> Result, Error> { Ok(serialize_request(&req)) } -fn io_to_err(e: std::io::Error) -> Error { - match e.raw_os_error() { - Some(code) => Error::Io(code), - None => Error::transport(e), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/cmd/unbounded-storage/src/backend/http.rs b/cmd/unbounded-storage/src/backend/http.rs index 61689f63b..24c1e9e71 100644 --- a/cmd/unbounded-storage/src/backend/http.rs +++ b/cmd/unbounded-storage/src/backend/http.rs @@ -49,20 +49,15 @@ use super::conn::{ OriginConnPool, body_response_reusable, head_response_reusable, send_request_read_head, }; use super::limiter::FetchLimiter; -use super::origin_ring::OriginRing; /// Origin backend that fetches stripe byte ranges from an HTTP/1.1 /// origin server (plaintext `http://` or kernel-TLS `https://`) into /// bufferpool pages. /// -/// Shard-pinned: the [`OriginRing`] and the raw `backing_base` pointer -/// are only ever touched on the owning shard thread that built this -/// backend, OR (for the RPC-handler instance) on the ephemeral -/// `fabric-rpc-worker` thread that uses an [`OriginRing::WorkerLocal`] -/// ring private to that thread. Either way a single thread drives any -/// one ring; see the `unsafe impl Send + Sync` below. +/// Shard-pinned: the [`NetHandle`] and raw `backing_base` pointer are +/// only ever touched on the owning shard thread that built this backend. pub struct HttpBackend { - ring: OriginRing, + handle: NetHandle, origin: SockAddr, /// Authority sent in the `Host:` header (`host` or `host:port`), /// derived from the configured endpoint URL. @@ -83,7 +78,7 @@ pub struct HttpBackend { // SAFETY: mirrors `crate::memory::Backing`. `HttpBackend` is // shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `OriginRing`, any `Rc`/ +// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ // `RefCell` it holds, and the raw `backing_base` pointer are never // shared across threads at runtime. The `Send + Sync` marker exists // solely to satisfy the `Backend: Send + Sync` bound the embedder @@ -94,7 +89,7 @@ unsafe impl Sync for HttpBackend {} impl HttpBackend { pub fn new( - ring: OriginRing, + handle: NetHandle, origin: SockAddr, host: String, sni_host: String, @@ -106,7 +101,7 @@ impl HttpBackend { http_concurrency: usize, ) -> Self { Self { - ring, + handle, origin, host, sni_host, @@ -185,10 +180,7 @@ impl HttpBackend { let tls = self.tls.clone(); let dsts_owned = dsts.to_vec(); - let handle = match self.ring.handle() { - Ok(h) => h, - Err(e) => return HttpFetchStream::immediate_err(io_to_err(e)), - }; + let handle = self.handle.clone(); let origin_addr = self.origin; let backing_base = self.backing_base; let page_size = self.page_size; @@ -319,14 +311,6 @@ impl<'a> HttpFetchStream<'a> { next: 0, } } - - fn immediate_err(err: Error) -> Self { - Self { - state: FetchState::Failed(Some(err)), - delivered: Vec::new(), - next: 0, - } - } } impl PageStream for HttpFetchStream<'_> { @@ -860,13 +844,6 @@ fn format_head_request(path: &str, host: &str) -> Result, Error> { Ok(serialize_request(&req)) } -fn io_to_err(e: std::io::Error) -> Error { - match e.raw_os_error() { - Some(code) => Error::Io(code), - None => Error::transport(e), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/cmd/unbounded-storage/src/backend/mod.rs b/cmd/unbounded-storage/src/backend/mod.rs index 70603f101..51c644344 100644 --- a/cmd/unbounded-storage/src/backend/mod.rs +++ b/cmd/unbounded-storage/src/backend/mod.rs @@ -19,9 +19,6 @@ mod limiter; #[cfg(target_os = "linux")] mod origin; -#[cfg(target_os = "linux")] -mod origin_ring; - #[cfg(target_os = "linux")] mod registry; @@ -49,9 +46,6 @@ pub use limiter::{Acquire, FetchLimiter, FetchPermit}; #[cfg(target_os = "linux")] pub use origin::{OriginBackend, OriginStream}; -#[cfg(target_os = "linux")] -pub use origin_ring::{FixedRegion, OriginRing}; - #[cfg(target_os = "linux")] pub use registry::{BackendRegistry, RegistryFetchStream}; diff --git a/cmd/unbounded-storage/src/backend/origin_ring.rs b/cmd/unbounded-storage/src/backend/origin_ring.rs deleted file mode 100644 index a279dab1b..000000000 --- a/cmd/unbounded-storage/src/backend/origin_ring.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! Where an origin backend gets the [`NetworkRing`] it drives its -//! cache-miss fetches on. -//! -//! An origin fetch (a ranged GET or a length HEAD) dials the origin over -//! a [`NetworkRing`] and self-pumps via [`NetHandle`]'s `'static` -//! futures (each future calls `progress()` on its own ring inside -//! `poll`). Which ring a fetch may touch depends entirely on which -//! thread runs it, because a `NetworkRing` is single-threaded -//! (`Rc>`, `!Sync`): -//! -//! - [`OriginRing::Shard`]: the pool-transport backend runs on the -//! pinned shard thread. It reuses that shard's own socket ring, the -//! very ring the shard's tick hook progresses. All access is on one -//! thread, so the shared ring stays sound under cooperative -//! interleaving. -//! - [`OriginRing::WorkerLocal`]: the RPC handler's backend serves a -//! peer cache-miss on an ephemeral `fabric-rpc-worker` thread (one OS -//! thread per inbound RPC; see `fabric::rpc`). It must NOT touch the -//! shard ring: the shard thread progresses that ring concurrently, and -//! a cross-thread borrow of its `RefCell` panics with "already -//! borrowed". Instead each worker thread lazily builds its OWN ring in -//! thread-local storage and drives the fetch entirely there. -//! -//! The worker-local ring registers the per-thread scratch backing as -//! fixed buffer index 0 (via [`FixedRegion`]) so an origin fetch can -//! `recv_fixed` response bytes directly into scratch pages with no -//! intermediate heap copy. The region is threaded through [`OriginRing`] -//! as a `(base, len)` pair because the scratch `Backing`'s Drop carrier -//! cannot be re-synthesized from raw parts across the thread boundary. - -use std::cell::RefCell; -use std::io; -use std::rc::Rc; - -use crate::ring::{NetHandle, NetworkRing}; - -/// A raw fixed-buffer region (`base`, `len`) to register as fixed buffer -/// index 0 on a worker-local ring. `len` is `page_size * page_count` of -/// the scratch backing. Carried by value (not as a `&Backing`) so it can -/// cross into the worker thread without the backing's Drop carrier. -#[derive(Clone, Copy)] -pub struct FixedRegion { - pub base: *mut u8, - pub len: usize, -} - -/// Source of the [`NetHandle`] an origin backend uses for one fetch. -/// -/// Cloned into the backend at construction. [`OriginRing::handle`] -/// resolves it to a concrete handle on the calling thread; see the -/// module docs for why the two variants exist. -#[derive(Clone)] -pub enum OriginRing { - /// Reuse the shard's socket ring. Sound only on the shard thread. - Shard(Rc>), - /// Use (and lazily create) a ring private to the current thread, - /// built with `queue_depth` submission/completion slots. When - /// `region` is `Some`, it is registered as fixed buffer index 0 on - /// first use so origin fetches can `recv_fixed` into scratch pages. - WorkerLocal { - queue_depth: u32, - region: Option, - }, -} - -impl OriginRing { - /// Obtain a [`NetHandle`] for issuing origin ops on the current - /// thread. - /// - /// `Shard` clones the shard ring. `WorkerLocal` returns a handle - /// over this thread's ring, building it on first use; the ring - /// lives in thread-local storage and is reused by every fetch that - /// later runs on the same thread. - pub fn handle(&self) -> io::Result { - match self { - OriginRing::Shard(ring) => Ok(NetHandle::new(Rc::clone(ring))), - OriginRing::WorkerLocal { - queue_depth, - region, - } => worker_local_handle(*queue_depth, *region), - } - } -} - -thread_local! { - /// The current thread's private origin ring, created on first - /// [`OriginRing::WorkerLocal`] use. An ephemeral `fabric-rpc-worker` - /// thread drops it when the thread exits; a reused worker thread - /// keeps it for subsequent requests. - static WORKER_RING: RefCell>>> = const { RefCell::new(None) }; -} - -fn worker_local_handle(queue_depth: u32, region: Option) -> io::Result { - WORKER_RING.with(|cell| { - if let Some(ring) = cell.borrow().as_ref() { - return Ok(NetHandle::new(Rc::clone(ring))); - } - let net = NetworkRing::new(queue_depth)?; - if let Some(region) = region { - net.register_region(region.base, region.len)?; - } - let ring = Rc::new(RefCell::new(net)); - *cell.borrow_mut() = Some(Rc::clone(&ring)); - Ok(NetHandle::new(ring)) - }) -} - -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use super::*; - - /// A `Shard` ring hands back a handle over the very ring it was - /// built from: the handle's `Rc` shares the same allocation, so the - /// shard thread and the backend drive one ring. - #[test] - fn shard_handle_shares_the_shard_ring() { - let ring = Rc::new(RefCell::new(NetworkRing::new(8).expect("ring"))); - let origin = OriginRing::Shard(Rc::clone(&ring)); - // Two handles drawn from the same shard source must alias the - // same ring allocation as the original `Rc`. - let before = Rc::strong_count(&ring); - let _h1 = origin.handle().expect("handle"); - let _h2 = origin.handle().expect("handle"); - assert!( - Rc::strong_count(&ring) > before, - "shard handles must clone the shared ring Rc" - ); - } - - /// A `WorkerLocal` ring is private to the thread that asks for it: - /// repeated calls on one thread reuse a single ring, while a second - /// thread builds an entirely separate one. This is the invariant - /// that keeps the RPC worker off the shard ring. - #[test] - fn worker_local_ring_is_one_per_thread() { - let queue_depth = 8; - let origin = OriginRing::WorkerLocal { - queue_depth, - region: None, - }; - - // Same thread: the second call reuses the cached ring, so the - // handles point at the same allocation. - let h1 = origin.handle().expect("handle"); - let h2 = origin.handle().expect("handle"); - assert!( - h1.same_ring(&h2), - "repeated WorkerLocal handles on one thread must share a ring" - ); - - // A different thread builds its own ring, distinct from this - // thread's. `OriginRing` is intentionally `!Send` (the `Shard` - // variant holds an `Rc`), so the worker rebuilds its source from - // the `Copy` queue depth rather than moving the enum across the - // boundary. - let joined = std::thread::spawn(move || { - let other = OriginRing::WorkerLocal { - queue_depth, - region: None, - }; - let a = other.handle().expect("handle"); - let b = other.handle().expect("handle"); - assert!(a.same_ring(&b), "worker thread must reuse its own ring"); - // Hand back the pointer identity so the parent can assert - // the two threads did not share a ring. - a.ring_addr() - }) - .join() - .expect("worker thread"); - - assert_ne!( - h1.ring_addr(), - joined, - "distinct threads must not share a WorkerLocal ring" - ); - } -} diff --git a/cmd/unbounded-storage/src/backend/registry.rs b/cmd/unbounded-storage/src/backend/registry.rs index 45e132b64..03730b292 100644 --- a/cmd/unbounded-storage/src/backend/registry.rs +++ b/cmd/unbounded-storage/src/backend/registry.rs @@ -31,23 +31,22 @@ use arc_swap::ArcSwap; use crate::bufferpool::{BulkRef, Error, PageRef, PageStream}; use crate::config::reconcile::BackendReconcileTarget; use crate::config::{BackendSpec, backend_spec}; +use crate::ring::NetHandle; use crate::storage::StripeReq; use crate::tls::{TlsConfig, TlsContext}; use super::url::parse_endpoint; use super::{ - AzureBackend, Backend, FakeBackend, HttpBackend, OriginBackend, OriginRing, OriginStream, - S3Backend, + AzureBackend, Backend, FakeBackend, HttpBackend, OriginBackend, OriginStream, S3Backend, }; /// The build context a registry needs to (re)construct an /// [`OriginBackend`] from a [`BackendSpec`]. Captured once per registry -/// instance because the pool transport and the RPC handler each own a -/// registry that differs only in which ring a cache-miss fetch drives -/// and which registered region origin bytes land in. +/// instance because every shard has its own network ring and registered +/// backing region. #[derive(Clone)] struct BuildCtx { - ring: OriginRing, + handle: NetHandle, page_size: usize, backing_base: *mut u8, } @@ -64,28 +63,27 @@ pub struct BackendRegistry { // SAFETY: shard-pinned exactly like the `OriginBackend`s it holds (see // `backend::http`/`backend::s3`). The `*mut u8` build base and the -// `Rc`-bearing `OriginRing::Shard` are only ever touched on the owning -// shard thread or the ephemeral `fabric-rpc-worker` thread driving a -// `WorkerLocal` ring; the marker only lets the registry live in the +// `Rc`-bearing `NetHandle` are only ever touched on the owning shard +// thread; the marker only lets the registry live in the // `Send + Sync` generic slots the transport/handler require. It is not // safe to move a registry to an unrelated thread. unsafe impl Send for BackendRegistry {} unsafe impl Sync for BackendRegistry {} impl BackendRegistry { - /// Build a registry seeded from `specs`, fetching through `ring` + /// Build a registry seeded from `specs`, fetching through `handle` /// into the region based at `backing_base`. A spec that fails to /// build (e.g. a URL that does not resolve) aborts the seed /// with the error, because startup must fail loudly rather than /// silently drop a configured backend. pub fn new( specs: &[BackendSpec], - ring: OriginRing, + handle: NetHandle, page_size: usize, backing_base: *mut u8, ) -> io::Result { let ctx = BuildCtx { - ring, + handle, page_size, backing_base, }; @@ -141,7 +139,7 @@ impl BuildCtx { )?; let origin = HttpBackend::resolve_origin(&endpoint.authority)?; Ok(OriginBackend::Http(HttpBackend::new( - self.ring.clone(), + self.handle.clone(), origin, endpoint.host, endpoint.sni_host, @@ -163,7 +161,7 @@ impl BuildCtx { )?; let origin = S3Backend::resolve_origin(&endpoint.authority)?; Ok(OriginBackend::S3(S3Backend::new( - self.ring.clone(), + self.handle.clone(), origin, endpoint.host, endpoint.sni_host, @@ -185,7 +183,7 @@ impl BuildCtx { )?; let origin = AzureBackend::resolve_origin(&endpoint.authority)?; Ok(OriginBackend::Azure(AzureBackend::new( - self.ring.clone(), + self.handle.clone(), origin, endpoint.host, endpoint.sni_host, @@ -343,10 +341,13 @@ impl PageStream for RegistryFetchStream<'_> { #[cfg(test)] mod tests { + use std::cell::RefCell; use std::ptr; + use std::rc::Rc; use std::task::{RawWaker, RawWakerVTable, Waker}; use crate::config::{FakeBackendConfig, HttpBackendConfig, backend_spec}; + use crate::ring::NetworkRing; use super::*; @@ -375,21 +376,10 @@ mod tests { } } - /// A registry over a worker-local ring with no fixed region and a - /// null backing base. Sound for tests that never drive a fetch (they - /// only exercise the id map and the unknown-backend path); the ring - /// is built lazily on first `handle()`, which these tests never hit. fn registry(specs: &[BackendSpec]) -> BackendRegistry { - BackendRegistry::new( - specs, - OriginRing::WorkerLocal { - queue_depth: 1, - region: None, - }, - 4096, - ptr::null_mut(), - ) - .expect("seed registry") + let ring = Rc::new(RefCell::new(NetworkRing::new(8).expect("network ring"))); + BackendRegistry::new(specs, NetHandle::new(ring), 4096, ptr::null_mut()) + .expect("seed registry") } fn noop_waker() -> Waker { diff --git a/cmd/unbounded-storage/src/backend/s3.rs b/cmd/unbounded-storage/src/backend/s3.rs index c8d7170f2..974246ea7 100644 --- a/cmd/unbounded-storage/src/backend/s3.rs +++ b/cmd/unbounded-storage/src/backend/s3.rs @@ -43,21 +43,16 @@ use super::conn::{ OriginConnPool, body_response_reusable, head_response_reusable, send_request_read_head, }; use super::limiter::FetchLimiter; -use super::origin_ring::OriginRing; /// Origin backend that fetches stripe byte ranges from an /// S3-compatible origin into bufferpool pages. The endpoint scheme /// selects plaintext HTTP/1.1 (`http://`) or TLS 1.3 with kTLS /// (`https://`); see the module docs. /// -/// Shard-pinned: the [`OriginRing`] and the raw `backing_base` pointer -/// are only ever touched on the owning shard thread that built this -/// backend, OR (for the RPC-handler instance) on the ephemeral -/// `fabric-rpc-worker` thread that uses an [`OriginRing::WorkerLocal`] -/// ring private to that thread. See the `unsafe impl Send + Sync` -/// below. +/// Shard-pinned: the [`NetHandle`] and raw `backing_base` pointer are +/// only ever touched on the owning shard thread that built this backend. pub struct S3Backend { - ring: OriginRing, + handle: NetHandle, origin: SockAddr, /// The origin hostname used for the `Host:` header. The TCP connect /// uses `origin` (the resolved IPv4), but the bucket's virtual-host @@ -79,7 +74,7 @@ pub struct S3Backend { // SAFETY: mirrors `HttpBackend`'s justification. `S3Backend` is // shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `OriginRing`, any `Rc`/ +// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ // `RefCell` it holds, and the raw `backing_base` pointer are never // shared across threads at runtime. The `Send + Sync` marker exists // solely to satisfy the `Backend: Send + Sync` bound the embedder @@ -91,7 +86,7 @@ unsafe impl Sync for S3Backend {} impl S3Backend { #[allow(clippy::too_many_arguments)] pub fn new( - ring: OriginRing, + handle: NetHandle, origin: SockAddr, host: String, sni_host: String, @@ -103,7 +98,7 @@ impl S3Backend { http_concurrency: usize, ) -> Self { Self { - ring, + handle, origin, host, sni_host, @@ -151,10 +146,7 @@ impl S3Backend { let path = origin.origin_object_id.clone(); let dsts_owned = dsts.to_vec(); - let handle = match self.ring.handle() { - Ok(h) => h, - Err(e) => return S3FetchStream::immediate_err(io_to_err(e)), - }; + let handle = self.handle.clone(); let origin_addr = self.origin; let backing_base = self.backing_base; let page_size = self.page_size; @@ -282,14 +274,6 @@ impl<'a> S3FetchStream<'a> { next: 0, } } - - fn immediate_err(err: Error) -> Self { - Self { - state: FetchState::Failed(Some(err)), - delivered: Vec::new(), - next: 0, - } - } } impl PageStream for S3FetchStream<'_> { @@ -798,13 +782,6 @@ fn format_head_request(path: &str, host: &str) -> Result, Error> { Ok(serialize_request(&req)) } -fn io_to_err(e: std::io::Error) -> Error { - match e.raw_os_error() { - Some(code) => Error::Io(code), - None => Error::transport(e), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index df2034d64..be3991d9d 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -15,7 +15,7 @@ use std::time::Duration; use clap::Parser; -use unbounded_storage::backend::{BackendRegistry, OriginRing}; +use unbounded_storage::backend::BackendRegistry; use unbounded_storage::bufferpool::{Pool, PoolConfig}; use unbounded_storage::config::{ self, BackendSpec, Config, FrontendSpec, ResolvedFrontendBinding, frontend_spec, @@ -672,7 +672,7 @@ fn run_shard( // shard socket ring and writes origin bytes into the pool backing. let transport_registry = match BackendRegistry::new( &backend_specs, - OriginRing::Shard(socket.clone()), + NetHandle::new(socket.clone()), page_size, backing_base, ) { From fd5fbe02467cd2cad19cc2c6899a32c2fc9184a2 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 20:20:44 +0000 Subject: [PATCH 12/56] unbounded-storage: remove legacy routing API --- cmd/unbounded-storage/ARCHITECTURE.md | 2 +- cmd/unbounded-storage/src/config/control.rs | 2 +- cmd/unbounded-storage/src/main.rs | 4 +- .../src/p2p/finger_router.rs | 86 ++++++++----------- cmd/unbounded-storage/src/p2p/handler.rs | 73 +--------------- cmd/unbounded-storage/src/p2p/mod.rs | 4 +- .../src/p2p/routing_handle.rs | 76 ++++++---------- cmd/unbounded-storage/src/p2p/transport.rs | 78 +---------------- 8 files changed, 77 insertions(+), 248 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 499d7a491..b59291e81 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -369,7 +369,7 @@ between them, and the origin `Backend` is the final fallback. - `RoutedTransport>` (the client side) makes the first-hop decision via a single Chord `next_hop(stripe_to_ring(key))`: - `None` -> this node owns the stripe; serve from the local origin `Backend`. - - `Some(peer)` -> hand off to a wrapped `FabricTransport` + - `Some(peer)` -> hand off to a wrapped `FabricTransport` with a `MAX_HOPS` TTL; recursion happens server-side. - `RecursiveHandler` (the server side) **resolves** every request (in contrast to `fabric::PoolHandler`, which only serves locally resident pages). It diff --git a/cmd/unbounded-storage/src/config/control.rs b/cmd/unbounded-storage/src/config/control.rs index 374a93472..69a189cd1 100644 --- a/cmd/unbounded-storage/src/config/control.rs +++ b/cmd/unbounded-storage/src/config/control.rs @@ -14,7 +14,7 @@ //! Every applicable change is realized in place: peers, disks, routing, //! backends, and frontends are all reconciled on the live shard layer //! without ever tearing it down or restarting the process. Routing is -//! republished through the shared [`RoutingHandle`], and each shard +//! republished through the shared [`RouteTableHandle`], and each shard //! reconciles its own backend and frontend registries from the //! broadcast config on its own thread. Startup-fixed knobs (the //! `[startup]` section: memory, fabric thread/endpoint/in-flight, and diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 0d7b8eca0..e4176ba6b 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -701,7 +701,7 @@ fn run_shard( Ok(t) => t, Err(e) => { let _ = tx.send(ShardReady::Failed(format!( - "worker={}: RoutedTransport::new: {e}", + "worker={}: RoutedTransport::with_routes: {e}", widx.0, ))); return; @@ -887,7 +887,7 @@ fn run_shard( // Control-drain tick hook: applies live config changes on this // shard's own thread so all `!Send` per-shard state stays // thread-local. Each `ShardCommand::ApplyConfig` republishes the - // routing surface through this shard's `RoutingHandle` (observed + // routing surface through this shard's `RouteTableHandle` (observed // atomically by its transport; the fabric RPC handlers are reloaded // separately by the `FabricGroup`), refreshes the stripe geometry, // reconciles the transport origin-backend registry and the frontend diff --git a/cmd/unbounded-storage/src/p2p/finger_router.rs b/cmd/unbounded-storage/src/p2p/finger_router.rs index 22efcdc60..0bf8283aa 100644 --- a/cmd/unbounded-storage/src/p2p/finger_router.rs +++ b/cmd/unbounded-storage/src/p2p/finger_router.rs @@ -11,60 +11,21 @@ //! node->peer map so ownership decisions stay consistent across the //! transport, the local-ownership pre-check, and the forwarding path. -use std::collections::HashMap; -use std::sync::Arc; - use crate::bufferpool::Req; use crate::fabric::{PeerId, PeerRouter}; -use crate::p2p::{FingerTable, NodeId, RouteTableHandle, RoutingHandle, stripe_to_ring}; - -/// `PeerRouter` backed by a finger table. Maps a request's stripe to -/// the owning peer's `PeerId` via the Chord `next_hop` lookup, then -/// the `node -> peer` map. The routing surface lives behind a shared -/// [`RoutingHandle`] so a peer-set change republished through that -/// handle is observed here without rebuilding the router. -pub struct FingerRouter { - routing: RoutingHandle, -} +use crate::p2p::{RouteTableHandle, stripe_to_ring}; +/// Routes requests through the finger table for their cache ID. pub struct ChainFingerRouter { routes: RouteTableHandle, } -impl FingerRouter { - /// Build a router over a freshly-seeded [`RoutingHandle`]. Retained - /// for callers (and tests) that hold the two maps directly; the - /// resulting router owns its own handle and is not live-reloadable - /// from elsewhere. - pub fn new(fingers: Arc, node_to_peer: Arc>) -> Self { - Self::from_handle(RoutingHandle::new(fingers, node_to_peer)) - } - - /// Build a router that shares an existing [`RoutingHandle`]. A - /// `store` on any clone of `routing` is observed by this router's - /// next `route` call, which is how a shard fans a peer-set reload - /// out to every consumer at once. - pub fn from_handle(routing: RoutingHandle) -> Self { - Self { routing } - } -} - impl ChainFingerRouter { pub fn new(routes: RouteTableHandle) -> Self { Self { routes } } } -impl PeerRouter for FingerRouter { - fn route(&self, req: &R) -> Option { - let target = stripe_to_ring(req.key()); - let snap = self.routing.snapshot(); - snap.fingers - .next_hop(target) - .and_then(|pe| snap.node_to_peer.get(&pe.node).copied()) - } -} - impl PeerRouter for ChainFingerRouter { fn route(&self, req: &R) -> Option { let snap = self.routes.route_for_req(req)?; @@ -86,13 +47,20 @@ mod tests { stripe_to_ring, }; - use super::FingerRouter; + use super::ChainFingerRouter; - struct TestReq(StripeKey); + struct TestReq { + key: StripeKey, + cache_id: String, + } impl Req for TestReq { fn key(&self) -> StripeKey { - self.0 + self.key + } + + fn cache_id(&self) -> Option<&String> { + Some(&self.cache_id) } } @@ -116,6 +84,26 @@ mod tests { Arc::new(nodes.iter().map(|&n| (NodeId(n), PeerId(n))).collect()) } + fn router( + fingers: Arc, + node_to_peer: Arc>, + ) -> ChainFingerRouter { + ChainFingerRouter::new(crate::p2p::RouteTableHandle::new(HashMap::from([( + "cache-a".to_string(), + crate::p2p::RoutingSnapshot { + fingers, + node_to_peer, + }, + )]))) + } + + fn req(key: StripeKey) -> TestReq { + TestReq { + key, + cache_id: "cache-a".to_string(), + } + } + #[test] fn key_for_ring_round_trips() { assert_eq!(stripe_to_ring(key_for_ring(12345)), RingId(12345)); @@ -124,7 +112,7 @@ mod tests { #[test] fn local_owned_stripe_routes_to_backend_and_router_returns_none() { // Single-node table: local owns everything, so next_hop is - // always None for any target. FingerRouter therefore yields + // always None for any target. ChainFingerRouter therefore yields // no peer (backend arm). let local = peer(1); let fingers = Arc::new(FingerTable::build( @@ -134,11 +122,11 @@ mod tests { )); let node_to_peer = node_to_peer_map(&[1]); - let router = FingerRouter::new(fingers.clone(), node_to_peer); + let router = router(fingers.clone(), node_to_peer); // Pick a target that is not the local ring id so the owner // check still exercises the predecessor path (single node // owns everything regardless). - let req = TestReq(key_for_ring(local.ring.0.wrapping_add(999))); + let req = req(key_for_ring(local.ring.0.wrapping_add(999))); assert!(fingers.next_hop(stripe_to_ring(req.key())).is_none()); assert!(PeerRouter::::route(&router, &req).is_none()); } @@ -156,10 +144,10 @@ mod tests { FingerTableConfig::with_k(8), )); let node_to_peer = node_to_peer_map(&[1, 2]); - let router = FingerRouter::new(fingers.clone(), node_to_peer); + let router = router(fingers.clone(), node_to_peer); // Target == other's ring id: local forwards to other. - let req = TestReq(key_for_ring(other.ring.0)); + let req = req(key_for_ring(other.ring.0)); let hop = fingers .next_hop(stripe_to_ring(req.key())) .expect("local should forward to a peer"); diff --git a/cmd/unbounded-storage/src/p2p/handler.rs b/cmd/unbounded-storage/src/p2p/handler.rs index 90e28f9bd..da92b498f 100644 --- a/cmd/unbounded-storage/src/p2p/handler.rs +++ b/cmd/unbounded-storage/src/p2p/handler.rs @@ -15,7 +15,6 @@ //! asks the next hop to RDMA-write into scratch, then relays that scratch //! page upstream. -use std::collections::HashMap; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -25,14 +24,10 @@ use serde::de::DeserializeOwned; use crate::bufferpool::{BulkRef, Error as PoolError, PageRef, PageStream, Req, StripeKey}; use crate::fabric::scratch::ScratchBacking; -use crate::fabric::{ - Fabric, FabricPage, FabricTransport, Handler, HandlerStream, MrHandle, PeerId, -}; +use crate::fabric::{Fabric, FabricPage, FabricTransport, Handler, HandlerStream, MrHandle}; use crate::fanout::{FetchChannel, FetchEvent, FetchStream}; use crate::memory::Backing; -use crate::p2p::{ - ChainFingerRouter, FingerTable, NodeId, RingId, RouteTableHandle, RoutingHandle, stripe_to_ring, -}; +use crate::p2p::{ChainFingerRouter, FingerTable, RingId, RouteTableHandle, stripe_to_ring}; use crate::runtime::{block_on_cooperative, noop_waker}; use crate::storage::disks::CacheDirectorySet; use crate::storage::{StripeReq, disk_for}; @@ -178,68 +173,6 @@ pub struct RecursiveHandler { } impl RecursiveHandler { - /// Build a recursive handler over a freshly-seeded routing surface. - #[allow(clippy::too_many_arguments)] - pub fn new( - scratch: Backing, - scratch_pages: u32, - fingers: Arc, - node_to_peer: Arc>, - fabric: Arc, - scratch_mr: MrHandle, - page_size: usize, - owners: OwnerShardTable, - ) -> crate::fabric::Result { - let mut routes = HashMap::new(); - routes.insert( - RouteTableHandle::LEGACY_ROUTE_ID.to_string(), - crate::p2p::RoutingSnapshot { - fingers, - node_to_peer, - }, - ); - Self::with_routes( - scratch, - scratch_pages, - RouteTableHandle::new(routes), - fabric, - scratch_mr, - page_size, - owners, - ) - } - - /// Build a recursive handler that shares `routing` with other - /// consumers. - pub fn with_routing( - scratch: Backing, - scratch_pages: u32, - routing: RoutingHandle, - fabric: Arc, - scratch_mr: MrHandle, - page_size: usize, - owners: OwnerShardTable, - ) -> crate::fabric::Result { - let snap = routing.snapshot(); - let mut routes = HashMap::new(); - routes.insert( - RouteTableHandle::LEGACY_ROUTE_ID.to_string(), - crate::p2p::RoutingSnapshot { - fingers: snap.fingers.clone(), - node_to_peer: snap.node_to_peer.clone(), - }, - ); - Self::with_routes( - scratch, - scratch_pages, - RouteTableHandle::new(routes), - fabric, - scratch_mr, - page_size, - owners, - ) - } - /// Build a recursive handler over an existing route table. pub fn with_routes( scratch: Backing, @@ -558,7 +491,7 @@ fn block_on_local(fut: F) -> F::Output { #[cfg(test)] mod tests { use super::*; - use crate::p2p::{FingerTableConfig, PeerEntry, TopologyTags, node_to_ring}; + use crate::p2p::{FingerTableConfig, NodeId, PeerEntry, TopologyTags, node_to_ring}; const PAGE: usize = 4096; const SCRATCH_PAGES: usize = 4; diff --git a/cmd/unbounded-storage/src/p2p/mod.rs b/cmd/unbounded-storage/src/p2p/mod.rs index 66c1d4c06..f488c91f2 100644 --- a/cmd/unbounded-storage/src/p2p/mod.rs +++ b/cmd/unbounded-storage/src/p2p/mod.rs @@ -21,7 +21,7 @@ mod types; #[cfg(test)] mod tests; -pub use finger_router::{ChainFingerRouter, FingerRouter}; +pub use finger_router::ChainFingerRouter; pub use fingers::{ FingerTable, FingerTableConfig, TopologyPrefixWeight, TopologySelection, TopologyWeighting, }; @@ -30,6 +30,6 @@ pub use handler::{ RecursiveHandlerStream, }; pub use ring::{node_id_from_name, node_to_ring, splitmix64, stripe_to_ring}; -pub use routing_handle::{RouteTableHandle, RouteTableSnapshot, RoutingHandle, RoutingSnapshot}; +pub use routing_handle::{RouteTableHandle, RouteTableSnapshot, RoutingSnapshot}; pub use transport::{RoutedStream, RoutedTransport}; pub use types::{NodeId, P2pReq, PeerEntry, RingId, TopologyTags}; diff --git a/cmd/unbounded-storage/src/p2p/routing_handle.rs b/cmd/unbounded-storage/src/p2p/routing_handle.rs index 2c82af1a0..e0df0172d 100644 --- a/cmd/unbounded-storage/src/p2p/routing_handle.rs +++ b/cmd/unbounded-storage/src/p2p/routing_handle.rs @@ -1,25 +1,24 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Live-reloadable routing surface shared across a shard's p2p -//! consumers. +//! Live-reloadable route table shared across p2p consumers. //! //! The finger table and the `node -> peer` map are derived together //! from the active cache keyspace projected by the config graph. Three independent //! consumers read them on the hot path: //! -//! * [`crate::p2p::FingerRouter`] (the first-hop lookup wrapped by +//! * [`crate::p2p::ChainFingerRouter`] (the first-hop lookup wrapped by //! every [`crate::fabric::FabricTransport`]), //! * [`crate::p2p::RoutedTransport`] (its local-ownership pre-check), //! * [`crate::p2p::RecursiveHandler`] (its server-side classify and //! forward path). //! //! When peers change, all three must observe the new surface -//! atomically and without a restart. [`RoutingHandle`] is the seam: +//! atomically and without a restart. [`RouteTableHandle`] is the seam: //! a cheaply-clonable handle over a single [`ArcSwap`] of a -//! [`RoutingSnapshot`]. Every consumer holds a clone of the SAME -//! handle, so one [`RoutingHandle::store`] publishes a new snapshot to -//! all of them at once. The swap is wait-free for readers, which +//! [`RouteTableSnapshot`]. Every consumer holds a clone of the same +//! handle, so one [`RouteTableHandle::store`] publishes all cache routes +//! at once. The swap is wait-free for readers, which //! matters because the [`RecursiveHandler`] is read concurrently by //! the fabric RPC worker threads while the shard thread performs the //! store. @@ -51,45 +50,11 @@ pub struct RouteTableSnapshot { pub routes: HashMap, } -/// Shared, live-reloadable handle to a [`RoutingSnapshot`]. -/// -/// Cloning is cheap (an `Arc` bump) and every clone observes stores -/// made through any other clone. This is the single seam through -/// which a shard republishes routing after a peer-set change. -#[derive(Clone)] -pub struct RoutingHandle(Arc>); - +/// Shared, live-reloadable handle to cache-keyed routing snapshots. #[derive(Clone)] pub struct RouteTableHandle(Arc>); -impl RoutingHandle { - /// Build a handle seeded with the initial routing surface. - pub fn new(fingers: Arc, node_to_peer: Arc>) -> Self { - Self(Arc::new(ArcSwap::from_pointee(RoutingSnapshot { - fingers, - node_to_peer, - }))) - } - - /// Atomically publish a new routing surface to every clone of this - /// handle. Wait-free for concurrent readers. - pub fn store(&self, fingers: Arc, node_to_peer: Arc>) { - self.0.store(Arc::new(RoutingSnapshot { - fingers, - node_to_peer, - })); - } - - /// Load the current snapshot. The two maps are guaranteed mutually - /// consistent (published together by [`Self::store`]). - pub fn snapshot(&self) -> Arc { - self.0.load_full() - } -} - impl RouteTableHandle { - pub const LEGACY_ROUTE_ID: &'static str = ""; - pub fn new(routes: HashMap) -> Self { Self(Arc::new(ArcSwap::from_pointee(RouteTableSnapshot { routes, @@ -160,14 +125,27 @@ mod tests { // every clone shares one ArcSwap, so this is the property the // shard relies on to fan a reload out to consumers it already // handed clones to. - let handle = RoutingHandle::new(table(1, &[]), map(&[1])); + let handle = RouteTableHandle::new(HashMap::from([( + "cache-a".to_string(), + RoutingSnapshot { + fingers: table(1, &[]), + node_to_peer: map(&[1]), + }, + )])); let consumer = handle.clone(); - assert!(consumer.snapshot().node_to_peer.contains_key(&NodeId(1))); - assert!(!consumer.snapshot().node_to_peer.contains_key(&NodeId(2))); - - handle.store(table(1, &[2]), map(&[1, 2])); - - let snap = consumer.snapshot(); + let before = consumer.route("cache-a").expect("route"); + assert!(before.node_to_peer.contains_key(&NodeId(1))); + assert!(!before.node_to_peer.contains_key(&NodeId(2))); + + handle.store(HashMap::from([( + "cache-a".to_string(), + RoutingSnapshot { + fingers: table(1, &[2]), + node_to_peer: map(&[1, 2]), + }, + )])); + + let snap = consumer.route("cache-a").expect("route"); assert!(snap.node_to_peer.contains_key(&NodeId(2))); assert_eq!(snap.node_to_peer.get(&NodeId(2)), Some(&PeerId(2))); } diff --git a/cmd/unbounded-storage/src/p2p/transport.rs b/cmd/unbounded-storage/src/p2p/transport.rs index e71a8f9c5..d7b26b686 100644 --- a/cmd/unbounded-storage/src/p2p/transport.rs +++ b/cmd/unbounded-storage/src/p2p/transport.rs @@ -20,10 +20,9 @@ //! from the local `Backend` (origin fetch into the pool's pages). //! * `next_hop(target) == Some(peer)` - some other node is the next //! hop; hand the request to the wrapped `FabricTransport`, whose -//! `FingerRouter` resolves the same peer. The request travels with +//! `ChainFingerRouter` resolves the same peer. The request travels with //! the default `MAX_HOPS` TTL; recursion happens on the server. -use std::collections::HashMap; use std::marker::PhantomData; use std::pin::Pin; use std::sync::Arc; @@ -35,10 +34,8 @@ use serde::de::DeserializeOwned; use crate::backend::Backend; use crate::bufferpool::{BulkRef, Error, PageRef, PageStream, Req, Transport}; use crate::fabric::Result as FabResult; -use crate::fabric::{Fabric, FabricTransport, MrHandle, PeerId}; -use crate::p2p::{ - ChainFingerRouter, FingerTable, NodeId, RouteTableHandle, RoutingHandle, stripe_to_ring, -}; +use crate::fabric::{Fabric, FabricTransport, MrHandle}; +use crate::p2p::{ChainFingerRouter, RouteTableHandle, stripe_to_ring}; /// Transport that selects the first hop per request by finger-table /// ownership: the local origin `Backend` when this node owns the @@ -46,7 +43,7 @@ use crate::p2p::{ pub struct RoutedTransport> { fabric_transport: FabricTransport, /// Shared, live-reloadable routing surface. The inner - /// `FabricTransport`'s `FingerRouter` shares this same handle, so a + /// `FabricTransport`'s router shares this same handle, so a /// peer-set republish through any clone updates both the /// local-ownership pre-check below and the fabric routing at once. routes: RouteTableHandle, @@ -55,65 +52,6 @@ pub struct RoutedTransport> { } impl> RoutedTransport { - /// Build a routed transport over a freshly-seeded routing surface. - /// The transport owns its own [`RoutingHandle`]; use - /// [`Self::with_routing`] to share one with other consumers for - /// live reload. - pub fn new( - fabric: Arc, - mr: MrHandle, - page_size: usize, - fingers: Arc, - node_to_peer: Arc>, - backend: B, - ) -> FabResult { - let mut routes = HashMap::new(); - routes.insert( - RouteTableHandle::LEGACY_ROUTE_ID.to_string(), - crate::p2p::RoutingSnapshot { - fingers, - node_to_peer, - }, - ); - Self::with_routes( - fabric, - mr, - page_size, - RouteTableHandle::new(routes), - backend, - ) - } - - /// Build a routed transport that shares `routing` with other - /// consumers. Constructs the inner `FabricTransport` with a - /// `FingerRouter` over a clone of the same handle, so the - /// local-ownership pre-check and the fabric routing always agree - /// and reload together. - pub fn with_routing( - fabric: Arc, - mr: MrHandle, - page_size: usize, - routing: RoutingHandle, - backend: B, - ) -> FabResult { - let snap = routing.snapshot(); - let mut routes = HashMap::new(); - routes.insert( - RouteTableHandle::LEGACY_ROUTE_ID.to_string(), - crate::p2p::RoutingSnapshot { - fingers: snap.fingers.clone(), - node_to_peer: snap.node_to_peer.clone(), - }, - ); - Self::with_routes( - fabric, - mr, - page_size, - RouteTableHandle::new(routes), - backend, - ) - } - pub fn with_routes( fabric: Arc, mr: MrHandle, @@ -206,14 +144,11 @@ where #[cfg(test)] mod tests { - use std::collections::HashMap; use std::pin::Pin; - use std::sync::Arc; use std::task::{Context, Poll}; use crate::backend::Backend; use crate::bufferpool::{BulkRef, Error, PageRef, PageStream, Req, StripeKey}; - use crate::fabric::PeerId; use crate::p2p::{ FingerTable, FingerTableConfig, NodeId, PeerEntry, RingId, TopologyTags, node_to_ring, stripe_to_ring, @@ -290,10 +225,6 @@ mod tests { StripeKey(k) } - fn node_to_peer_map(nodes: &[u64]) -> Arc> { - Arc::new(nodes.iter().map(|&n| (NodeId(n), PeerId(n))).collect()) - } - /// Build a `RoutedTransport` whose `FabricTransport` is never /// driven. Constructing it without a live fabric is infeasible, so /// these tests exercise the routing decision (`owns_locally`) and @@ -310,7 +241,6 @@ mod tests { // routing decision is always "Backend arm". let local = peer(1); let fingers = FingerTable::build(local.clone(), &[], FingerTableConfig::with_k(8)); - let _ = node_to_peer_map(&[1]); for target in [0u64, 1, 999, u64::MAX / 2, u64::MAX] { assert!( fingers From e4aef69037f0c81db089259b147e87af5026fb35 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 23:07:27 +0000 Subject: [PATCH 13/56] unbounded-storage: enforce shard-local backends --- cmd/unbounded-storage/src/backend/azure.rs | 11 ----------- cmd/unbounded-storage/src/backend/fake.rs | 11 +---------- cmd/unbounded-storage/src/backend/http.rs | 11 ----------- cmd/unbounded-storage/src/backend/mod.rs | 7 +++---- cmd/unbounded-storage/src/backend/registry.rs | 9 --------- cmd/unbounded-storage/src/backend/s3.rs | 11 ----------- cmd/unbounded-storage/src/tls/context.rs | 7 ------- 7 files changed, 4 insertions(+), 63 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/azure.rs b/cmd/unbounded-storage/src/backend/azure.rs index 159fa2b2a..da2baba51 100644 --- a/cmd/unbounded-storage/src/backend/azure.rs +++ b/cmd/unbounded-storage/src/backend/azure.rs @@ -72,17 +72,6 @@ pub struct AzureBackend { conns: OriginConnPool, } -// SAFETY: mirrors `S3Backend`'s justification. `AzureBackend` is -// shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ -// `RefCell` it holds, and the raw `backing_base` pointer are never -// shared across threads at runtime. The `Send + Sync` marker exists -// solely to satisfy the `Backend: Send + Sync` bound the embedder -// requires when it stores the backend in a cross-shard registry; it is -// not an invitation to touch the backend off its shard. -unsafe impl Send for AzureBackend {} -unsafe impl Sync for AzureBackend {} - impl AzureBackend { #[allow(clippy::too_many_arguments)] pub fn new( diff --git a/cmd/unbounded-storage/src/backend/fake.rs b/cmd/unbounded-storage/src/backend/fake.rs index e3ab849ac..17ebe67a7 100644 --- a/cmd/unbounded-storage/src/backend/fake.rs +++ b/cmd/unbounded-storage/src/backend/fake.rs @@ -32,7 +32,7 @@ use super::http::copy_body_into_pages; /// [`HttpBackend`](super::http::HttpBackend): the raw `backing_base` /// pointer is only ever written on the single thread that drives the /// backend. Unlike the HTTP backend it holds no ring, so the only -/// cross-thread concern is the backing pointer. +/// only unsafe concern is the backing pointer. pub struct FakeBackend { backend_id: String, stripe_size: u64, @@ -41,15 +41,6 @@ pub struct FakeBackend { backing_base: *mut u8, } -// SAFETY: mirrors `HttpBackend`. `FakeBackend` is shard-pinned: the -// embedder constructs it on, and only ever drives it from, a single -// pinned shard thread, so the raw `backing_base` pointer is never -// touched concurrently. The `Send + Sync` marker exists solely to -// satisfy the `Backend: Send + Sync` bound the cross-shard registry -// requires; it is not an invitation to touch the backend off its shard. -unsafe impl Send for FakeBackend {} -unsafe impl Sync for FakeBackend {} - impl FakeBackend { pub fn new( backend_id: String, diff --git a/cmd/unbounded-storage/src/backend/http.rs b/cmd/unbounded-storage/src/backend/http.rs index 24c1e9e71..6de035c4c 100644 --- a/cmd/unbounded-storage/src/backend/http.rs +++ b/cmd/unbounded-storage/src/backend/http.rs @@ -76,17 +76,6 @@ pub struct HttpBackend { conns: OriginConnPool, } -// SAFETY: mirrors `crate::memory::Backing`. `HttpBackend` is -// shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ -// `RefCell` it holds, and the raw `backing_base` pointer are never -// shared across threads at runtime. The `Send + Sync` marker exists -// solely to satisfy the `Backend: Send + Sync` bound the embedder -// requires when it stores the backend in a cross-shard registry; it is -// not an invitation to touch the backend off its shard. -unsafe impl Send for HttpBackend {} -unsafe impl Sync for HttpBackend {} - impl HttpBackend { pub fn new( handle: NetHandle, diff --git a/cmd/unbounded-storage/src/backend/mod.rs b/cmd/unbounded-storage/src/backend/mod.rs index 51c644344..c0dd5b809 100644 --- a/cmd/unbounded-storage/src/backend/mod.rs +++ b/cmd/unbounded-storage/src/backend/mod.rs @@ -62,7 +62,7 @@ pub use fake::{FakeBackend, FakeFetchStream}; /// `Backend` resolves a `BulkRef` from an authoritative origin (as /// opposed to a peer) into the supplied destination pages, yielding /// one page at a time through a `PageStream`. -pub trait Backend: Send + Sync { +pub trait Backend { type Req: Req; /// Stream of pages produced by `bulk_get`. One `poll_next` may @@ -84,9 +84,8 @@ pub trait Backend: Send + Sync { ) -> Self::Stream<'a>; } -/// Blanket impl mirroring `Transport for Arc`, so a `Backend` can -/// be shared across shards by handing each consumer an `Arc`-wrapped -/// clone instead of an owned instance. +/// Blanket impl mirroring `Transport for Arc` for callers that +/// already own a backend through an `Arc`. impl Backend for Arc { type Req = T::Req; diff --git a/cmd/unbounded-storage/src/backend/registry.rs b/cmd/unbounded-storage/src/backend/registry.rs index 03730b292..8230bd7ef 100644 --- a/cmd/unbounded-storage/src/backend/registry.rs +++ b/cmd/unbounded-storage/src/backend/registry.rs @@ -61,15 +61,6 @@ pub struct BackendRegistry { ctx: BuildCtx, } -// SAFETY: shard-pinned exactly like the `OriginBackend`s it holds (see -// `backend::http`/`backend::s3`). The `*mut u8` build base and the -// `Rc`-bearing `NetHandle` are only ever touched on the owning shard -// thread; the marker only lets the registry live in the -// `Send + Sync` generic slots the transport/handler require. It is not -// safe to move a registry to an unrelated thread. -unsafe impl Send for BackendRegistry {} -unsafe impl Sync for BackendRegistry {} - impl BackendRegistry { /// Build a registry seeded from `specs`, fetching through `handle` /// into the region based at `backing_base`. A spec that fails to diff --git a/cmd/unbounded-storage/src/backend/s3.rs b/cmd/unbounded-storage/src/backend/s3.rs index 974246ea7..a121eaea3 100644 --- a/cmd/unbounded-storage/src/backend/s3.rs +++ b/cmd/unbounded-storage/src/backend/s3.rs @@ -72,17 +72,6 @@ pub struct S3Backend { conns: OriginConnPool, } -// SAFETY: mirrors `HttpBackend`'s justification. `S3Backend` is -// shard-pinned: the embedder constructs it on, and only ever drives it -// from, a single pinned shard thread. The `NetHandle`, any `Rc`/ -// `RefCell` it holds, and the raw `backing_base` pointer are never -// shared across threads at runtime. The `Send + Sync` marker exists -// solely to satisfy the `Backend: Send + Sync` bound the embedder -// requires when it stores the backend in a cross-shard registry; it is -// not an invitation to touch the backend off its shard. -unsafe impl Send for S3Backend {} -unsafe impl Sync for S3Backend {} - impl S3Backend { #[allow(clippy::too_many_arguments)] pub fn new( diff --git a/cmd/unbounded-storage/src/tls/context.rs b/cmd/unbounded-storage/src/tls/context.rs index c62d1f466..12b86b312 100644 --- a/cmd/unbounded-storage/src/tls/context.rs +++ b/cmd/unbounded-storage/src/tls/context.rs @@ -59,13 +59,6 @@ pub struct TlsContext { verify: bool, } -// SAFETY: the same shard-pinned justification as the backends. The -// `SSL_CTX` is only used from the single thread that owns the context; -// the Send + Sync bounds exist solely so a backend holding a -// `TlsContext` can satisfy the `Backend: Send + Sync` supertrait. -unsafe impl Send for TlsContext {} -unsafe impl Sync for TlsContext {} - impl TlsContext { /// Build a client TLS context from `config`. pub fn new(config: &TlsConfig) -> Result { From 7f582c4dbbc6774a7cc8302c350a22a89b67cdbd Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 19:58:22 +0000 Subject: [PATCH 14/56] unbounded-storage: unify disk registry entries --- .../src/storage/disks/mod.rs | 123 ++++++++++-------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/disks/mod.rs b/cmd/unbounded-storage/src/storage/disks/mod.rs index 96f82eab3..fe82fa120 100644 --- a/cmd/unbounded-storage/src/storage/disks/mod.rs +++ b/cmd/unbounded-storage/src/storage/disks/mod.rs @@ -116,10 +116,21 @@ pub struct DiskReport { pub struct DiskRegistry { target: T, disk_slots: Vec, - handles: HashMap, - channels: HashMap, - applied: HashMap, - placement: HashMap>, + entries: HashMap>, +} + +struct OpenDiskEntry { + channel: PageChannel, + spec: DiskSpec, + placement: Option, + handle: H, +} + +impl OpenDiskEntry { + fn close(self) { + drop(self.channel); + drop(self.handle); + } } pub struct DiskRegistrySet { @@ -137,10 +148,7 @@ impl DiskRegistry { Self { target, disk_slots, - handles: HashMap::new(), - channels: HashMap::new(), - applied: HashMap::new(), - placement: HashMap::new(), + entries: HashMap::new(), } } @@ -171,19 +179,17 @@ impl DiskRegistry { } let to_remove: Vec = self - .handles + .entries .keys() .filter(|p| match desired_paths.get(p.as_path()) { None => true, - Some(spec) => specs_drifted(self.applied.get(*p), spec), + Some(spec) => specs_drifted(self.entries.get(*p).map(|entry| &entry.spec), spec), }) .cloned() .collect(); for path in to_remove { - if self.handles.remove(&path).is_some() { - self.channels.remove(&path); - self.applied.remove(&path); - self.placement.remove(&path); + if let Some(entry) = self.entries.remove(&path) { + entry.close(); report.removed += 1; } } @@ -196,25 +202,35 @@ impl DiskRegistry { // invariant). This is idempotent across reconciles (same // desired set -> same assignment) and never leaks a slot when a // disk churns out and back in. + let open_placements: HashMap> = self + .entries + .iter() + .map(|(path, entry)| (path.clone(), entry.placement)) + .collect(); let assignment = if reserved_slots.is_empty() { - assign_disk_cpus(desired, &self.disk_slots, &self.placement) + assign_disk_cpus(desired, &self.disk_slots, &open_placements) } else { - assign_disk_cpus_reserved(desired, &self.disk_slots, &self.placement, reserved_slots) + assign_disk_cpus_reserved(desired, &self.disk_slots, &open_placements, reserved_slots) }; for spec in desired { let path = disk_path(spec); - if self.handles.contains_key(Path::new(path)) { - self.applied.insert(PathBuf::from(path), spec.clone()); + if let Some(entry) = self.entries.get_mut(Path::new(path)) { + entry.spec = spec.clone(); continue; } let pin = assignment.get(Path::new(path)).copied().flatten(); match self.target.open(spec, pin) { Ok((handle, channel)) => { - self.handles.insert(PathBuf::from(path), handle); - self.channels.insert(PathBuf::from(path), channel); - self.applied.insert(PathBuf::from(path), spec.clone()); - self.placement.insert(PathBuf::from(path), pin); + self.entries.insert( + PathBuf::from(path), + OpenDiskEntry { + channel, + spec: spec.clone(), + placement: pin, + handle, + }, + ); report.added += 1; } Err(e) => { @@ -235,21 +251,16 @@ impl DiskRegistry { /// stripe's serving shard on the same node as its disk. pub fn channels_snapshot(&self) -> Vec<(PathBuf, PageChannel, Option, bool)> { let mut out: Vec<(PathBuf, PageChannel, Option, bool)> = self - .channels + .entries .iter() - .map(|(p, c)| { - let numa = self - .placement - .get(p) - .copied() - .flatten() - .and_then(|s| s.numa); - let page_cache_enabled = self - .applied - .get(p) - .map(|spec| !spec.disable_page_cache) - .unwrap_or(true); - (p.clone(), c.clone(), numa, page_cache_enabled) + .map(|(path, entry)| { + let numa = entry.placement.and_then(|slot| slot.numa); + ( + path.clone(), + entry.channel.clone(), + numa, + !entry.spec.disable_page_cache, + ) }) .collect(); out.sort_by(|a, b| a.0.cmp(&b.0)); @@ -261,15 +272,14 @@ impl DiskRegistry { /// to the storage cores; that lets the storage cores observe the /// channel disconnect and exit promptly. pub fn drain(mut self) { - self.channels.clear(); - self.handles.clear(); - self.applied.clear(); - self.placement.clear(); + for (_, entry) in self.entries.drain() { + entry.close(); + } } /// Paths whose handle is currently open. Order is unspecified. pub fn current_paths(&self) -> Vec { - self.handles.keys().cloned().collect() + self.entries.keys().cloned().collect() } /// The CPU slot assigned to each currently-open disk, in @@ -278,9 +288,9 @@ impl DiskRegistry { /// logs. pub fn placement_snapshot(&self) -> Vec<(PathBuf, Option)> { let mut out: Vec<(PathBuf, Option)> = self - .placement + .entries .iter() - .map(|(p, h)| (p.clone(), *h)) + .map(|(path, entry)| (path.clone(), entry.placement)) .collect(); out.sort_by(|a, b| a.0.cmp(&b.0)); out @@ -344,7 +354,12 @@ impl DiskRegistrySet { self.registries .iter() .filter(|(id, _)| id.as_str() != cache_id) - .flat_map(|(_, registry)| registry.placement.values().copied().flatten()) + .flat_map(|(_, registry)| { + registry + .entries + .values() + .filter_map(|entry| entry.placement) + }) .collect() } } @@ -603,7 +618,7 @@ mod tests { assert_eq!(report.added, 0); assert_eq!(report.removed, 0); assert_eq!(state.lock().unwrap().open_calls, calls_after_first); - assert!(reg.applied[&PathBuf::from("/a")].disable_page_cache); + assert!(reg.entries[&PathBuf::from("/a")].spec.disable_page_cache); assert!(!reg.channels_snapshot()[0].3); } @@ -615,7 +630,7 @@ mod tests { let report = reg.reconcile(&[spec("/a", Some(32))]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert_eq!(reg.applied[&PathBuf::from("/a")].queue_depth, Some(32)); + assert_eq!(reg.entries[&PathBuf::from("/a")].spec.queue_depth, Some(32)); } #[test] @@ -626,8 +641,8 @@ mod tests { let report = reg.reconcile(&[spec("/b", None)]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied.contains_key(&PathBuf::from("/b"))); - assert!(!reg.applied.contains_key(&PathBuf::from("/a"))); + assert!(reg.entries.contains_key(&PathBuf::from("/b"))); + assert!(!reg.entries.contains_key(&PathBuf::from("/a"))); } #[test] @@ -879,7 +894,7 @@ mod tests { assert_eq!(report.added, 1); assert_eq!(report.removed, 1); assert_eq!( - reg.applied[&PathBuf::from("/a")].page_size_bytes, + reg.entries[&PathBuf::from("/a")].spec.page_size_bytes, Some(4096) ); } @@ -924,7 +939,7 @@ mod tests { let report = reg.reconcile(&[next]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied[&PathBuf::from("/a")].skip_recovery_scan); + assert!(reg.entries[&PathBuf::from("/a")].spec.skip_recovery_scan); } #[test] @@ -937,7 +952,7 @@ mod tests { let report = reg.reconcile(&[next]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied[&PathBuf::from("/a")].force_format); + assert!(reg.entries[&PathBuf::from("/a")].spec.force_format); } #[test] @@ -950,7 +965,7 @@ mod tests { let report = reg.reconcile(&[next]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied[&PathBuf::from("/a")].bypass_admission); + assert!(reg.entries[&PathBuf::from("/a")].spec.bypass_admission); } #[test] @@ -963,7 +978,7 @@ mod tests { let report = reg.reconcile(&[next]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied[&PathBuf::from("/a")].bypass_index_read); + assert!(reg.entries[&PathBuf::from("/a")].spec.bypass_index_read); } #[test] @@ -976,7 +991,7 @@ mod tests { let report = reg.reconcile(&[next]); assert_eq!(report.added, 1); assert_eq!(report.removed, 1); - assert!(reg.applied[&PathBuf::from("/a")].bypass_checksum); + assert!(reg.entries[&PathBuf::from("/a")].spec.bypass_checksum); } #[test] From a44f94b6e7e7b42a311ca9bf88d16b5f9ac2db9e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:08:03 +0000 Subject: [PATCH 15/56] unbounded-storage: remove redundant peer map --- cmd/unbounded-storage/ARCHITECTURE.md | 6 ++-- cmd/unbounded-storage/src/fabric/tests.rs | 32 +++-------------- .../src/frontend/loadgen_serve.rs | 12 +------ cmd/unbounded-storage/src/main.rs | 9 +---- .../src/p2p/finger_router.rs | 30 +++++----------- .../src/p2p/routing_handle.rs | 35 +++++++------------ 6 files changed, 32 insertions(+), 92 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index b59291e81..58a88130c 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -144,9 +144,9 @@ excluded from the live-reload diff. publication surface for disk channels. 7. Read-only shared state (`Arc>`, `Arc>`; startup-fixed fabric settings come from the config `[startup]` section via - `StartupSettings`) and routing (`build_routing` -> `Arc` plus - `Arc>`) are constructed once and shared across - shards. + `StartupSettings`) and routing (`build_routing` -> `Arc`) are + constructed once and shared across shards. `PeerId` uses the same stable + numeric identity as `NodeId`. 8. Each shard is spawned with `rt.spawn_pinned(widx, name, Box)`. The `!Send` shard objects are constructed **inside** `run_shard`, after pinning. 9. After every shard reports `Up`, peers are reconciled per shard, the disk diff --git a/cmd/unbounded-storage/src/fabric/tests.rs b/cmd/unbounded-storage/src/fabric/tests.rs index 553083c9c..acdc95723 100644 --- a/cmd/unbounded-storage/src/fabric/tests.rs +++ b/cmd/unbounded-storage/src/fabric/tests.rs @@ -967,8 +967,6 @@ fn pool_handler_reports_miss_for_non_resident_stripe() { // the production `FetchService` provides. // --------------------------------------------------------------- -use std::collections::HashMap; - use crate::fanout::{ FetchChannel, FetchChannelReceiver, FetchCommand, FetchEvent, FetchPage, PageLoc, }; @@ -1080,7 +1078,6 @@ fn start_recursive_node( scratch_mr: MrHandle, page_size: usize, fingers: Arc, - node_to_peer: Arc>, owners: OwnerShardTable, ) -> crate::fabric::RpcServerHandle { let handler = Arc::new( @@ -1088,15 +1085,9 @@ fn start_recursive_node( scratch, RECURSIVE_SCRATCH_PAGES as u32, RouteTableHandle::new( - [( - CHAIN_CACHE_ID.to_string(), - RoutingSnapshot { - fingers, - node_to_peer, - }, - )] - .into_iter() - .collect(), + [(CHAIN_CACHE_ID.to_string(), RoutingSnapshot { fingers })] + .into_iter() + .collect(), ), fabric.clone(), scratch_mr, @@ -1200,11 +1191,6 @@ fn recursive_chain( std::slice::from_ref(&ring_peer(2, 100)), FingerTableConfig::with_k(8), )); - let b_n2p: Arc> = - Arc::new([(NodeId(3), PeerId(3))].into_iter().collect()); - let c_n2p: Arc> = - Arc::new([(NodeId(2), PeerId(2))].into_iter().collect()); - let empty_owners = OwnerShardTable::new( Vec::new(), crate::storage::disks::CacheDirectorySet::new(), @@ -1223,16 +1209,8 @@ fn recursive_chain( page_size, ); - let b_handle = start_recursive_node( - &b, - b_scratch, - b_mr, - page_size, - b_fingers, - b_n2p, - empty_owners, - ); - let c_handle = start_recursive_node(&c, c_scratch, c_mr, page_size, c_fingers, c_n2p, c_owners); + let b_handle = start_recursive_node(&b, b_scratch, b_mr, page_size, b_fingers, empty_owners); + let c_handle = start_recursive_node(&c, c_scratch, c_mr, page_size, c_fingers, c_owners); let transport = FabricTransport::::new( a.clone(), diff --git a/cmd/unbounded-storage/src/frontend/loadgen_serve.rs b/cmd/unbounded-storage/src/frontend/loadgen_serve.rs index 5cb1b1089..fbb43fe11 100644 --- a/cmd/unbounded-storage/src/frontend/loadgen_serve.rs +++ b/cmd/unbounded-storage/src/frontend/loadgen_serve.rs @@ -559,18 +559,8 @@ mod tests { &[peer], FingerTableConfig::with_k(4), )); - let node_to_peer = std::sync::Arc::new(std::collections::HashMap::from([( - NodeId(2), - crate::fabric::PeerId(2), - )])); let mut routes = std::collections::HashMap::new(); - routes.insert( - "cache".to_string(), - crate::p2p::RoutingSnapshot { - fingers, - node_to_peer, - }, - ); + routes.insert("cache".to_string(), crate::p2p::RoutingSnapshot { fingers }); RouteTableHandle::new(routes) } diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index e4176ba6b..28fb0043a 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -28,7 +28,7 @@ use unbounded_storage::frontend::{ HttpDriver, HttpFrontend, LoadgenDriver, LoadgenFrontend, S3Driver, S3Frontend, }; use unbounded_storage::p2p::{ - FingerTable, FingerTableConfig, NodeId, PeerEntry, RouteTableHandle, RouteTableSnapshot, + FingerTable, FingerTableConfig, PeerEntry, RouteTableHandle, RouteTableSnapshot, RoutedTransport, RoutingSnapshot, TopologyPrefixWeight, TopologySelection, TopologyTags, TopologyWeighting, node_to_ring, }; @@ -1119,12 +1119,6 @@ fn build_routes(config: &Config) -> RouteTableSnapshot { ring: node_to_ring(mesh.self_node_id), tags: TopologyTags(mesh.self_tags.clone()), }; - let node_to_peer: HashMap = mesh - .peers - .iter() - .map(|peer| (peer.node_id, peer.fabric_peer_id)) - .collect(); - let node_to_peer = Arc::new(node_to_peer); let fingers = if let Some(plan) = &mesh.routing_plan { let peer_by_name: HashMap<&str, &config::RuntimePeer> = mesh .peers @@ -1179,7 +1173,6 @@ fn build_routes(config: &Config) -> RouteTableSnapshot { id.clone(), RoutingSnapshot { fingers: fingers.clone(), - node_to_peer: node_to_peer.clone(), }, ) }) diff --git a/cmd/unbounded-storage/src/p2p/finger_router.rs b/cmd/unbounded-storage/src/p2p/finger_router.rs index 0bf8283aa..ae0fd6ca3 100644 --- a/cmd/unbounded-storage/src/p2p/finger_router.rs +++ b/cmd/unbounded-storage/src/p2p/finger_router.rs @@ -4,11 +4,12 @@ //! `PeerRouter` backed by the p2p finger table. //! //! Maps a request's stripe to the owning peer's `PeerId` via the -//! Chord `next_hop` lookup, then the `node -> peer` map. This is the +//! Chord `next_hop` lookup. Node and fabric peer IDs share the same +//! stable numeric identity. This is the //! router the client-side `FabricTransport` consults to pick the //! first hop; the server-side `RecursiveHandler` reuses the same type -//! to forward to the next hop. Both share the same `FingerTable` and -//! node->peer map so ownership decisions stay consistent across the +//! to forward to the next hop. Both share the same `FingerTable` so +//! ownership decisions stay consistent across the //! transport, the local-ownership pre-check, and the forwarding path. use crate::bufferpool::Req; @@ -31,7 +32,7 @@ impl PeerRouter for ChainFingerRouter { let snap = self.routes.route_for_req(req)?; snap.fingers .next_hop(stripe_to_ring(req.key())) - .and_then(|pe| snap.node_to_peer.get(&pe.node).copied()) + .map(|peer| PeerId(peer.node.0)) } } @@ -80,20 +81,10 @@ mod tests { StripeKey(k) } - fn node_to_peer_map(nodes: &[u64]) -> Arc> { - Arc::new(nodes.iter().map(|&n| (NodeId(n), PeerId(n))).collect()) - } - - fn router( - fingers: Arc, - node_to_peer: Arc>, - ) -> ChainFingerRouter { + fn router(fingers: Arc) -> ChainFingerRouter { ChainFingerRouter::new(crate::p2p::RouteTableHandle::new(HashMap::from([( "cache-a".to_string(), - crate::p2p::RoutingSnapshot { - fingers, - node_to_peer, - }, + crate::p2p::RoutingSnapshot { fingers }, )]))) } @@ -120,9 +111,7 @@ mod tests { &[], FingerTableConfig::with_k(8), )); - let node_to_peer = node_to_peer_map(&[1]); - - let router = router(fingers.clone(), node_to_peer); + let router = router(fingers.clone()); // Pick a target that is not the local ring id so the owner // check still exercises the predecessor path (single node // owns everything regardless). @@ -143,8 +132,7 @@ mod tests { std::slice::from_ref(&other), FingerTableConfig::with_k(8), )); - let node_to_peer = node_to_peer_map(&[1, 2]); - let router = router(fingers.clone(), node_to_peer); + let router = router(fingers.clone()); // Target == other's ring id: local forwards to other. let req = req(key_for_ring(other.ring.0)); diff --git a/cmd/unbounded-storage/src/p2p/routing_handle.rs b/cmd/unbounded-storage/src/p2p/routing_handle.rs index e0df0172d..f54e4d2ee 100644 --- a/cmd/unbounded-storage/src/p2p/routing_handle.rs +++ b/cmd/unbounded-storage/src/p2p/routing_handle.rs @@ -3,9 +3,8 @@ //! Live-reloadable route table shared across p2p consumers. //! -//! The finger table and the `node -> peer` map are derived together -//! from the active cache keyspace projected by the config graph. Three independent -//! consumers read them on the hot path: +//! The finger table is derived from the active cache keyspace projected +//! by the config graph. Three independent consumers read it on the hot path: //! //! * [`crate::p2p::ChainFingerRouter`] (the first-hop lookup wrapped by //! every [`crate::fabric::FabricTransport`]), @@ -30,19 +29,14 @@ use std::sync::Arc; use arc_swap::ArcSwap; -use crate::fabric::PeerId; -use crate::p2p::{FingerTable, NodeId}; +use crate::p2p::FingerTable; -/// Immutable view of the routing surface. Published as a unit so a -/// reader can never observe a new finger table paired with a stale -/// `node -> peer` map (or vice versa). +/// Immutable routing surface for one cache. /// -/// Cloning is cheap: both fields are `Arc`s, so a clone is two atomic -/// refcount bumps and shares the underlying tables. +/// Cloning shares the underlying finger table. #[derive(Clone)] pub struct RoutingSnapshot { pub fingers: Arc, - pub node_to_peer: Arc>, } #[derive(Clone, Default)] @@ -96,7 +90,7 @@ impl RouteTableHandle { #[cfg(test)] mod tests { use super::*; - use crate::p2p::{FingerTableConfig, PeerEntry, TopologyTags, node_to_ring}; + use crate::p2p::{FingerTableConfig, NodeId, PeerEntry, TopologyTags, node_to_ring}; fn peer(node: u64) -> PeerEntry { PeerEntry { @@ -115,10 +109,6 @@ mod tests { )) } - fn map(nodes: &[u64]) -> Arc> { - Arc::new(nodes.iter().map(|&n| (NodeId(n), PeerId(n))).collect()) - } - #[test] fn store_is_observed_by_existing_clones() { // A clone taken BEFORE the store must observe the new snapshot: @@ -129,24 +119,25 @@ mod tests { "cache-a".to_string(), RoutingSnapshot { fingers: table(1, &[]), - node_to_peer: map(&[1]), }, )])); let consumer = handle.clone(); let before = consumer.route("cache-a").expect("route"); - assert!(before.node_to_peer.contains_key(&NodeId(1))); - assert!(!before.node_to_peer.contains_key(&NodeId(2))); + assert!(before.fingers.next_hop(node_to_ring(NodeId(2))).is_none()); handle.store(HashMap::from([( "cache-a".to_string(), RoutingSnapshot { fingers: table(1, &[2]), - node_to_peer: map(&[1, 2]), }, )])); let snap = consumer.route("cache-a").expect("route"); - assert!(snap.node_to_peer.contains_key(&NodeId(2))); - assert_eq!(snap.node_to_peer.get(&NodeId(2)), Some(&PeerId(2))); + assert_eq!( + snap.fingers + .next_hop(node_to_ring(NodeId(2))) + .map(|peer| peer.node), + Some(NodeId(2)), + ); } } From 9f16c96527bce0ffbf7602c41303588963ed33ac Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 23:18:47 +0000 Subject: [PATCH 16/56] unbounded-storage: make backend registry shard-local --- cmd/unbounded-storage/src/backend/azure.rs | 4 +- cmd/unbounded-storage/src/backend/http.rs | 3 +- cmd/unbounded-storage/src/backend/origin.rs | 8 +- cmd/unbounded-storage/src/backend/registry.rs | 116 ++++++++++++------ cmd/unbounded-storage/src/backend/s3.rs | 4 +- 5 files changed, 86 insertions(+), 49 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/azure.rs b/cmd/unbounded-storage/src/backend/azure.rs index da2baba51..4a9e2e6d0 100644 --- a/cmd/unbounded-storage/src/backend/azure.rs +++ b/cmd/unbounded-storage/src/backend/azure.rs @@ -121,8 +121,8 @@ impl AzureBackend { /// Owned-stream variant of [`Backend::bulk_get`]. Mirrors /// [`super::S3Backend::fetch_stream`]: the returned stream borrows /// nothing from `self`, so it is `'static` and can be handed out by - /// a [`super::registry::BackendRegistry`] holding the backend behind - /// an `Arc`/`ArcSwap`. + /// a [`super::registry::BackendRegistry`] owning the backend through + /// an `Rc`. pub fn fetch_stream( &self, req: &StripeReq, diff --git a/cmd/unbounded-storage/src/backend/http.rs b/cmd/unbounded-storage/src/backend/http.rs index 6de035c4c..4c96e18fa 100644 --- a/cmd/unbounded-storage/src/backend/http.rs +++ b/cmd/unbounded-storage/src/backend/http.rs @@ -152,8 +152,7 @@ impl HttpBackend { /// destination pages are copied into an owned `Vec`, and the ring /// handle is owned. That makes the stream `'static`, which is what /// lets a [`super::registry::BackendRegistry`] hand out streams from - /// a backend it only holds behind an `Arc`/`ArcSwap` (the temporary - /// `Arc` guard does not have to outlive the stream). + /// a backend it owns through an `Rc` without borrowing the registry. pub fn fetch_stream( &self, req: &StripeReq, diff --git a/cmd/unbounded-storage/src/backend/origin.rs b/cmd/unbounded-storage/src/backend/origin.rs index 285efb1e6..31a2627a1 100644 --- a/cmd/unbounded-storage/src/backend/origin.rs +++ b/cmd/unbounded-storage/src/backend/origin.rs @@ -13,7 +13,7 @@ //! embedder selects at construction time. use std::pin::Pin; -use std::sync::Arc; +use std::rc::Rc; use std::task::{Context, Poll}; use crate::bufferpool::{BulkRef, Error, PageRef, PageStream}; @@ -33,11 +33,11 @@ pub enum OriginBackend { impl OriginBackend { /// Owned-stream variant used by [`super::registry::BackendRegistry`]. - /// The registry gives the stream an `Arc` guard so a backend can be + /// The registry gives the stream an `Rc` guard so a backend can be /// borrowed safely without forcing every concrete backend to copy its /// destination-page slice. pub fn fetch_stream<'a>( - backend: Arc, + backend: Rc, req: &StripeReq, src: BulkRef, dsts: &'a [PageRef], @@ -89,7 +89,7 @@ pub enum OriginStream<'a> { S3(::Stream<'static>), Azure(::Stream<'static>), Fake { - backend: Arc, + backend: Rc, delivered: &'a [PageRef], state: FakeOwnedState, next: usize, diff --git a/cmd/unbounded-storage/src/backend/registry.rs b/cmd/unbounded-storage/src/backend/registry.rs index 8230bd7ef..654418f36 100644 --- a/cmd/unbounded-storage/src/backend/registry.rs +++ b/cmd/unbounded-storage/src/backend/registry.rs @@ -8,10 +8,10 @@ //! [`OriginRef::backend_id`](crate::storage::OriginRef), stamped by the //! frontend that accepted it, so the data path can resolve which //! configured backend to fetch from per request. [`BackendRegistry`] -//! owns the `name -> OriginBackend` map behind an [`ArcSwap`] so the +//! owns the `name -> OriginBackend` map in shard-local storage so the //! config watcher can add, remove, or replace a backend at runtime -//! without tearing down the shard: a reconcile pass builds the new map -//! and publishes it atomically, and the next fetch observes it. +//! without tearing down the shard. In-flight streams retain their chosen +//! backend independently of later registry changes. //! //! The registry implements [`Backend`] itself, so it drops into the //! same generic slot the single [`OriginBackend`] used to fill in the @@ -19,15 +19,13 @@ //! [`BackendReconcileTarget`] so the live-config reconciler drives it //! the same way it drives the peer fabric. +use std::cell::RefCell; use std::collections::HashMap; use std::io; use std::pin::Pin; use std::rc::Rc; -use std::sync::Arc; use std::task::{Context, Poll}; -use arc_swap::ArcSwap; - use crate::bufferpool::{BulkRef, Error, PageRef, PageStream}; use crate::config::reconcile::BackendReconcileTarget; use crate::config::{BackendSpec, backend_spec}; @@ -52,12 +50,12 @@ struct BuildCtx { } /// A shard-local set of origin backends keyed by component name, swappable at -/// runtime. Clones share the same underlying map (cheap `Arc` clone), +/// runtime. Clones share the same underlying map (cheap `Rc` clone), /// so a clone handed to the control-drain tick hook reconciles the /// very map the data path reads. #[derive(Clone)] pub struct BackendRegistry { - backends: Arc>>>, + backends: Rc>>>, ctx: BuildCtx, } @@ -78,12 +76,12 @@ impl BackendRegistry { page_size, backing_base, }; - let mut map: HashMap> = HashMap::with_capacity(specs.len()); + let mut map: HashMap> = HashMap::with_capacity(specs.len()); for spec in specs { - map.insert(spec.name.clone(), Arc::new(ctx.build(spec)?)); + map.insert(spec.name.clone(), Rc::new(ctx.build(spec)?)); } Ok(Self { - backends: Arc::new(ArcSwap::from_pointee(map)), + backends: Rc::new(RefCell::new(map)), ctx, }) } @@ -91,21 +89,13 @@ impl BackendRegistry { /// Number of backends currently registered. #[cfg(test)] pub fn len(&self) -> usize { - self.backends.load().len() + self.backends.borrow().len() } /// Whether a backend with `name` is currently registered. #[cfg(test)] pub fn contains(&self, id: &str) -> bool { - self.backends.load().contains_key(id) - } - - /// Copy-on-write mutate the live map: clone the current contents, - /// apply `f`, then publish the result atomically. - fn mutate>)>(&self, f: F) { - let mut next = HashMap::clone(&self.backends.load()); - f(&mut next); - self.backends.store(Arc::new(next)); + self.backends.borrow().contains_key(id) } } @@ -253,14 +243,11 @@ impl Backend for BackendRegistry { Some(origin) => origin.backend_id.as_str(), None => return RegistryFetchStream::unknown(""), }; - let map = self.backends.load(); - match map.get(backend_id) { - Some(backend) => RegistryFetchStream::Origin(OriginBackend::fetch_stream( - Arc::clone(backend), - req, - src, - dsts, - )), + let backend = self.backends.borrow().get(backend_id).cloned(); + match backend { + Some(backend) => { + RegistryFetchStream::Origin(OriginBackend::fetch_stream(backend, req, src, dsts)) + } None => RegistryFetchStream::unknown(backend_id), } } @@ -268,27 +255,23 @@ impl Backend for BackendRegistry { impl BackendReconcileTarget for BackendRegistry { fn list(&self) -> Vec { - self.backends.load().keys().cloned().collect() + self.backends.borrow().keys().cloned().collect() } fn add(&self, spec: &BackendSpec) -> Result<(), String> { - // Build before taking the swap so a failed build leaves the - // live map untouched. + // Build before mutating so a failed build leaves the live map untouched. let backend = self .ctx .build(spec) .map_err(|e| format!("build backend {}: {e}", spec.name))?; - let built = Arc::new(backend); - self.mutate(|map| { - map.insert(spec.name.clone(), built); - }); + self.backends + .borrow_mut() + .insert(spec.name.clone(), Rc::new(backend)); Ok(()) } fn remove(&self, id: &str) -> Result<(), String> { - self.mutate(|map| { - map.remove(id); - }); + self.backends.borrow_mut().remove(id); Ok(()) } } @@ -337,8 +320,10 @@ mod tests { use std::rc::Rc; use std::task::{RawWaker, RawWakerVTable, Waker}; + use crate::bufferpool::StripeKey; use crate::config::{FakeBackendConfig, HttpBackendConfig, backend_spec}; use crate::ring::NetworkRing; + use crate::storage::{OriginRef, synthetic_object_id}; use super::*; @@ -457,4 +442,57 @@ mod tests { assert_eq!(reg.len(), 1); assert!(reg.contains("synthetic")); } + + #[test] + fn selected_backend_outlives_removal() { + let reg = registry(&[fake_spec("synthetic", 1024 * 1024)]); + let selected = Rc::downgrade( + reg.backends + .borrow() + .get("synthetic") + .expect("seeded backend"), + ); + let origin = OriginRef::new("synthetic", synthetic_object_id(1, 2), 0); + let req = StripeReq::new(StripeKey([0; 32])).with_origin(origin); + let dsts = []; + let src = BulkRef { + stripe: StripeKey([0; 32]), + offset: 0, + len: 0, + }; + let stream = reg.bulk_get(&req, src, &dsts); + + reg.remove("synthetic").expect("remove backend"); + assert!(selected.upgrade().is_some()); + + drop(stream); + assert!(selected.upgrade().is_none()); + } + + #[test] + fn selected_backend_outlives_same_name_replacement() { + let reg = registry(&[fake_spec("synthetic", 1024 * 1024)]); + let selected = Rc::downgrade( + reg.backends + .borrow() + .get("synthetic") + .expect("seeded backend"), + ); + let origin = OriginRef::new("synthetic", synthetic_object_id(3, 4), 0); + let req = StripeReq::new(StripeKey([0; 32])).with_origin(origin); + let dsts = []; + let src = BulkRef { + stripe: StripeKey([0; 32]), + offset: 0, + len: 0, + }; + let stream = reg.bulk_get(&req, src, &dsts); + + reg.add(&fake_spec("synthetic", 2 * 1024 * 1024)) + .expect("replace backend"); + assert!(selected.upgrade().is_some()); + + drop(stream); + assert!(selected.upgrade().is_none()); + } } diff --git a/cmd/unbounded-storage/src/backend/s3.rs b/cmd/unbounded-storage/src/backend/s3.rs index a121eaea3..6ae4017b8 100644 --- a/cmd/unbounded-storage/src/backend/s3.rs +++ b/cmd/unbounded-storage/src/backend/s3.rs @@ -121,8 +121,8 @@ impl S3Backend { /// Owned-stream variant of [`Backend::bulk_get`]. Mirrors /// [`super::HttpBackend::fetch_stream`]: the returned stream borrows /// nothing from `self`, so it is `'static` and can be handed out by - /// a [`super::registry::BackendRegistry`] holding the backend behind - /// an `Arc`/`ArcSwap`. + /// a [`super::registry::BackendRegistry`] owning the backend through + /// an `Rc`. pub fn fetch_stream( &self, req: &StripeReq, From df110d3f5b1ff8af76af857b4dd8c9f9ae630323 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 20:00:25 +0000 Subject: [PATCH 17/56] unbounded-storage: flatten disk channel snapshots --- cmd/unbounded-storage/src/fanout/router.rs | 8 +- cmd/unbounded-storage/src/p2p/handler.rs | 6 +- .../src/storage/disks/channels.rs | 133 ++++++------------ .../src/storage/disks/shard_view.rs | 17 +-- .../tests/lifecycle/workload.rs | 8 +- 5 files changed, 67 insertions(+), 105 deletions(-) diff --git a/cmd/unbounded-storage/src/fanout/router.rs b/cmd/unbounded-storage/src/fanout/router.rs index be49e22b5..7cee22a0b 100644 --- a/cmd/unbounded-storage/src/fanout/router.rs +++ b/cmd/unbounded-storage/src/fanout/router.rs @@ -279,8 +279,12 @@ impl FanoutTable { cache_id: Option<&str>, stripe_off: u64, ) -> Owner<'_> { - let drive_numa = self.disk_channels.drive_numa(cache_id); - let pick = numa_local_shard(key, stripe_off, &drive_numa, &self.numa_shards); + let disk_snapshot = self.disk_channels.snapshot(cache_id); + let drive_numa = disk_snapshot + .as_ref() + .map(|snapshot| snapshot.drive_numa.as_slice()) + .unwrap_or_default(); + let pick = numa_local_shard(key, stripe_off, drive_numa, &self.numa_shards); if pick.cross_numa { crate::metrics::fanout_cross_numa_fetch(); diff --git a/cmd/unbounded-storage/src/p2p/handler.rs b/cmd/unbounded-storage/src/p2p/handler.rs index 51888c9d8..90e28f9bd 100644 --- a/cmd/unbounded-storage/src/p2p/handler.rs +++ b/cmd/unbounded-storage/src/p2p/handler.rs @@ -78,7 +78,11 @@ impl OwnerShardTable { return None; } - let drive_numa = self.cache_directories.drive_numa(req.cache_id()); + let disk_snapshot = self.cache_directories.snapshot(req.cache_id()); + let drive_numa = disk_snapshot + .as_ref() + .map(|snapshot| snapshot.drive_numa.as_slice()) + .unwrap_or_default(); let target_numa = if drive_numa.is_empty() { None } else { diff --git a/cmd/unbounded-storage/src/storage/disks/channels.rs b/cmd/unbounded-storage/src/storage/disks/channels.rs index 26d394a4c..7acbb9913 100644 --- a/cmd/unbounded-storage/src/storage/disks/channels.rs +++ b/cmd/unbounded-storage/src/storage/disks/channels.rs @@ -5,11 +5,11 @@ //! //! `DiskChannelDirectory` owns the set of currently-open disks (each //! represented by a [`PageChannel`] to its storage core) and -//! publishes a `(channels, generation)` pair via `ArcSwap`. A +//! publishes one immutable snapshot via `ArcSwap`. A //! [`Self::apply_channels`] call that actually changes the set builds //! a fresh pair and stores it as a single atomic publication, so -//! consumers that resolve through [`Self::snapshot`] observe the -//! channel set and its generation together; an apply of the identical +//! consumers that resolve through [`Self::snapshot`] observe channels, +//! topology, policy, and generation together; an apply of the identical //! set is skipped so the generation does not churn. Per-shard //! registration caches key off that generation, so "registered //! against gen N" always refers to the snapshot that was published as @@ -27,15 +27,14 @@ use arc_swap::ArcSwap; use crate::storage::PageChannel; -/// Snapshot of the currently-published channels. `None` means no -/// disks are open and the data path is offline. -pub type ChannelSnapshot = Option>; - /// Published channel metadata, aligned index-for-index by path-sorted /// disk order. pub struct ChannelSet { pub channels: Vec, pub page_cache_enabled: Vec, + pub drive_numa: Vec>, + pub generation: u64, + key: Vec<(PathBuf, usize, Option)>, } /// Owns the published per-disk channel set. A change-bearing @@ -43,27 +42,7 @@ pub struct ChannelSet { /// fresh, path-sorted [`PageChannel`] vector so `disk_for` indices /// stay stable across consumers. pub struct DiskChannelDirectory { - current: ArcSwap, -} - -/// Published unit: the channel snapshot and the generation it was -/// published under, kept together so a single `ArcSwap` load returns -/// a consistent pair. Consumers must go through -/// [`DiskChannelDirectory::snapshot`] for any staleness decision so the -/// pair cannot tear between two adjacent publications. -struct ChannelSnapshotInner { - channels: ChannelSnapshot, - generation: u64, - /// Per-drive NUMA node, aligned index-for-index with `channels` - /// (and therefore with the `disk_for` drive index). `None` for a - /// drive whose storage core is unpinned or whose node is unknown. - /// Empty (not `None`-filled) when no channels are published, which - /// is the signal the router uses to fall back to plain hashing. - drive_numa: Arc>>, - /// Path-sorted `(path, service identity, numa)` of the published - /// set. Policy-only changes republish the snapshot without bumping - /// the registration generation. - key: Vec<(PathBuf, usize, Option)>, + current: ArcSwap, } impl DiskChannelDirectory { @@ -71,10 +50,11 @@ impl DiskChannelDirectory { /// channels. pub fn new() -> Arc { Arc::new(Self { - current: ArcSwap::new(Arc::new(ChannelSnapshotInner { - channels: None, + current: ArcSwap::new(Arc::new(ChannelSet { + channels: Vec::new(), + page_cache_enabled: Vec::new(), + drive_numa: Vec::new(), generation: 0, - drive_numa: Arc::new(Vec::new()), key: Vec::new(), })), }) @@ -107,12 +87,7 @@ impl DiskChannelDirectory { // reading the previous bundle and storing the new one is // race-free. let prev = self.current.load(); - let prev_page_cache_enabled = prev - .channels - .as_ref() - .map(|set| set.page_cache_enabled.as_slice()) - .unwrap_or(&[]); - if prev.key == key && prev_page_cache_enabled == page_cache_enabled.as_slice() { + if prev.key == key && prev.page_cache_enabled == page_cache_enabled { // Identical publication: nothing downstream needs to // reseat, so skip the generation bump and the store. return; @@ -121,56 +96,35 @@ impl DiskChannelDirectory { let drive_numa: Vec> = channels.iter().map(|(_, _, numa, _)| *numa).collect(); let ordered: Vec = channels.into_iter().map(|(_, c, _, _)| c).collect(); let n = ordered.len(); - let snapshot: ChannelSnapshot = if ordered.is_empty() { - None - } else { - Some(Arc::new(ChannelSet { - channels: ordered, - page_cache_enabled, - })) - }; - // The atomic `ArcSwap::store` is the entire publication: any - // consumer that loads the bundle sees both fields from the - // same generation. + // consumer sees all fields from the same generation. let gen_n = if prev.key == key { prev.generation } else { prev.generation + 1 }; - self.current.store(Arc::new(ChannelSnapshotInner { - channels: snapshot, + self.current.store(Arc::new(ChannelSet { + channels: ordered, + page_cache_enabled, + drive_numa, generation: gen_n, - drive_numa: Arc::new(drive_numa), key, })); eprintln!("disks: hot-swap to generation {gen_n} (cache cold; channel count={n})"); } - /// Atomically load the currently-published channel set paired - /// with the generation it was published under. This is the only - /// API that yields a consistent (snapshot, gen) pair; staleness - /// checks must use it. - pub fn snapshot(&self) -> (ChannelSnapshot, u64) { - let inner = self.current.load_full(); - (inner.channels.clone(), inner.generation) + /// Atomically load the current channels, topology, policy, and + /// generation as one immutable snapshot. + pub fn snapshot(&self) -> Arc { + self.current.load_full() } /// Load the currently-published channel set. `None` when no /// disks are open. Prefer [`Self::snapshot`] in any context that /// also needs the generation. - pub fn current(&self) -> ChannelSnapshot { - self.current.load_full().channels.clone() - } - - /// Per-drive NUMA node, aligned with the published channel order - /// (and therefore with the `disk_for` drive index). Empty when no - /// disks are open, which the router treats as "no NUMA hint, hash - /// across all shards". A single atomic load, so the returned - /// vector's length always matches the channel set it was published - /// with. - pub fn drive_numa(&self) -> Arc>> { - self.current.load_full().drive_numa.clone() + pub fn current(&self) -> Option> { + let snapshot = self.snapshot(); + (!snapshot.channels.is_empty()).then_some(snapshot) } /// Generation of the currently-published snapshot. Observability @@ -196,7 +150,7 @@ mod tests { let t = DiskChannelDirectory::new(); assert!(t.current().is_none()); assert_eq!(t.generation(), 0); - assert!(t.drive_numa().is_empty()); + assert!(t.snapshot().drive_numa.is_empty()); } #[test] @@ -233,10 +187,11 @@ mod tests { (PathBuf::from("/a"), dummy_channel(), Some(0), true), (PathBuf::from("/b"), dummy_channel(), None, true), ]); - let numa = t.drive_numa(); + let snapshot = t.snapshot(); + let numa = &snapshot.drive_numa; // Path order /a,/b,/c -> NUMA 0, None, 2. - assert_eq!(&*numa, &[Some(0), None, Some(2)]); - assert_eq!(numa.len(), t.current().unwrap().channels.len()); + assert_eq!(numa, &[Some(0), None, Some(2)]); + assert_eq!(numa.len(), snapshot.channels.len()); } #[test] @@ -269,10 +224,10 @@ mod tests { let t = DiskChannelDirectory::new(); t.apply_channels(vec![(PathBuf::from("/a"), dummy_channel(), Some(0), true)]); assert!(t.current().is_some()); - assert_eq!(t.drive_numa().len(), 1); + assert_eq!(t.snapshot().drive_numa.len(), 1); t.apply_channels(vec![]); assert!(t.current().is_none()); - assert!(t.drive_numa().is_empty()); + assert!(t.snapshot().drive_numa.is_empty()); assert_eq!(t.generation(), 2); } @@ -284,22 +239,20 @@ mod tests { // matches the observability accessor; distinct apply calls // publish distinct snapshot `Arc`s. let t = DiskChannelDirectory::new(); - let (c0, g0) = t.snapshot(); - assert!(c0.is_none()); - assert_eq!(g0, 0); - assert_eq!(g0, t.generation()); + let c0 = t.snapshot(); + assert!(c0.channels.is_empty()); + assert_eq!(c0.generation, 0); + assert_eq!(c0.generation, t.generation()); t.apply_channels(vec![(PathBuf::from("/a"), dummy_channel(), None, true)]); - let (c1, g1) = t.snapshot(); - assert_eq!(g1, 1); - assert_eq!(g1, t.generation()); - let c1 = c1.expect("snapshot present after apply"); + let c1 = t.snapshot(); + assert_eq!(c1.generation, 1); + assert_eq!(c1.generation, t.generation()); t.apply_channels(vec![(PathBuf::from("/b"), dummy_channel(), None, true)]); - let (c2, g2) = t.snapshot(); - assert_eq!(g2, 2); - assert_eq!(g2, t.generation()); - let c2 = c2.expect("snapshot present after second apply"); + let c2 = t.snapshot(); + assert_eq!(c2.generation, 2); + assert_eq!(c2.generation, t.generation()); // Each apply publishes a fresh vector, so the returned `Arc`s // do not alias across generations. @@ -346,11 +299,11 @@ mod tests { let c = dummy_channel(); t.apply_channels(vec![(PathBuf::from("/a"), c.clone(), Some(0), true)]); assert_eq!(t.generation(), 1); - assert_eq!(&*t.drive_numa(), &[Some(0)]); + assert_eq!(t.snapshot().drive_numa, vec![Some(0)]); t.apply_channels(vec![(PathBuf::from("/a"), c.clone(), Some(1), true)]); assert_eq!(t.generation(), 2, "NUMA move must bump"); - assert_eq!(&*t.drive_numa(), &[Some(1)]); + assert_eq!(t.snapshot().drive_numa, vec![Some(1)]); } #[test] diff --git a/cmd/unbounded-storage/src/storage/disks/shard_view.rs b/cmd/unbounded-storage/src/storage/disks/shard_view.rs index f4d9d23d9..279ecdd5e 100644 --- a/cmd/unbounded-storage/src/storage/disks/shard_view.rs +++ b/cmd/unbounded-storage/src/storage/disks/shard_view.rs @@ -14,7 +14,7 @@ //! touches the device directly. //! //! On every `BlockStore` call the view loads the directory's published -//! `(channels, generation)` pair atomically and compares the +//! channel snapshot and compares its generation //! generation against the one it last replayed against; on a mismatch //! it re-registers every recorded backing against every channel in //! the newly-published set before delegating. Because the snapshot @@ -149,11 +149,10 @@ impl CacheDirectorySet { self.get_or_create(cache_id).apply_channels(channels); } - pub fn drive_numa(&self, cache_id: Option<&str>) -> Arc>> { + pub fn snapshot(&self, cache_id: Option<&str>) -> Option> { cache_id .and_then(|id| self.get(id)) - .map(|directory| directory.drive_numa()) - .unwrap_or_else(|| Arc::new(Vec::new())) + .map(|directory| directory.snapshot()) } } @@ -243,12 +242,14 @@ impl LiveShardLocalStore { /// [`DiskChannelDirectory::snapshot`] load, so the pair is /// consistent. Returns `None` when the directory has no channels. fn current_or_replay(&self) -> Option> { - let (set, gen_n) = self.directory.snapshot(); - let set = set?; + let set = self.directory.snapshot(); + if set.channels.is_empty() { + return None; + } let mut guard = self.state.lock().unwrap(); - if guard.last_seen_generation != Some(gen_n) { + if guard.last_seen_generation != Some(set.generation) { Self::replay_locked(&set.channels, &guard.registered); - guard.last_seen_generation = Some(gen_n); + guard.last_seen_generation = Some(set.generation); } Some(set) } diff --git a/cmd/unbounded-storage/tests/lifecycle/workload.rs b/cmd/unbounded-storage/tests/lifecycle/workload.rs index f4910789c..c3799fb4c 100644 --- a/cmd/unbounded-storage/tests/lifecycle/workload.rs +++ b/cmd/unbounded-storage/tests/lifecycle/workload.rs @@ -765,13 +765,13 @@ fn spawn_clients( yield_once().await; } for (op_idx, op) in client.ops.iter().enumerate() { - let (snapshot, generation) = directory.snapshot(); - max_snapshot_generation.set(max_snapshot_generation.get().max(generation)); - let Some(channels) = snapshot.as_ref() else { + let channels = directory.snapshot(); + max_snapshot_generation.set(max_snapshot_generation.get().max(channels.generation)); + if channels.channels.is_empty() { channel_errors.set(channel_errors.get() + 1); completed_ops.set(completed_ops.get() + 1); continue; - }; + } let slot = start_slot + op_idx; let base = unsafe { (pool_base as *mut u8).add(slot * page_size) }; match op { From a48236098a08091ceea49931d361e5c9bd3f99ff Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:20:01 +0000 Subject: [PATCH 18/56] unbounded-storage: share routing across caches --- cmd/unbounded-storage/src/fabric/tests.rs | 9 +-- .../src/frontend/loadgen_serve.rs | 10 ++- cmd/unbounded-storage/src/main.rs | 24 +++---- .../src/p2p/finger_router.rs | 14 ++-- cmd/unbounded-storage/src/p2p/handler.rs | 2 +- cmd/unbounded-storage/src/p2p/mod.rs | 2 +- .../src/p2p/routing_handle.rs | 66 +++++++++---------- cmd/unbounded-storage/src/p2p/transport.rs | 4 +- 8 files changed, 56 insertions(+), 75 deletions(-) diff --git a/cmd/unbounded-storage/src/fabric/tests.rs b/cmd/unbounded-storage/src/fabric/tests.rs index acdc95723..c95a91ea1 100644 --- a/cmd/unbounded-storage/src/fabric/tests.rs +++ b/cmd/unbounded-storage/src/fabric/tests.rs @@ -7,6 +7,7 @@ //! if the tcp provider is not installed in the libfabric build //! available to the test binary. +use std::collections::HashSet; use std::ffi::CString; use std::ptr; use std::sync::Arc; @@ -972,7 +973,7 @@ use crate::fanout::{ }; use crate::p2p::{ FingerTable, FingerTableConfig, NodeId, OwnerShardSource, OwnerShardTable, PeerEntry, - RecursiveHandler, RingId, RouteTableHandle, RoutingSnapshot, TopologyTags, + RecursiveHandler, RingId, RouteTableHandle, TopologyTags, }; use crate::storage::StripeReq; @@ -1084,11 +1085,7 @@ fn start_recursive_node( RecursiveHandler::with_routes( scratch, RECURSIVE_SCRATCH_PAGES as u32, - RouteTableHandle::new( - [(CHAIN_CACHE_ID.to_string(), RoutingSnapshot { fingers })] - .into_iter() - .collect(), - ), + RouteTableHandle::new(HashSet::from([CHAIN_CACHE_ID.to_string()]), fingers), fabric.clone(), scratch_mr, page_size, diff --git a/cmd/unbounded-storage/src/frontend/loadgen_serve.rs b/cmd/unbounded-storage/src/frontend/loadgen_serve.rs index fbb43fe11..108d60684 100644 --- a/cmd/unbounded-storage/src/frontend/loadgen_serve.rs +++ b/cmd/unbounded-storage/src/frontend/loadgen_serve.rs @@ -256,11 +256,11 @@ fn first_stripe_routes_remote(cfg: &LoadgenRun, object: SyntheticObjectId) -> bo let object_id = synthetic_object_id(object.seed, object.ordinal); let req = request_from_origin(OriginRef::new(&cfg.backend_id, &object_id, 0), cfg); - let Some(route) = cfg.routes.route_for_req(&req) else { + let Some(fingers) = cfg.routes.route_for_req(&req) else { return false; }; - route.fingers.next_hop(stripe_to_ring(req.key())).is_some() + fingers.next_hop(stripe_to_ring(req.key())).is_some() } struct YieldOnce { @@ -481,6 +481,7 @@ fn unit_f64(rng: &mut WorkerRng) -> f64 { #[cfg(test)] mod tests { use super::*; + use std::collections::HashSet; use std::pin::Pin; use std::task::{RawWaker, RawWakerVTable, Waker}; @@ -559,10 +560,7 @@ mod tests { &[peer], FingerTableConfig::with_k(4), )); - let mut routes = std::collections::HashMap::new(); - routes.insert("cache".to_string(), crate::p2p::RoutingSnapshot { fingers }); - - RouteTableHandle::new(routes) + RouteTableHandle::new(HashSet::from(["cache".to_string()]), fingers) } fn find_object_with_remote_route(cfg: &LoadgenRun, want_remote: bool) -> SyntheticObjectId { diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 28fb0043a..2acc947ae 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -29,8 +29,8 @@ use unbounded_storage::frontend::{ }; use unbounded_storage::p2p::{ FingerTable, FingerTableConfig, PeerEntry, RouteTableHandle, RouteTableSnapshot, - RoutedTransport, RoutingSnapshot, TopologyPrefixWeight, TopologySelection, TopologyTags, - TopologyWeighting, node_to_ring, + RoutedTransport, TopologyPrefixWeight, TopologySelection, TopologyTags, TopologyWeighting, + node_to_ring, }; use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; @@ -1109,7 +1109,8 @@ fn build_routes(config: &Config) -> RouteTableSnapshot { let projection = config::runtime_projection(config).expect("loaded config projects to runtime"); if projection.caches.is_empty() { return RouteTableSnapshot { - routes: HashMap::new(), + cache_ids: HashSet::new(), + fingers: None, }; } @@ -1165,19 +1166,10 @@ fn build_routes(config: &Config) -> RouteTableSnapshot { }, )) }; - let routes = projection - .caches - .keys() - .map(|id| { - ( - id.clone(), - RoutingSnapshot { - fingers: fingers.clone(), - }, - ) - }) - .collect(); - RouteTableSnapshot { routes } + RouteTableSnapshot { + cache_ids: projection.caches.keys().cloned().collect(), + fingers: Some(fingers), + } } fn p2p_topology_weighting(weighting: &config::TopologyWeighting) -> TopologyWeighting { diff --git a/cmd/unbounded-storage/src/p2p/finger_router.rs b/cmd/unbounded-storage/src/p2p/finger_router.rs index ae0fd6ca3..e6379bd6f 100644 --- a/cmd/unbounded-storage/src/p2p/finger_router.rs +++ b/cmd/unbounded-storage/src/p2p/finger_router.rs @@ -29,8 +29,8 @@ impl ChainFingerRouter { impl PeerRouter for ChainFingerRouter { fn route(&self, req: &R) -> Option { - let snap = self.routes.route_for_req(req)?; - snap.fingers + self.routes + .route_for_req(req)? .next_hop(stripe_to_ring(req.key())) .map(|peer| PeerId(peer.node.0)) } @@ -38,7 +38,7 @@ impl PeerRouter for ChainFingerRouter { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::HashSet; use std::sync::Arc; use crate::bufferpool::{Req, StripeKey}; @@ -82,10 +82,10 @@ mod tests { } fn router(fingers: Arc) -> ChainFingerRouter { - ChainFingerRouter::new(crate::p2p::RouteTableHandle::new(HashMap::from([( - "cache-a".to_string(), - crate::p2p::RoutingSnapshot { fingers }, - )]))) + ChainFingerRouter::new(crate::p2p::RouteTableHandle::new( + HashSet::from(["cache-a".to_string()]), + fingers, + )) } fn req(key: StripeKey) -> TestReq { diff --git a/cmd/unbounded-storage/src/p2p/handler.rs b/cmd/unbounded-storage/src/p2p/handler.rs index da92b498f..de48afdfd 100644 --- a/cmd/unbounded-storage/src/p2p/handler.rs +++ b/cmd/unbounded-storage/src/p2p/handler.rs @@ -210,7 +210,7 @@ impl Handler for RecursiveHandler { ) -> Self::Stream<'a> { RecursiveHandlerStream { scratch: self.scratch.clone(), - fingers: self.routes.route_for_req(req).map(|route| route.fingers), + fingers: self.routes.route_for_req(req), forward: &self.forward, owners: &self.owners, req, diff --git a/cmd/unbounded-storage/src/p2p/mod.rs b/cmd/unbounded-storage/src/p2p/mod.rs index f488c91f2..d2098a82a 100644 --- a/cmd/unbounded-storage/src/p2p/mod.rs +++ b/cmd/unbounded-storage/src/p2p/mod.rs @@ -30,6 +30,6 @@ pub use handler::{ RecursiveHandlerStream, }; pub use ring::{node_id_from_name, node_to_ring, splitmix64, stripe_to_ring}; -pub use routing_handle::{RouteTableHandle, RouteTableSnapshot, RoutingSnapshot}; +pub use routing_handle::{RouteTableHandle, RouteTableSnapshot}; pub use transport::{RoutedStream, RoutedTransport}; pub use types::{NodeId, P2pReq, PeerEntry, RingId, TopologyTags}; diff --git a/cmd/unbounded-storage/src/p2p/routing_handle.rs b/cmd/unbounded-storage/src/p2p/routing_handle.rs index f54e4d2ee..05a52c5a3 100644 --- a/cmd/unbounded-storage/src/p2p/routing_handle.rs +++ b/cmd/unbounded-storage/src/p2p/routing_handle.rs @@ -24,35 +24,29 @@ //! //! [`RecursiveHandler`]: crate::p2p::RecursiveHandler -use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; use arc_swap::ArcSwap; use crate::p2p::FingerTable; -/// Immutable routing surface for one cache. -/// -/// Cloning shares the underlying finger table. -#[derive(Clone)] -pub struct RoutingSnapshot { - pub fingers: Arc, -} - #[derive(Clone, Default)] pub struct RouteTableSnapshot { - pub routes: HashMap, + pub cache_ids: HashSet, + pub fingers: Option>, } -/// Shared, live-reloadable handle to cache-keyed routing snapshots. +/// Shared, live-reloadable handle to one routing table and its cache membership. #[derive(Clone)] pub struct RouteTableHandle(Arc>); impl RouteTableHandle { - pub fn new(routes: HashMap) -> Self { - Self(Arc::new(ArcSwap::from_pointee(RouteTableSnapshot { - routes, - }))) + pub fn new(cache_ids: HashSet, fingers: Arc) -> Self { + Self::from_snapshot(RouteTableSnapshot { + cache_ids, + fingers: Some(fingers), + }) } pub fn from_snapshot(snapshot: RouteTableSnapshot) -> Self { @@ -60,11 +54,14 @@ impl RouteTableHandle { } pub fn empty() -> Self { - Self::new(HashMap::new()) + Self::from_snapshot(RouteTableSnapshot::default()) } - pub fn store(&self, routes: HashMap) { - self.0.store(Arc::new(RouteTableSnapshot { routes })); + pub fn store(&self, cache_ids: HashSet, fingers: Arc) { + self.store_snapshot(RouteTableSnapshot { + cache_ids, + fingers: Some(fingers), + }); } pub fn store_snapshot(&self, snapshot: RouteTableSnapshot) { @@ -75,14 +72,19 @@ impl RouteTableHandle { self.0.load_full() } - pub fn route(&self, route_id: &str) -> Option { - self.snapshot().routes.get(route_id).cloned() + pub fn route(&self, route_id: &str) -> Option> { + let snapshot = self.snapshot(); + snapshot + .cache_ids + .contains(route_id) + .then(|| snapshot.fingers.clone()) + .flatten() } pub fn route_for_req( &self, req: &R, - ) -> Option { + ) -> Option> { self.route(req.cache_id()?.as_str()) } } @@ -115,26 +117,18 @@ mod tests { // every clone shares one ArcSwap, so this is the property the // shard relies on to fan a reload out to consumers it already // handed clones to. - let handle = RouteTableHandle::new(HashMap::from([( - "cache-a".to_string(), - RoutingSnapshot { - fingers: table(1, &[]), - }, - )])); + let handle = RouteTableHandle::new(HashSet::from(["cache-a".to_string()]), table(1, &[])); let consumer = handle.clone(); let before = consumer.route("cache-a").expect("route"); - assert!(before.fingers.next_hop(node_to_ring(NodeId(2))).is_none()); + assert!(before.next_hop(node_to_ring(NodeId(2))).is_none()); + assert!(consumer.route("cache-b").is_none()); - handle.store(HashMap::from([( - "cache-a".to_string(), - RoutingSnapshot { - fingers: table(1, &[2]), - }, - )])); + handle.store(HashSet::from(["cache-b".to_string()]), table(1, &[2])); - let snap = consumer.route("cache-a").expect("route"); + assert!(consumer.route("cache-a").is_none()); + let fingers = consumer.route("cache-b").expect("route"); assert_eq!( - snap.fingers + fingers .next_hop(node_to_ring(NodeId(2))) .map(|peer| peer.node), Some(NodeId(2)), diff --git a/cmd/unbounded-storage/src/p2p/transport.rs b/cmd/unbounded-storage/src/p2p/transport.rs index d7b26b686..34903acfd 100644 --- a/cmd/unbounded-storage/src/p2p/transport.rs +++ b/cmd/unbounded-storage/src/p2p/transport.rs @@ -77,10 +77,10 @@ impl> RoutedTransport { where R: Req, { - let Some(route) = self.routes.route_for_req(req) else { + let Some(fingers) = self.routes.route_for_req(req) else { return true; }; - route.fingers.next_hop(stripe_to_ring(req.key())).is_none() + fingers.next_hop(stripe_to_ring(req.key())).is_none() } } From 001a71f2eaabff2671384a53a1629a29ff5c91bc Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 23:20:08 +0000 Subject: [PATCH 19/56] unbounded-storage: make fetch limiter shard-local --- cmd/unbounded-storage/src/backend/limiter.rs | 79 +++++++++----------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/limiter.rs b/cmd/unbounded-storage/src/backend/limiter.rs index ed8f836a6..97ae905b3 100644 --- a/cmd/unbounded-storage/src/backend/limiter.rs +++ b/cmd/unbounded-storage/src/backend/limiter.rs @@ -9,23 +9,17 @@ //! blocks a thread: it parks the fetch future and wakes it when a //! permit frees up. //! -//! The interior is `Arc>` rather than `Rc>` -//! because a single backend is shared by the data-path transport (the -//! shard thread) and the recursive RPC handler (the ephemeral -//! `fabric-rpc-worker` threads that drive a `WorkerLocal` ring). Those -//! threads can clone the limiter into their `'static` fetch futures -//! concurrently, so the permit pool must be a thread-safe handle. The -//! critical sections are tiny (a counter check plus a waker push/pop) -//! and uncontended on the single-threaded shard path. +//! The limiter is shard-local. Its handles may outlive a backend registry +//! entry through in-flight fetch futures, but they never cross threads. +use std::cell::RefCell; use std::collections::VecDeque; use std::future::Future; use std::pin::Pin; -use std::sync::{Arc, Mutex}; +use std::rc::Rc; use std::task::{Context, Poll, Waker}; -/// A clonable handle to a permit pool shared across the shard thread -/// and the RPC worker threads. +/// A clonable shard-local handle to an origin fetch permit pool. /// /// `limit` permits are available at construction. `acquire().await` /// yields a [`FetchPermit`] once a permit is free; dropping the permit @@ -33,7 +27,7 @@ use std::task::{Context, Poll, Waker}; /// is treated as one so a backend always makes progress. #[derive(Clone)] pub struct FetchLimiter { - inner: Arc>, + inner: Rc>, } struct LimiterInner { @@ -52,7 +46,7 @@ struct LimiterInner { impl FetchLimiter { pub fn new(limit: usize) -> Self { Self { - inner: Arc::new(Mutex::new(LimiterInner { + inner: Rc::new(RefCell::new(LimiterInner { available: limit.max(1), waiters: VecDeque::new(), next_id: 0, @@ -63,7 +57,7 @@ impl FetchLimiter { /// Acquire a permit, parking until one is free. pub fn acquire(&self) -> Acquire { Acquire { - inner: Arc::clone(&self.inner), + inner: Rc::clone(&self.inner), id: None, } } @@ -71,11 +65,11 @@ impl FetchLimiter { /// Permits currently free. Exposed for tests. #[cfg(test)] pub fn available(&self) -> usize { - self.inner.lock().expect("limiter poisoned").available + self.inner.borrow().available } } -/// Future returned by [`FetchLimiter::acquire`]. Owns an `Arc` clone of +/// Future returned by [`FetchLimiter::acquire`]. Owns an `Rc` clone of /// the pool so it can live inside a `'static` fetch future. /// /// While parked it holds a uniquely-identified slot in the pool's waiter @@ -83,7 +77,7 @@ impl FetchLimiter { /// (cancelled), so a cancelled acquirer can never leave an orphaned /// waker behind to swallow a later wake-up meant for a live waiter. pub struct Acquire { - inner: Arc>, + inner: Rc>, /// Id of this future's slot in `waiters`, assigned the first time it /// parks. `None` until then (and irrelevant once resolved). id: Option, @@ -93,20 +87,20 @@ impl Future for Acquire { type Output = FetchPermit; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // `Acquire` is `Unpin` (it holds only an `Arc` and an `Option`), + // `Acquire` is `Unpin` (it holds only an `Rc` and an `Option`), // so taking a `&mut Self` out of the pin is sound. let this = self.get_mut(); - let mut guard = this.inner.lock().expect("limiter poisoned"); - if guard.available > 0 { - guard.available -= 1; + let mut inner = this.inner.borrow_mut(); + if inner.available > 0 { + inner.available -= 1; // We are about to resolve: drop any slot we registered so a // later release does not waste a wake on us. if let Some(id) = this.id.take() { - guard.waiters.retain(|(i, _)| *i != id); + inner.waiters.retain(|(i, _)| *i != id); } - drop(guard); + drop(inner); return Poll::Ready(FetchPermit { - inner: Arc::clone(&this.inner), + inner: Rc::clone(&this.inner), }); } // No permit free: register (or refresh) our waker under our own @@ -115,16 +109,16 @@ impl Future for Acquire { let id = match this.id { Some(id) => id, None => { - let id = guard.next_id; - guard.next_id += 1; + let id = inner.next_id; + inner.next_id += 1; this.id = Some(id); id } }; - if let Some(slot) = guard.waiters.iter_mut().find(|(i, _)| *i == id) { + if let Some(slot) = inner.waiters.iter_mut().find(|(i, _)| *i == id) { slot.1 = cx.waker().clone(); } else { - guard.waiters.push_back((id, cx.waker().clone())); + inner.waiters.push_back((id, cx.waker().clone())); } Poll::Pending } @@ -148,16 +142,15 @@ impl Drop for Acquire { fn drop(&mut self) { if let Some(id) = self.id { let waker = { - let mut guard = self.inner.lock().expect("limiter poisoned"); - guard.waiters.retain(|(i, _)| *i != id); - if guard.available > 0 { - guard.waiters.front().map(|(_, w)| w.clone()) + let mut inner = self.inner.borrow_mut(); + inner.waiters.retain(|(i, _)| *i != id); + if inner.available > 0 { + inner.waiters.front().map(|(_, w)| w.clone()) } else { None } }; - // Wake outside the lock so a waker that re-enters the limiter - // cannot deadlock, matching `FetchPermit::drop`. + // Wake outside the borrow so an eager re-poll can re-enter. if let Some(waker) = waker { waker.wake(); } @@ -168,18 +161,17 @@ impl Drop for Acquire { /// An in-flight permit. Returns itself to the pool on drop and wakes /// the next parked waiter, if any. pub struct FetchPermit { - inner: Arc>, + inner: Rc>, } impl Drop for FetchPermit { fn drop(&mut self) { let waker = { - let mut guard = self.inner.lock().expect("limiter poisoned"); - guard.available += 1; - guard.waiters.pop_front().map(|(_, w)| w) + let mut inner = self.inner.borrow_mut(); + inner.available += 1; + inner.waiters.pop_front().map(|(_, w)| w) }; - // Wake outside the lock so a waker that re-enters the limiter - // (e.g. an eager re-poll on the same thread) cannot deadlock. + // Wake outside the borrow so an eager re-poll can re-enter. if let Some(waker) = waker { waker.wake(); } @@ -267,10 +259,9 @@ mod tests { /// A cancelled (dropped-while-parked) acquirer must not swallow the /// wake a later release owes to a live waiter behind it. This is the - /// cross-task case the limiter actually faces: the shard thread and - /// the RPC worker threads share one limiter, so parked acquirers can - /// carry distinct wakers. Regression for the orphaned-waker lost - /// wakeup. + /// cross-task case the limiter actually faces: concurrent shard-local + /// fetches can carry distinct wakers. Regression for the orphaned-waker + /// lost wakeup. #[test] fn cancelled_waiter_does_not_strand_live_waiter() { let limiter = FetchLimiter::new(1); From 9d1531f69f6afb551782d342a76a14a5f469e769 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 20:10:57 +0000 Subject: [PATCH 20/56] unbounded-storage: deduplicate backing registration --- .../src/storage/disks/channels.rs | 4 +- .../src/storage/disks/shard_view.rs | 190 +++++++++++------- .../src/storage/page_channel.rs | 21 +- 3 files changed, 131 insertions(+), 84 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/disks/channels.rs b/cmd/unbounded-storage/src/storage/disks/channels.rs index 7acbb9913..6287cbd6b 100644 --- a/cmd/unbounded-storage/src/storage/disks/channels.rs +++ b/cmd/unbounded-storage/src/storage/disks/channels.rs @@ -34,7 +34,7 @@ pub struct ChannelSet { pub page_cache_enabled: Vec, pub drive_numa: Vec>, pub generation: u64, - key: Vec<(PathBuf, usize, Option)>, + key: Vec<(PathBuf, u64, Option)>, } /// Owns the published per-disk channel set. A change-bearing @@ -73,7 +73,7 @@ impl DiskChannelDirectory { /// do not needlessly reseat per-shard registrations. pub fn apply_channels(&self, mut channels: Vec<(PathBuf, PageChannel, Option, bool)>) { channels.sort_by(|a, b| a.0.cmp(&b.0)); - let key: Vec<(PathBuf, usize, Option)> = channels + let key: Vec<(PathBuf, u64, Option)> = channels .iter() .map(|(p, c, numa, _)| (p.clone(), c.service_id(), *numa)) .collect(); diff --git a/cmd/unbounded-storage/src/storage/disks/shard_view.rs b/cmd/unbounded-storage/src/storage/disks/shard_view.rs index 279ecdd5e..9f60514a2 100644 --- a/cmd/unbounded-storage/src/storage/disks/shard_view.rs +++ b/cmd/unbounded-storage/src/storage/disks/shard_view.rs @@ -4,8 +4,7 @@ //! Per-shard [`BlockStore`] view over a hot-swappable //! [`DiskChannelDirectory`]. //! -//! Each shard holds a [`LiveShardLocalStore`] and registers its -//! NUMA-local backing through [`Self::register_backing`]. The view +//! Each shard registers one NUMA-local backing. The view //! resolves the `PageRef`s emitted by its `Pool` to absolute byte //! ranges in that backing, picks the owning disk with //! [`crate::storage::local::disk_for`], and ships the raw `(ptr, len)` @@ -13,14 +12,10 @@ //! and its ring run entirely on the storage core; the shard never //! touches the device directly. //! -//! On every `BlockStore` call the view loads the directory's published -//! channel snapshot and compares its generation -//! generation against the one it last replayed against; on a mismatch -//! it re-registers every recorded backing against every channel in -//! the newly-published set before delegating. Because the snapshot -//! and generation come from a single `ArcSwap` load, "last_seen == -//! N" always means the recorded set was registered against the -//! snapshot published as gen N. +//! `LiveShardLocalStore` replays registrations by directory generation. +//! The production `ChainLocalStore` instead tracks storage service IDs +//! once per shard, so cache IDs that publish the same channels do not +//! repeat fixed-buffer registration. //! //! When the directory has no channels published (no projected cache disks) //! reads and writes return [`Error::Transport`] - the data path is @@ -54,20 +49,19 @@ pub struct CacheDirectorySet { /// cache's own [`DiskChannelDirectory`] and replay state. pub struct ChainLocalStore { directories: Arc, - stores: Mutex>, - registered: Mutex>, -} - -struct StoreEntry { - directory: Arc, - store: Arc, + state: Mutex, } struct RegisteredBacking { base: *mut u8, page_size: usize, page_count: usize, - keepalive: Arc, + _keepalive: Arc, +} + +struct ChainState { + backing: Option, + registered_services: HashSet, } /// Registered backings plus the directory generation we last replayed @@ -160,37 +154,48 @@ impl ChainLocalStore { pub fn new(directories: Arc) -> Arc { Arc::new(Self { directories, - stores: Mutex::new(HashMap::new()), - registered: Mutex::new(Vec::new()), + state: Mutex::new(ChainState { + backing: None, + registered_services: HashSet::new(), + }), }) } - fn store_for(&self, cache_id: &str) -> Arc { - let directory = self.directories.get_or_create(cache_id); - let mut guard = self.stores.lock().unwrap(); - if let Some(entry) = guard.get(cache_id) { - if Arc::ptr_eq(&entry.directory, &directory) { - return entry.store.clone(); + fn prepare( + &self, + channels: &super::channels::ChannelSet, + page: PageRef, + ) -> Result<(*mut u8, usize), Error> { + let mut state = self.state.lock().unwrap(); + let Some(backing) = state.backing.as_ref() else { + return Err(Error::Io(libc::ENXIO)); + }; + let base = backing.base; + let page_size = backing.page_size; + let page_count = backing.page_count; + let bytes = page_size * page_count; + let mut first_error = None; + + for channel in &channels.channels { + let service_id = channel.service_id(); + if !state.registered_services.contains(&service_id) { + if let Err(error) = channel.register_buffer(base, bytes) { + eprintln!("disks: register_buffer failed: {error:?}"); + first_error.get_or_insert(error); + continue; + } + state.registered_services.insert(service_id); } } - let store = Arc::new(LiveShardLocalStore::new(directory.clone())); - for backing in self.registered.lock().unwrap().iter() { - let backing = crate::memory::Backing { - base: backing.base, - page_size: backing.page_size, - page_count: backing.page_count, - keepalive: backing.keepalive.clone(), - }; - let _ = store.register_pages(&backing); + if let Some(error) = first_error { + return Err(error); } - guard.insert( - cache_id.to_string(), - StoreEntry { - directory, - store: store.clone(), - }, - ); - store + + debug_assert!((page.page_idx as usize) < page_count); + // SAFETY: `page_idx < page_count` and the pool guarantees + // `offset + len <= page_size` for its shard-owned backing. + let ptr = unsafe { base.add(page.page_idx as usize * page_size + page.offset as usize) }; + Ok((ptr, page.len as usize)) } } @@ -357,22 +362,18 @@ impl BlockStore for LiveShardLocalStore { impl BlockStore for ChainLocalStore { fn register_pages(&self, backing: &crate::memory::Backing) -> Result<(), Error> { - let mut registered = self.registered.lock().unwrap(); - if !registered.is_empty() { + let mut state = self.state.lock().unwrap(); + if state.backing.is_some() { return Err(Error::BadConfig( "register_pages: chain store already has a registered backing", )); } - registered.push(RegisteredBacking { + state.backing = Some(RegisteredBacking { base: backing.base, page_size: backing.page_size, page_count: backing.page_count, - keepalive: backing.keepalive.clone(), + _keepalive: backing.keepalive.clone(), }); - drop(registered); - for entry in self.stores.lock().unwrap().values() { - entry.store.register_pages(backing)?; - } Ok(()) } @@ -412,11 +413,15 @@ impl BlockStore for ChainLocalStore { let Some(directory) = self.directories.get(cache_id) else { return Ok(false); }; - if directory.current().is_none() { + let channels = directory.snapshot(); + if channels.channels.is_empty() { return Ok(false); } - self.store_for(cache_id) - .read_page(req, stripe_off, dst) + let (p, len) = self.prepare(&channels, dst)?; + let idx = disk_for(&req.key(), stripe_off, channels.channels.len()); + let slice = std::ptr::slice_from_raw_parts_mut(p, len); + channels.channels[idx] + .read_page(req.key(), stripe_off, slice) .await } @@ -432,11 +437,15 @@ impl BlockStore for ChainLocalStore { let Some(directory) = self.directories.get(cache_id) else { return Ok(()); }; - if directory.current().is_none() { + let channels = directory.snapshot(); + if channels.channels.is_empty() { return Ok(()); } - self.store_for(cache_id) - .write_page(req, stripe_off, page) + let (p, len) = self.prepare(&channels, page)?; + let idx = disk_for(&req.key(), stripe_off, channels.channels.len()); + let slice = std::ptr::slice_from_raw_parts(p.cast_const(), len); + channels.channels[idx] + .write_page(req.key(), stripe_off, slice) .await } } @@ -746,7 +755,7 @@ mod tests { } #[test] - fn cache_id_readd_rebinds_chain_store_to_new_directory() { + fn cache_id_readd_registers_replacement_service() { let dirs = CacheDirectorySet::new(); dirs.reconcile(["cache-a"]); let core1 = Core::spawn(); @@ -758,7 +767,15 @@ mod tests { let view = ChainLocalStore::new(dirs.clone()); let (_buf, backing) = make_backing(2); view.register_pages(&backing).unwrap(); - let first = view.store_for("cache-a"); + let req = StripeReq::new(StripeKey([0x22; 32])).with_cache_id(Some("cache-a".into())); + let page = PageRef { + page_idx: 0, + offset: 0, + len: 4096, + }; + let before1 = core1.registers.load(Ordering::Relaxed); + block_on(view.write_page(&req, 0, page)).unwrap(); + assert_eq!(core1.registers.load(Ordering::Relaxed), before1 + 1); dirs.reconcile(std::iter::empty::<&str>()); dirs.reconcile(["cache-a"]); @@ -768,17 +785,9 @@ mod tests { vec![(PathBuf::from("/b"), core2.channel.clone(), None, true)], ); - let second = view.store_for("cache-a"); - assert!( - !Arc::ptr_eq(&first, &second), - "re-added cache id must get a store bound to the new directory", - ); - assert_eq!( - second.state.lock().unwrap().registered.len(), - 1, - "registered backing must be replayed into replacement store", - ); - assert!(second.current_or_replay().is_some()); + let before2 = core2.registers.load(Ordering::Relaxed); + block_on(view.write_page(&req, 0, page)).unwrap(); + assert_eq!(core2.registers.load(Ordering::Relaxed), before2 + 1); core1.shutdown(); core2.shutdown(); @@ -796,8 +805,8 @@ mod tests { assert!(!view.page_cache_enabled(&req, 0)); assert!( - view.stores.lock().unwrap().is_empty(), - "policy lookup must not instantiate or replay a live store", + view.state.lock().unwrap().registered_services.is_empty(), + "policy lookup must not register a backing", ); } @@ -813,11 +822,42 @@ mod tests { assert!(!view.page_cache_enabled(&req, 0)); assert!( - view.stores.lock().unwrap().is_empty(), - "no-cache policy lookup must not instantiate a live store", + view.state.lock().unwrap().registered_services.is_empty(), + "no-cache policy lookup must not register a backing", ); } + #[test] + fn shared_service_is_registered_once_across_cache_ids() { + let dirs = CacheDirectorySet::new(); + dirs.reconcile(["cache-a", "cache-b"]); + let core = Core::spawn(); + for cache_id in ["cache-a", "cache-b"] { + dirs.apply_channels( + cache_id, + vec![(PathBuf::from("/a"), core.channel.clone(), None, true)], + ); + } + + let view = ChainLocalStore::new(dirs); + let (_buf, backing) = make_backing(2); + view.register_pages(&backing).unwrap(); + let page = PageRef { + page_idx: 0, + offset: 0, + len: 4096, + }; + let before = core.registers.load(Ordering::Relaxed); + for cache_id in ["cache-a", "cache-b"] { + let req = + StripeReq::new(StripeKey([0x33; 32])).with_cache_id(Some(cache_id.to_string())); + block_on(view.write_page(&req, 0, page)).unwrap(); + } + assert_eq!(core.registers.load(Ordering::Relaxed), before + 1); + + core.shutdown(); + } + #[test] fn chain_store_rejects_second_backing() { let t = DiskChannelDirectory::new(); @@ -830,6 +870,6 @@ mod tests { let (_b2, backing2) = make_backing(4); let err = view.register_pages(&backing2); assert!(matches!(err, Err(Error::BadConfig(_)))); - assert_eq!(view.registered.lock().unwrap().len(), 1); + assert!(view.state.lock().unwrap().backing.is_some()); } } diff --git a/cmd/unbounded-storage/src/storage/page_channel.rs b/cmd/unbounded-storage/src/storage/page_channel.rs index 51edf49e9..9c0bc6356 100644 --- a/cmd/unbounded-storage/src/storage/page_channel.rs +++ b/cmd/unbounded-storage/src/storage/page_channel.rs @@ -38,7 +38,7 @@ use std::future::Future; use std::pin::Pin; use std::ptr::NonNull; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; @@ -48,6 +48,8 @@ use crate::runtime::noop_waker; use crate::storage::blockdev::BlockDevice; use crate::storage::engine::StorageEngine; +static NEXT_SERVICE_ID: AtomicU64 = AtomicU64::new(1); + /// `Send + Sync` clonable handle to a storage core's /// [`PageService`]. Every clone targets the same service; when the /// last handle is dropped the channel disconnects and the service @@ -56,6 +58,7 @@ use crate::storage::engine::StorageEngine; /// See the module docs for the buffer-lifetime contract. pub struct PageChannel { cmd_tx: Sender, + service_id: u64, /// Flipped to `false` by the service during shutdown so a send /// that raced the service's exit resolves with `EIO` instead of /// parking on a reply slot that nobody will set. Same mechanism @@ -70,9 +73,13 @@ impl PageChannel { pub fn new() -> (PageChannel, PageChannelReceiver) { let (tx, rx) = channel(); let service_alive = Arc::new(AtomicBool::new(true)); + let service_id = NEXT_SERVICE_ID + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .expect("page service id space exhausted"); ( PageChannel { cmd_tx: tx, + service_id, service_alive: service_alive.clone(), }, PageChannelReceiver { rx, service_alive }, @@ -157,12 +164,11 @@ impl PageChannel { /// Stable identity of the storage-core service this channel /// targets. All clones of one channel share it; a channel built by /// a distinct [`Self::new`] has a distinct identity. Used by the - /// channel directory to skip republishing an unchanged channel set - /// (the `service_alive` allocation is kept alive by the holder for - /// as long as the channel is published, so the pointer is a sound - /// identity for live channels). - pub(crate) fn service_id(&self) -> usize { - Arc::as_ptr(&self.service_alive) as usize + /// channel directory and shard registration cache to identify the + /// service across channel clones. IDs are never reused within a + /// process, including after a service is removed. + pub(crate) fn service_id(&self) -> u64 { + self.service_id } } @@ -170,6 +176,7 @@ impl Clone for PageChannel { fn clone(&self) -> Self { Self { cmd_tx: self.cmd_tx.clone(), + service_id: self.service_id, service_alive: self.service_alive.clone(), } } From 7fa2271e9ac36ab3bb3973a87548562f56bb9bd1 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:26:03 +0000 Subject: [PATCH 21/56] unbounded-storage: remove unused pool group --- cmd/unbounded-storage/ARCHITECTURE.md | 5 +- cmd/unbounded-storage/src/bufferpool/group.rs | 169 ------------------ cmd/unbounded-storage/src/bufferpool/mod.rs | 2 - cmd/unbounded-storage/src/fanout/router.rs | 5 +- cmd/unbounded-storage/src/main.rs | 21 +-- cmd/unbounded-storage/src/shard_layer.rs | 35 +--- 6 files changed, 14 insertions(+), 223 deletions(-) delete mode 100644 cmd/unbounded-storage/src/bufferpool/group.rs diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 58a88130c..d495b09b0 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -346,9 +346,8 @@ stripe, and knows how to fill a miss from either a peer or the local disk. local, futures-core-free `Stream` of `Result`); and `Transport { bulk_get(&req, src: BulkRef, dsts: &[PageRef]) -> Stream }`, which fetches from a **peer**. Blanket impls cover `Arc`. -- Public surface also includes `PoolGroup`/`ShardDescriptor`/`ShardRouter` - (sharding helpers), `PageGuard`/`ReadStream`/`WindowedRead` (read API), and - `NullBlockStore`. +- Public surface also includes `PageGuard`/`ReadStream`/`WindowedRead` (read + API) and `NullBlockStore`. A shard's data path therefore has two miss sources: the `Transport` (peer pull over fabric) and the `BlockStore` (local NVMe). The `RoutedTransport` decides diff --git a/cmd/unbounded-storage/src/bufferpool/group.rs b/cmd/unbounded-storage/src/bufferpool/group.rs deleted file mode 100644 index 02710e504..000000000 --- a/cmd/unbounded-storage/src/bufferpool/group.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::sync::Arc; - -use crate::bufferpool::traits::Req; -use crate::runtime::WorkerIdx; - -/// Static metadata about one NUMA shard. Cheap to clone; held by -/// `PoolGroup` so cross-shard routing can answer "which shard -/// owns request R?" without touching the shard's `Pool` (which -/// lives only inside the shard thread). -#[derive(Clone, Debug)] -pub struct ShardDescriptor { - /// Worker slot this shard runs on. Matches the `WorkerIdx` - /// passed to `Threading::spawn_pinned` for the shard's thread. - pub worker_idx: WorkerIdx, - /// NUMA node this shard is pinned against, if the runtime is - /// NUMA-aware. `None` for the TCP-fallback / non-pinning case. - pub numa: Option, -} - -/// Routing function: `req -> shard index`. Returns an index into -/// the `PoolGroup`'s shard table. Implementations should be pure -/// (no I/O, no interior mutability) so a single request always -/// routes to the same shard for the lifetime of the group. -pub trait ShardRouter: Send + Sync + 'static { - fn route(&self, req: &R) -> usize; -} - -impl ShardRouter for F -where - R: Req, - F: Fn(&R) -> usize + Send + Sync + 'static, -{ - fn route(&self, req: &R) -> usize { - (self)(req) - } -} - -/// Per-process fan-out over NUMA shards. `R` is the bufferpool -/// request type; the embedder selects the shard for each request -/// via [`ShardRouter`]. -/// -/// Construction: build a `Vec` mirroring whatever -/// per-shard threads the embedder spawned, then wrap in a -/// `PoolGroup` together with a router closure. The descriptors are -/// authoritative for membership; the router is authoritative for -/// "which one for this request." -pub struct PoolGroup { - shards: Arc<[ShardDescriptor]>, - router: Arc>, -} - -impl PoolGroup { - pub fn new(shards: Vec, router: S) -> Self - where - S: ShardRouter, - { - assert!(!shards.is_empty(), "PoolGroup requires at least one shard"); - Self { - shards: Arc::from(shards.into_boxed_slice()), - router: Arc::new(router), - } - } - - pub fn shards(&self) -> &[ShardDescriptor] { - &self.shards - } - - pub fn len(&self) -> usize { - self.shards.len() - } - - /// Returns `true` iff [`Self::len`] is zero. Provided to match - /// clippy's `len_zero` lint expectation; `PoolGroup::new` - /// panics on empty input so in practice this always returns - /// `false`. - pub fn is_empty(&self) -> bool { - self.shards.is_empty() - } - - /// Resolve a request to the descriptor of the shard that owns - /// it. Routing is delegated to the embedder's `ShardRouter`; - /// `PoolGroup` only enforces that the returned index is in - /// range and panics otherwise (a routing bug is a programming - /// error, not a runtime error). - pub fn route(&self, req: &R) -> &ShardDescriptor { - let idx = self.router.route(req); - assert!( - idx < self.shards.len(), - "ShardRouter returned out-of-range index: {idx} >= {}", - self.shards.len(), - ); - &self.shards[idx] - } -} - -impl Clone for PoolGroup { - fn clone(&self) -> Self { - Self { - shards: self.shards.clone(), - router: self.router.clone(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bufferpool::types::StripeKey; - - #[derive(Clone)] - struct K(u8); - impl Req for K { - fn key(&self) -> StripeKey { - StripeKey([self.0; 32]) - } - } - - #[test] - fn route_dispatches_by_closure() { - let shards = vec![ - ShardDescriptor { - worker_idx: WorkerIdx(0), - numa: Some(0), - }, - ShardDescriptor { - worker_idx: WorkerIdx(1), - numa: Some(1), - }, - ]; - let g: PoolGroup = PoolGroup::new(shards, |k: &K| (k.0 as usize) % 2); - assert_eq!(g.len(), 2); - assert_eq!(g.route(&K(0)).worker_idx, WorkerIdx(0)); - assert_eq!(g.route(&K(1)).worker_idx, WorkerIdx(1)); - assert_eq!(g.route(&K(4)).worker_idx, WorkerIdx(0)); - } - - #[test] - #[should_panic(expected = "out-of-range index")] - fn route_panics_on_out_of_range_index() { - let shards = vec![ShardDescriptor { - worker_idx: WorkerIdx(0), - numa: None, - }]; - let g: PoolGroup = PoolGroup::new(shards, |_: &K| 7); - let _ = g.route(&K(0)); - } - - #[test] - #[should_panic(expected = "at least one shard")] - fn new_panics_on_empty() { - let _: PoolGroup = PoolGroup::new(Vec::new(), |_: &K| 0); - } - - #[test] - fn descriptor_clone_and_len() { - let shards = vec![ShardDescriptor { - worker_idx: WorkerIdx(3), - numa: None, - }]; - let g: PoolGroup = PoolGroup::new(shards, |_: &K| 0); - assert_eq!(g.len(), 1); - assert!(!g.is_empty()); - let g2 = g.clone(); - assert_eq!(g2.shards()[0].worker_idx, WorkerIdx(3)); - } -} diff --git a/cmd/unbounded-storage/src/bufferpool/mod.rs b/cmd/unbounded-storage/src/bufferpool/mod.rs index 5b5877ae9..5db3fd256 100644 --- a/cmd/unbounded-storage/src/bufferpool/mod.rs +++ b/cmd/unbounded-storage/src/bufferpool/mod.rs @@ -2,7 +2,6 @@ // Licensed under the MIT License. mod free_list; -mod group; mod inflight; mod null; mod pipeline; @@ -18,7 +17,6 @@ mod owned_future_tests; mod tests; pub use free_list::RecvQuarantineHandle; -pub use group::{PoolGroup, ShardDescriptor, ShardRouter}; pub use null::NullBlockStore; pub use pipeline::{PipelinedRead, StripePlan}; pub use pool::Pool; diff --git a/cmd/unbounded-storage/src/fanout/router.rs b/cmd/unbounded-storage/src/fanout/router.rs index 7cee22a0b..3deb31ec2 100644 --- a/cmd/unbounded-storage/src/fanout/router.rs +++ b/cmd/unbounded-storage/src/fanout/router.rs @@ -4,9 +4,8 @@ //! Stripe-to-shard ownership routing for cross-shard fan-out. //! //! [`owner_shard`] is the content-hash fallback: "which shard owns this -//! stripe key?" with no topology input. It is still used by the binary's -//! off-path [`crate::bufferpool::PoolGroup`] router closure and as the -//! spread function whenever NUMA-local routing cannot place a stripe. +//! stripe key?" with no topology input. It is the spread function whenever +//! NUMA-local routing cannot place a stripe. //! //! [`numa_local_shard`] is the NUMA-aware data-path router: it maps a //! stripe key to the drive that backs it (via [`disk_for`], the same diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 2acc947ae..df2034d64 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -16,7 +16,7 @@ use std::time::Duration; use clap::Parser; use unbounded_storage::backend::{BackendRegistry, OriginRing}; -use unbounded_storage::bufferpool::{Pool, PoolConfig, ShardDescriptor, StripeKey}; +use unbounded_storage::bufferpool::{Pool, PoolConfig}; use unbounded_storage::config::{ self, BackendSpec, Config, FrontendSpec, ResolvedFrontendBinding, frontend_spec, }; @@ -782,10 +782,7 @@ fn run_shard( // reverse declaration order; the shared `fabric` is released by the // `FabricGroup`, not here). let _ = tx.send(ShardReady::Up { - descriptor: ShardDescriptor { - worker_idx: widx, - numa: shard.numa, - }, + worker_idx: widx, publish: ShardPublish { backing_base: backing_base as usize, backing_len, @@ -1212,15 +1209,6 @@ fn reconcile_cache_disks( } } -/// Hash a `StripeKey` into a shard index. Delegates to the library -/// [`unbounded_storage::fanout::owner_shard`] so the binary's -/// `PoolGroup` router and the library frontend's fan-out path share one -/// ownership function. The modulus is the shard count; `shard_count` is -/// asserted non-zero by `PoolGroup::new`. -fn stripe_key_to_shard(key: &StripeKey, shard_count: usize) -> usize { - unbounded_storage::fanout::owner_shard(key, shard_count) -} - /// Validate and log the configured backends. /// /// [`OriginBackend`] is now the active origin tier: it is built per @@ -1477,7 +1465,7 @@ impl config::reconcile::FrontendReconcileTarget for FrontendRegistry { /// failed during bring-up. enum ShardReady { Up { - descriptor: ShardDescriptor, + worker_idx: WorkerIdx, /// Cross-shard fan-out endpoints this shard exposes: its /// registered backing region and the channel to its fetch /// service. The layer collects these from every shard and @@ -1517,8 +1505,7 @@ struct ShardPublish { /// One entry in the broadcast peer list every shard receives in phase B. /// `shard_index` is the position in the worker-index-sorted shard order, -/// matching [`stripe_key_to_shard`] and the process `PoolGroup` so -/// ownership indices are consistent across the data path. +/// which keeps ownership indices consistent across the fan-out data path. struct PeerPublish { shard_index: usize, worker_idx: WorkerIdx, diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 94d74d822..ffcc73c09 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -29,7 +29,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; -use unbounded_storage::bufferpool::{PoolGroup, Req}; use unbounded_storage::config::{ self, ApplyError, Config, ConfigApplyTarget, ConfigDiff, ShardControlGroup, }; @@ -78,10 +77,6 @@ pub struct ShardLayer { fabric_group: FabricGroup, /// Blocking fan-out/fan-in over every shard's control channel. control: ShardControlGroup, - /// Process-wide-routing dispatcher kept alive for the layer's - /// lifetime (observability today; the seam cross-shard routing will - /// use). `None` only for the degenerate zero-shard layer. - _pool_group: Option>, /// Retires just this layer's shards (layer teardown at shutdown) /// without tripping the process-wide [`crate::SHUTDOWN`]. Each shard /// ORs it into its run-loop predicate. @@ -232,17 +227,15 @@ pub fn spawn_shard_layer( // thread. Shards that come up park holding their sender, so they // never close the channel; only a panic-before-report or a clean // failure surfaces here. - let mut descriptors = Vec::new(); let mut publishes = Vec::new(); let mut errors = Vec::new(); for _ in 0..joins.len() { match ready_rx.recv() { Ok(crate::ShardReady::Up { - descriptor, + worker_idx, publish, }) => { - publishes.push((descriptor.worker_idx, publish)); - descriptors.push(descriptor); + publishes.push((worker_idx, publish)); } Ok(crate::ShardReady::Failed(err)) => { eprintln!("shard failed: {err}"); @@ -268,11 +261,10 @@ pub fn spawn_shard_layer( return Err(errors); } - // Assemble the broadcast peer set: sort the up-shards by worker index - // so `shard_index` (the position here) matches the `PoolGroup` - // ordering and `stripe_key_to_shard`, then hand every shard the full - // list. Each shard locates its own entry by worker index, registers - // the others' backings, and reports phase-B readiness below. + // Assemble the broadcast peer set in worker-index order so + // `shard_index` is stable across every shard. Each shard locates its + // own entry, registers the others' backings, and reports phase-B + // readiness below. publishes.sort_by_key(|(widx, _)| widx.0); // Retain every shard's backing Drop carrier for the layer's whole // life. Split out here as the `ShardPublish` values are consumed @@ -366,23 +358,10 @@ pub fn spawn_shard_layer( } } - descriptors.sort_by_key(|d| d.worker_idx.0); - let shard_count = descriptors.len(); - let pool_group = if descriptors.is_empty() { - None - } else { - let group: PoolGroup = PoolGroup::new(descriptors, move |req: &StripeReq| { - crate::stripe_key_to_shard(&req.key(), shard_count) - }); - eprintln!("pool group up: shards={shard_count}"); - Some(group) - }; - Ok(ShardLayer { joins, fabric_group, control: ShardControlGroup::new(control_senders), - _pool_group: pool_group, layer_stop, _backing_keepalives: backing_keepalives, }) @@ -399,7 +378,6 @@ pub fn teardown_shard_layer(layer: ShardLayer) { joins, fabric_group, control, - _pool_group, layer_stop, _backing_keepalives, } = layer; @@ -412,7 +390,6 @@ pub fn teardown_shard_layer(layer: ShardLayer) { // Drop the control senders only after every shard thread has // exited, so nothing observes a half-torn layer. drop(control); - drop(_pool_group); // Now retire the shared fabric endpoints. Each unit drops its RPC // server first (signalling shutdown and joining the RPC worker // pool), then its `Fabric`, which closes every memory region From aa0d4a28e46e4efe0ad0dbb00b7ab56632dd7241 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:32:17 +0000 Subject: [PATCH 22/56] unbounded-storage: remove unused io facade --- cmd/unbounded-storage/ARCHITECTURE.md | 2 +- .../src/fabric/completion.rs | 94 ------ cmd/unbounded-storage/src/io/mod.rs | 278 ------------------ cmd/unbounded-storage/src/lib.rs | 1 - cmd/unbounded-storage/src/ring/core.rs | 33 --- 5 files changed, 1 insertion(+), 407 deletions(-) delete mode 100644 cmd/unbounded-storage/src/io/mod.rs diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index d495b09b0..981afbb05 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -215,7 +215,7 @@ scratch MR, and tears libfabric down). ## 6. Module Map (`src/lib.rs`) All subsystems are `pub mod`: `backend`, `bufferpool`, `config`, `fabric`, -`fanout`, `frontend`, `http`, `io`, `memory`, `metrics`, `obs`, `p2p`, +`fanout`, `frontend`, `http`, `memory`, `metrics`, `obs`, `p2p`, `ring`, `runtime`, `storage`, `tls`, and `topology`. The `profiling` module is exported when the `profiling` feature is enabled. diff --git a/cmd/unbounded-storage/src/fabric/completion.rs b/cmd/unbounded-storage/src/fabric/completion.rs index ec1a1833f..3055cc51a 100644 --- a/cmd/unbounded-storage/src/fabric/completion.rs +++ b/cmd/unbounded-storage/src/fabric/completion.rs @@ -283,46 +283,6 @@ impl CompletionRegistry { } } -/// Collapse a libfabric completion outcome to the shared boundary -/// type. `Cq` errors keep their numeric detail; the remaining -/// `FabricError` variants have no numeric boundary form and render to -/// [`crate::io::IoError::Other`]. Backend-specific detail (the full -/// `CompletionInfo` with tag/src_addr) stays available to callers that -/// take the `Result` directly instead of going through -/// this collapse. -impl crate::io::CompletionOutcome for Result { - fn into_io_result(self) -> crate::io::IoResult { - match self { - Ok(info) => Ok(crate::io::Completed { bytes: info.bytes }), - Err(FabricError::Cq { prov_errno, err }) => { - Err(crate::io::IoError::Provider { prov_errno, err }) - } - Err(other) => Err(crate::io::IoError::Other(other.to_string())), - } - } -} - -/// libfabric admits via the registry's capacity check: [`allocate`] -/// fails once `live == cap`, so the policy is -/// [`crate::io::BackPressurePolicy::Capacity`]. This impl is the shared -/// introspection surface; the actual capacity check stays in -/// [`CompletionRegistry::allocate`]. -/// -/// [`allocate`]: CompletionRegistry::allocate -impl crate::io::BackPressure for CompletionRegistry { - fn capacity(&self) -> usize { - self.shared.cap - } - - fn in_flight(&self) -> usize { - self.live_count() - } - - fn policy(&self) -> crate::io::BackPressurePolicy { - crate::io::BackPressurePolicy::Capacity - } -} - #[cfg(test)] mod tests { use super::*; @@ -444,58 +404,4 @@ mod tests { _ => panic!("expected Ready(Ok)"), } } - - #[test] - fn completion_info_collapses_to_unified_result() { - use crate::io::{CompletionOutcome, IoError}; - - let ok: Result = Ok(CompletionInfo { - flags: 0, - bytes: 4096, - tag: 7, - src_addr: 9, - op_context: 0, - data: 0, - }); - match ok.into_io_result() { - Ok(c) => assert_eq!(c.bytes, 4096), - Err(e) => panic!("expected Ok, got {e}"), - } - - let cq: Result = Err(FabricError::Cq { - prov_errno: -3, - err: -5, - }); - match cq.into_io_result() { - Err(IoError::Provider { prov_errno, err }) => { - assert_eq!(prov_errno, -3); - assert_eq!(err, -5); - } - other => panic!("expected Provider, got {other:?}"), - } - - let other: Result = Err(FabricError::BadConfig("nope")); - match other.into_io_result() { - Err(IoError::Other(msg)) => assert!(msg.contains("nope")), - other => panic!("expected Other, got {other:?}"), - } - } - - #[test] - fn registry_reports_capacity_backpressure_policy() { - use crate::io::{BackPressure, BackPressurePolicy}; - - let reg = CompletionRegistry::new(2); - assert_eq!(reg.capacity(), 2); - assert_eq!(BackPressure::in_flight(&*reg), 0); - assert_eq!(reg.available(), 2); - assert!(reg.admits()); - assert_eq!(reg.policy(), BackPressurePolicy::Capacity); - - let (_s0, _f0) = reg.allocate().unwrap(); - let (_s1, _f1) = reg.allocate().unwrap(); - assert_eq!(BackPressure::in_flight(&*reg), 2); - assert_eq!(reg.available(), 0); - assert!(!reg.admits()); - } } diff --git a/cmd/unbounded-storage/src/io/mod.rs b/cmd/unbounded-storage/src/io/mod.rs deleted file mode 100644 index 9933ea2f5..000000000 --- a/cmd/unbounded-storage/src/io/mod.rs +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! Shared async-IO completion vocabulary spanning the crate's two -//! asynchronous IO backends: the io_uring engine ([`crate::ring`]) and -//! the libfabric engine ([`crate::fabric`]). -//! -//! The two backends are intentionally NOT one type. io_uring is -//! `!Send` (single-core, `Rc`/`RefCell`, op identity is a monotonic -//! `user_data` into a per-thread slot map); libfabric is `Send + Sync` -//! (`Arc`/atomics, op identity is a boxed `op_context` pointer handed -//! to the provider, completed cross-thread by a progress thread). That -//! divergence is deliberate and is preserved. -//! -//! What this module unifies is the *contract* both backends already -//! honor, expressed as a small shared vocabulary: -//! -//! * [`IoResult`] / [`Completed`] / [`IoError`] - the one result type -//! the io_uring `i32` CQE `res` and the libfabric `CompletionInfo` -//! both collapse to at the consumer boundary. -//! * [`CompletionOutcome`] - implemented by each backend's raw -//! completion value so it can collapse to [`IoResult`]. -//! * [`Unified`] - an adapter future that drives any backend -//! completion future and yields [`IoResult`]. Wrapping does not -//! change drop behavior, so each backend's "dropping the future -//! does NOT cancel the in-flight op" contract is preserved exactly. -//! * [`BackPressure`] / [`BackPressurePolicy`] - names the admission -//! policy each backend uses (io_uring parks; libfabric fails on a -//! full registry) and exposes a uniform introspection surface. -//! -//! The shared types are opt-in: backend-specific detail (raw `i32`, -//! `CompletionInfo` with its tag/src_addr, the parking `SubmitSlot`, -//! the capacity-checked `allocate`) stays available where a caller -//! needs it. This module is the common surface, not a replacement. - -use std::fmt; -use std::future::Future; -use std::pin::Pin; -use std::task::{Context, Poll}; - -/// Successful completion outcome at the unified boundary. `bytes` is -/// the transferred byte count (io_uring's non-negative CQE `res`, or -/// libfabric's `CompletionInfo::bytes`). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Completed { - pub bytes: usize, -} - -/// Unified failure at the boundary. Backend-specific richness is -/// preserved as far as it maps cleanly; anything that does not is -/// rendered into [`IoError::Other`]. -#[derive(Debug, Clone)] -pub enum IoError { - /// An OS errno. The io_uring path produces these (a negative CQE - /// `res` of `-errno` maps to `Os(errno)`). - Os(i32), - /// A provider/completion-queue error carrying the provider errno - /// alongside the libfabric error code. - Provider { prov_errno: i32, err: i32 }, - /// Any other backend error, rendered to a message. Used for the - /// libfabric error variants that have no numeric boundary form. - Other(String), -} - -impl fmt::Display for IoError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - IoError::Os(errno) => write!(f, "io error: errno={errno}"), - IoError::Provider { prov_errno, err } => { - write!(f, "io provider error: err={err} prov_errno={prov_errno}") - } - IoError::Other(msg) => write!(f, "io error: {msg}"), - } - } -} - -impl std::error::Error for IoError {} - -/// Unified result both backends collapse to at the consumer boundary. -pub type IoResult = Result; - -/// A backend's raw completion value that can collapse to the unified -/// [`IoResult`]. Implemented for the io_uring `i32` CQE `res` here, and -/// for the libfabric `Result` next to that type. -pub trait CompletionOutcome { - fn into_io_result(self) -> IoResult; -} - -impl CompletionOutcome for i32 { - /// io_uring convention: negative is `-errno`, non-negative is a - /// byte count (or other success value). - fn into_io_result(self) -> IoResult { - if self < 0 { - Err(IoError::Os(-self)) - } else { - Ok(Completed { - bytes: self as usize, - }) - } - } -} - -/// Adapter future that drives any backend completion future and -/// collapses its backend-specific output to the unified [`IoResult`]. -/// -/// Wrapping is semantically transparent: dropping a `Unified` drops -/// the inner future, so whatever drop behavior the backend defines -/// (io_uring issues a best-effort cancel and keeps the slot alive -/// until reaped; libfabric lets the result land unwatched) is -/// preserved unchanged. In particular, dropping a `Unified` does NOT -/// cancel the in-flight operation. -pub struct Unified { - inner: F, -} - -impl Unified { - pub fn new(inner: F) -> Self { - Self { inner } - } - - /// Recover the wrapped backend future. - pub fn into_inner(self) -> F { - self.inner - } -} - -impl Future for Unified -where - F: Future + Unpin, - F::Output: CompletionOutcome, -{ - type Output = IoResult; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::new(&mut self.inner) - .poll(cx) - .map(CompletionOutcome::into_io_result) - } -} - -/// How a backend admits new in-flight operations. Naming the policy -/// keeps the divergence explicit at the shared boundary; the actual -/// admission still lives in each backend (io_uring's `SubmitSlot` -/// park, libfabric's capacity-checked `allocate`). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BackPressurePolicy { - /// Submitters PARK until an in-flight op completes and frees a - /// slot. Used by the io_uring engine, whose ring depth is a hard - /// kernel bound. - Parking, - /// Admission FAILS immediately once the in-flight count reaches - /// capacity. Used by the libfabric engine, whose registry returns - /// an error the transport surfaces as back-pressure. - Capacity, -} - -/// Uniform introspection over a backend's admission control. This -/// describes the bound and the live count; it does not itself admit or -/// release (each backend owns that, to keep its counters authoritative). -pub trait BackPressure { - /// Maximum number of simultaneously in-flight operations. - fn capacity(&self) -> usize; - - /// Operations currently submitted and not yet reaped. - fn in_flight(&self) -> usize; - - /// Which admission policy this backend applies on overflow. - fn policy(&self) -> BackPressurePolicy; - - /// Remaining admission headroom. - fn available(&self) -> usize { - self.capacity().saturating_sub(self.in_flight()) - } - - /// Whether one more operation would be admitted right now without - /// parking (Parking policy) or erroring (Capacity policy). - fn admits(&self) -> bool { - self.in_flight() < self.capacity() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::runtime::noop_waker; - - #[test] - fn i32_outcome_maps_sign() { - match 4096_i32.into_io_result() { - Ok(Completed { bytes }) => assert_eq!(bytes, 4096), - Err(e) => panic!("expected Ok, got {e}"), - } - match (-libc::EIO).into_io_result() { - Err(IoError::Os(errno)) => assert_eq!(errno, libc::EIO), - other => panic!("expected Os errno, got {other:?}"), - } - match 0_i32.into_io_result() { - Ok(Completed { bytes }) => assert_eq!(bytes, 0), - Err(e) => panic!("expected Ok(0), got {e}"), - } - } - - /// A minimal backend-agnostic completion future used to exercise - /// the [`Unified`] adapter without pulling in either real engine. - struct ReadyOutcome(i32); - impl Future for ReadyOutcome { - type Output = i32; - fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { - Poll::Ready(self.0) - } - } - - #[test] - fn unified_collapses_inner_output() { - let w = noop_waker(); - let mut cx = Context::from_waker(&w); - - let mut ok = Unified::new(ReadyOutcome(512)); - match Pin::new(&mut ok).poll(&mut cx) { - Poll::Ready(Ok(Completed { bytes })) => assert_eq!(bytes, 512), - other => panic!("expected Ready Ok, got {other:?}"), - } - - let mut err = Unified::new(ReadyOutcome(-libc::ENOSPC)); - match Pin::new(&mut err).poll(&mut cx) { - Poll::Ready(Err(IoError::Os(errno))) => assert_eq!(errno, libc::ENOSPC), - other => panic!("expected Ready Os err, got {other:?}"), - } - } - - struct FakeBp { - cap: usize, - live: usize, - policy: BackPressurePolicy, - } - impl BackPressure for FakeBp { - fn capacity(&self) -> usize { - self.cap - } - fn in_flight(&self) -> usize { - self.live - } - fn policy(&self) -> BackPressurePolicy { - self.policy - } - } - - #[test] - fn backpressure_defaults_derive_from_capacity_and_live() { - let full = FakeBp { - cap: 4, - live: 4, - policy: BackPressurePolicy::Capacity, - }; - assert_eq!(full.available(), 0); - assert!(!full.admits()); - assert_eq!(full.policy(), BackPressurePolicy::Capacity); - - let room = FakeBp { - cap: 8, - live: 3, - policy: BackPressurePolicy::Parking, - }; - assert_eq!(room.available(), 5); - assert!(room.admits()); - assert_eq!(room.policy(), BackPressurePolicy::Parking); - - // `available` saturates rather than underflowing if live ever - // exceeds capacity (defensive; should not happen in practice). - let over = FakeBp { - cap: 2, - live: 5, - policy: BackPressurePolicy::Capacity, - }; - assert_eq!(over.available(), 0); - assert!(!over.admits()); - } -} diff --git a/cmd/unbounded-storage/src/lib.rs b/cmd/unbounded-storage/src/lib.rs index 31d4782f7..08c48e5c0 100644 --- a/cmd/unbounded-storage/src/lib.rs +++ b/cmd/unbounded-storage/src/lib.rs @@ -8,7 +8,6 @@ pub mod fabric; pub mod fanout; pub mod frontend; pub mod http; -pub mod io; pub mod memory; pub mod metrics; pub mod obs; diff --git a/cmd/unbounded-storage/src/ring/core.rs b/cmd/unbounded-storage/src/ring/core.rs index dfb806531..8fc3ef3fd 100644 --- a/cmd/unbounded-storage/src/ring/core.rs +++ b/cmd/unbounded-storage/src/ring/core.rs @@ -800,25 +800,6 @@ impl RingCore { } } -/// io_uring admits via the [`SubmitSlot`] park: when `submitted == -/// queue_depth` a submitter yields `Pending` until a completion frees a -/// slot, so the policy is [`BackPressurePolicy::Parking`]. This impl is -/// purely the shared introspection surface; the actual parking still -/// lives in [`SubmitSlot`]. -impl crate::io::BackPressure for RingCore { - fn capacity(&self) -> usize { - self.queue_depth as usize - } - - fn in_flight(&self) -> usize { - self.submitted.get() as usize - } - - fn policy(&self) -> crate::io::BackPressurePolicy { - crate::io::BackPressurePolicy::Parking - } -} - impl Drop for RingCore { fn drop(&mut self) { // Best-effort: release the kernel-pinned buffer/file tables @@ -1021,20 +1002,6 @@ mod tests { assert_eq!(core.resolve_buf_index(base_b, PAGE).expect("resolve b"), 1); } - /// The ring reports itself through the shared [`crate::io::BackPressure`] - /// surface as a parking backend bounded by its queue depth. - #[test] - fn backpressure_surface_reports_parking_policy() { - use crate::io::{BackPressure, BackPressurePolicy}; - - let core = RingCore::new(8, RingSetup::default()).expect("core"); - assert_eq!(core.capacity(), 8); - assert_eq!(core.in_flight(), 0); - assert_eq!(core.available(), 8); - assert!(core.admits()); - assert_eq!(core.policy(), BackPressurePolicy::Parking); - } - #[test] fn submit_slot_replaces_waiter_instead_of_accumulating() { let core = RingCore::new(1, RingSetup::default()).expect("core"); From bb8dbe5419df869b471db9d416371182c9f3a2fe Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 23:37:04 +0000 Subject: [PATCH 23/56] unbounded-storage: simplify origin connection pool --- cmd/unbounded-storage/src/backend/azure.rs | 4 +- cmd/unbounded-storage/src/backend/conn.rs | 57 ++++++---------------- cmd/unbounded-storage/src/backend/http.rs | 7 ++- cmd/unbounded-storage/src/backend/s3.rs | 4 +- cmd/unbounded-storage/src/ring/network.rs | 6 --- 5 files changed, 23 insertions(+), 55 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/azure.rs b/cmd/unbounded-storage/src/backend/azure.rs index 4a9e2e6d0..e51c6c3a3 100644 --- a/cmd/unbounded-storage/src/backend/azure.rs +++ b/cmd/unbounded-storage/src/backend/azure.rs @@ -442,7 +442,7 @@ async fn fetch( leading.len(), filled, ) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } @@ -509,7 +509,7 @@ async fn fetch_metadata( let body = ObjectMetadata::new(length).encode()?; copy_body_into_pages(&body, &dsts, backing_base, page_size)?; if head_response_reusable(version_minor, connection.as_deref(), header_end, buf.len()) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } diff --git a/cmd/unbounded-storage/src/backend/conn.rs b/cmd/unbounded-storage/src/backend/conn.rs index 272d774fd..701d8d468 100644 --- a/cmd/unbounded-storage/src/backend/conn.rs +++ b/cmd/unbounded-storage/src/backend/conn.rs @@ -3,11 +3,9 @@ //! Reusable origin TCP connections for HTTP-family backends. -use std::collections::HashMap; +use std::cell::RefCell; use std::os::fd::RawFd; use std::rc::Rc; -use std::sync::{Arc, Mutex}; -use std::thread::ThreadId; use crate::bufferpool::Error; use crate::http::{ResponseHead, StatusCode, response_keep_alive}; @@ -16,64 +14,41 @@ use crate::tls::TlsContext; #[derive(Clone)] pub(super) struct OriginConnPool { - inner: Arc>, + inner: Rc>, } struct PoolInner { - max_idle_per_ring: usize, - idle: HashMap>, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -struct PoolKey { - thread: ThreadId, - ring: usize, + max_idle: usize, + idle: Vec, } impl OriginConnPool { - pub(super) fn new(max_idle_per_ring: usize) -> Self { + pub(super) fn new(max_idle: usize) -> Self { Self { - inner: Arc::new(Mutex::new(PoolInner { - max_idle_per_ring: max_idle_per_ring.max(1), - idle: HashMap::new(), + inner: Rc::new(RefCell::new(PoolInner { + max_idle: max_idle.max(1), + idle: Vec::new(), })), } } - pub(super) fn checkout(&self, handle: &NetHandle) -> Option { - let key = pool_key(handle); + pub(super) fn checkout(&self) -> Option { loop { - let conn = self - .inner - .lock() - .expect("origin connection pool poisoned") - .idle - .get_mut(&key) - .and_then(Vec::pop)?; + let conn = self.inner.borrow_mut().idle.pop()?; if tcp_conn_is_reusable(conn.fd()) { return Some(conn); } } } - pub(super) fn put(&self, handle: &NetHandle, conn: TcpConn) { - let key = pool_key(handle); - let mut inner = self.inner.lock().expect("origin connection pool poisoned"); - let max_idle = inner.max_idle_per_ring; - let idle = inner.idle.entry(key).or_default(); - if idle.len() < max_idle { - idle.push(conn); + pub(super) fn put(&self, conn: TcpConn) { + let mut inner = self.inner.borrow_mut(); + if inner.idle.len() < inner.max_idle { + inner.idle.push(conn); } } } -fn pool_key(handle: &NetHandle) -> PoolKey { - PoolKey { - thread: std::thread::current().id(), - ring: handle.ring_key(), - } -} - pub(super) struct TcpConn { fd: RawFd, tls: bool, @@ -129,7 +104,7 @@ async fn checkout_or_connect( tls: &Option>, sni_host: &str, ) -> Result { - if let Some(conn) = pool.checkout(handle) { + if let Some(conn) = pool.checkout() { return Ok(OriginConn { conn, reused: true }); } @@ -165,7 +140,7 @@ pub(super) async fn send_request_read_head( match handle.send(fd, request.clone()).await { Ok(_) => {} - Err(e) if reused && stale_retry => { + Err(_) if reused && stale_retry => { stale_retry = false; continue; } diff --git a/cmd/unbounded-storage/src/backend/http.rs b/cmd/unbounded-storage/src/backend/http.rs index 4c96e18fa..136168b17 100644 --- a/cmd/unbounded-storage/src/backend/http.rs +++ b/cmd/unbounded-storage/src/backend/http.rs @@ -12,8 +12,7 @@ //! //! Origin connections are kept alive when the response has a known, //! fully-consumed body and the peer did not ask to close the stream. Idle -//! sockets stay ring-local so the shard ring and worker-local RPC rings -//! never share fds. +//! sockets remain in the backend's shard-local pool. //! //! ## Address resolution and the `Host` header //! @@ -499,7 +498,7 @@ async fn fetch( leading.len(), filled, ) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } @@ -571,7 +570,7 @@ async fn fetch_metadata( let body = ObjectMetadata::new(length).encode()?; copy_body_into_pages(&body, &dsts, backing_base, page_size)?; if head_response_reusable(version_minor, connection.as_deref(), header_end, buf.len()) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } diff --git a/cmd/unbounded-storage/src/backend/s3.rs b/cmd/unbounded-storage/src/backend/s3.rs index 6ae4017b8..df1987d55 100644 --- a/cmd/unbounded-storage/src/backend/s3.rs +++ b/cmd/unbounded-storage/src/backend/s3.rs @@ -460,7 +460,7 @@ async fn fetch( leading.len(), filled, ) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } @@ -527,7 +527,7 @@ async fn fetch_metadata( let body = ObjectMetadata::new(length).encode()?; copy_body_into_pages(&body, &dsts, backing_base, page_size)?; if head_response_reusable(version_minor, connection.as_deref(), header_end, buf.len()) { - conns.put(&handle, conn); + conns.put(conn); } Ok(()) } diff --git a/cmd/unbounded-storage/src/ring/network.rs b/cmd/unbounded-storage/src/ring/network.rs index a5375a789..0fd84e456 100644 --- a/cmd/unbounded-storage/src/ring/network.rs +++ b/cmd/unbounded-storage/src/ring/network.rs @@ -487,12 +487,6 @@ impl NetHandle { Rc::as_ptr(&self.ring) as usize } - /// Address identity of the underlying ring allocation for components - /// that must keep resources ring-local across reused handles. - pub(crate) fn ring_key(&self) -> usize { - Rc::as_ptr(&self.ring) as usize - } - /// `'static` counterpart of [`NetworkRing::accept`]. pub fn accept(&self, listen_fd: RawFd) -> impl Future> + 'static { let ring = Rc::clone(&self.ring); From 9efdb4c80daafde1faf5aad34c7018dfddc9fc93 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 20:12:51 +0000 Subject: [PATCH 24/56] unbounded-storage: remove multi-registry disk layer --- cmd/unbounded-storage/ARCHITECTURE.md | 8 +- cmd/unbounded-storage/src/main.rs | 12 +- cmd/unbounded-storage/src/shard_layer.rs | 8 +- .../src/storage/disks/mod.rs | 159 +----------------- 4 files changed, 17 insertions(+), 170 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index b7519e97d..499d7a491 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -519,10 +519,10 @@ bufferpool `Backing`. **Disk lifecycle** (`disks/`): `trait DiskTarget { open(spec, pin) -> (Handle, PageChannel) }`, with `UringDiskTarget` in production (runs the disk on -a pinned storage core) and a mock for tests. `DiskRegistrySet` is seeded with -the plan's disjoint NVMe `DiskCpuSlot`s, keeps those slots globally disjoint -across per-cache registries, and `reconcile(desired)` closes missing paths, opens -new ones, and treats any spec drift (kind/numa/size/queue_depth/page_size/ +a pinned storage core) and a mock for tests. `DiskRegistry` is seeded with +the plan's disjoint NVMe `DiskCpuSlot`s. `reconcile(desired)` closes missing +paths, opens new ones, and treats any spec drift +(kind/numa/size/queue_depth/page_size/ skip_recovery_scan) as a remove + add. `assign_disk_cpus` keeps survivors on their physical pin (preserving the disjoint-CPU invariant and idempotence under churn) and assigns new disks diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 18b0ee4c1..0d7b8eca0 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -36,7 +36,7 @@ use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; use unbounded_storage::storage::StripeReq; use unbounded_storage::storage::disks::{ - CacheDirectorySet, ChainLocalStore, DiskRegistrySet, UringDiskTarget, + CacheDirectorySet, ChainLocalStore, DiskRegistry, UringDiskTarget, }; use unbounded_storage::topology::{CorePlan, CorePlanConfig, DiskCpuSlot, Host, ServingShard}; @@ -49,7 +49,6 @@ mod shard_layer; const DEFAULT_CONFIG_PATH: &str = "/etc/unbounded-storage/config.toml"; const SHUTDOWN_POLL: Duration = Duration::from_millis(100); -const SHARED_DISK_REGISTRY_ID: &str = "shared"; /// Stripe granularity used to build an inert origin backend on shards /// that have no configured backend. Such a backend is never exercised @@ -408,7 +407,7 @@ fn main() -> ExitCode { numa: core.numa, }) .collect(); - let mut disk_registry = DiskRegistrySet::new(UringDiskTarget::new(runtime.clone()), disk_slots); + let mut disk_registry = DiskRegistry::new(UringDiskTarget::new(runtime.clone()), disk_slots); // Bring up the initial shard layer. A bring-up failure is fatal: // there is no running process to reconcile into. @@ -1202,17 +1201,16 @@ fn p2p_topology_weighting(weighting: &config::TopologyWeighting) -> TopologyWeig } fn reconcile_cache_disks( - disk_registry: &mut DiskRegistrySet, + disk_registry: &mut DiskRegistry, cache_directories: &CacheDirectorySet, projection: &config::RuntimeGraph, ) { let mut cache_ids: Vec = projection.caches.keys().cloned().collect(); cache_ids.sort(); - disk_registry.reconcile_ids([SHARED_DISK_REGISTRY_ID.to_string()]); cache_directories.reconcile(cache_ids.iter().cloned()); let disks = config::runtime_disks(projection); - let report = disk_registry.reconcile_cache(SHARED_DISK_REGISTRY_ID, &disks); + let report = disk_registry.reconcile(&disks); eprintln!( "config: shared disks: added={} removed={} failures={}", report.added, @@ -1223,7 +1221,7 @@ fn reconcile_cache_disks( eprintln!("disk {}: open failed: {msg}", path.display()); } - let channels = disk_registry.channels_snapshot(SHARED_DISK_REGISTRY_ID); + let channels = disk_registry.channels_snapshot(); for cache_id in cache_ids { cache_directories.apply_channels(&cache_id, channels.clone()); } diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index cc32f091c..94d74d822 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -37,7 +37,7 @@ use unbounded_storage::fabric::PeerId; use unbounded_storage::p2p::RouteTableHandle; use unbounded_storage::runtime::{JoinHandle, Threading, WorkerIdx}; use unbounded_storage::storage::StripeReq; -use unbounded_storage::storage::disks::{CacheDirectorySet, DiskRegistrySet, UringDiskTarget}; +use unbounded_storage::storage::disks::{CacheDirectorySet, DiskRegistry, UringDiskTarget}; use unbounded_storage::topology::ServingShard; use crate::StartupSettings; @@ -435,14 +435,14 @@ pub struct ProcessApplyTarget { /// `Option` so the layer can be moved out at shutdown via /// [`Self::into_parts`]. Always `Some` while the process is serving. layer: Option, - disk_registry: DiskRegistrySet, + disk_registry: DiskRegistry, cache_directories: Arc, } impl ProcessApplyTarget { pub fn new( layer: ShardLayer, - disk_registry: DiskRegistrySet, + disk_registry: DiskRegistry, cache_directories: Arc, ) -> Self { Self { @@ -462,7 +462,7 @@ impl ProcessApplyTarget { /// Consume the target at shutdown, returning the live layer (if any) /// and the disk registry so the caller can tear them down in the /// correct order (shards first, then disks). - pub fn into_parts(self) -> (Option, DiskRegistrySet) { + pub fn into_parts(self) -> (Option, DiskRegistry) { (self.layer, self.disk_registry) } } diff --git a/cmd/unbounded-storage/src/storage/disks/mod.rs b/cmd/unbounded-storage/src/storage/disks/mod.rs index fe82fa120..cb5d7374b 100644 --- a/cmd/unbounded-storage/src/storage/disks/mod.rs +++ b/cmd/unbounded-storage/src/storage/disks/mod.rs @@ -9,15 +9,13 @@ //! [`UringDiskTarget`]; tests plug in a mock target. //! //! CPU placement for disk storage cores comes from the topology -//! [`CorePlan`](crate::topology::CorePlan): the registry set is seeded +//! [`CorePlan`](crate::topology::CorePlan): the registry is seeded //! with one [`DiskCpuSlot`](crate::topology::DiskCpuSlot) per //! [`StorageCore`](crate::topology::StorageCore) (one per NVMe drive), //! which are disjoint from the NIC-worker and serving-shard CPUs by //! construction. On each reconcile a deterministic assignment maps //! disks to slots, preferring NUMA-local slots; disks beyond the -//! available slot count run unpinned. Per-cache registries reserve slots -//! already held by other caches so the storage-core pins stay globally -//! disjoint. +//! available slot count run unpinned. //! //! Each successful open returns both a per-disk handle (whose `Drop` //! tears down the disk thread) and a [`PageChannel`] published into @@ -25,9 +23,8 @@ //! operations to the storage core that owns the engine and ring; see //! the sub-module docs for the device-side details. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::Arc; use crate::config::schema::DiskSpec; use crate::storage::PageChannel; @@ -67,18 +64,6 @@ pub trait DiskTarget: Send + Sync + 'static { ) -> Result<(Self::Handle, PageChannel), DiskError>; } -impl DiskTarget for Arc { - type Handle = T::Handle; - - fn open( - &self, - spec: &DiskSpec, - pin: Option, - ) -> Result<(Self::Handle, PageChannel), DiskError> { - (**self).open(spec, pin) - } -} - /// Reasons a disk open can fail. Kept simple by design: every variant /// carries a human-readable string so callers can log without pulling /// in an error crate. @@ -133,12 +118,6 @@ impl OpenDiskEntry { } } -pub struct DiskRegistrySet { - target: Arc, - disk_slots: Vec, - registries: HashMap>>, -} - impl DiskRegistry { /// Build an empty registry. `disk_slots` are the topology plan's /// disjoint NVMe CPU slots; the registry assigns them to disks @@ -163,14 +142,6 @@ impl DiskRegistry { /// Partial failures during opens are reported but do not abort the /// reconcile. pub fn reconcile(&mut self, desired: &[DiskSpec]) -> DiskReport { - self.reconcile_with_reserved(desired, &[]) - } - - fn reconcile_with_reserved( - &mut self, - desired: &[DiskSpec], - reserved_slots: &[DiskCpuSlot], - ) -> DiskReport { let mut report = DiskReport::default(); let mut desired_paths: HashMap<&Path, &DiskSpec> = HashMap::new(); @@ -207,11 +178,7 @@ impl DiskRegistry { .iter() .map(|(path, entry)| (path.clone(), entry.placement)) .collect(); - let assignment = if reserved_slots.is_empty() { - assign_disk_cpus(desired, &self.disk_slots, &open_placements) - } else { - assign_disk_cpus_reserved(desired, &self.disk_slots, &open_placements, reserved_slots) - }; + let assignment = assign_disk_cpus(desired, &self.disk_slots, &open_placements); for spec in desired { let path = disk_path(spec); @@ -297,73 +264,6 @@ impl DiskRegistry { } } -impl DiskRegistrySet { - pub fn new(target: T, disk_slots: Vec) -> Self { - Self { - target: Arc::new(target), - disk_slots, - registries: HashMap::new(), - } - } - - pub fn reconcile_cache(&mut self, cache_id: &str, disks: &[DiskSpec]) -> DiskReport { - let reserved_slots = self.reserved_slots_except(cache_id); - - self.registries - .entry(cache_id.to_string()) - .or_insert_with(|| DiskRegistry::new(self.target.clone(), self.disk_slots.clone())) - .reconcile_with_reserved(disks, &reserved_slots) - } - - pub fn channels_snapshot( - &self, - cache_id: &str, - ) -> Vec<(PathBuf, PageChannel, Option, bool)> { - self.registries - .get(cache_id) - .map(DiskRegistry::channels_snapshot) - .unwrap_or_default() - } - - pub fn reconcile_ids(&mut self, cache_ids: I) - where - I: IntoIterator, - I::Item: Into, - { - let desired: HashSet = cache_ids.into_iter().map(Into::into).collect(); - let removed: Vec = self - .registries - .keys() - .filter(|id| !desired.contains(*id)) - .cloned() - .collect(); - for id in removed { - if let Some(registry) = self.registries.remove(&id) { - registry.drain(); - } - } - } - - pub fn drain(mut self) { - for (_, registry) in self.registries.drain() { - registry.drain(); - } - } - - fn reserved_slots_except(&self, cache_id: &str) -> Vec { - self.registries - .iter() - .filter(|(id, _)| id.as_str() != cache_id) - .flat_map(|(_, registry)| { - registry - .entries - .values() - .filter_map(|entry| entry.placement) - }) - .collect() - } -} - /// Deterministically assign disk CPU slots to the `desired` disks. /// /// `open` holds the slot each currently-open (surviving) disk already @@ -380,15 +280,6 @@ fn assign_disk_cpus( desired: &[DiskSpec], slots: &[DiskCpuSlot], open: &HashMap>, -) -> HashMap> { - assign_disk_cpus_reserved(desired, slots, open, &[]) -} - -fn assign_disk_cpus_reserved( - desired: &[DiskSpec], - slots: &[DiskCpuSlot], - open: &HashMap>, - reserved_slots: &[DiskCpuSlot], ) -> HashMap> { let mut sorted: Vec<&DiskSpec> = desired.iter().collect(); sorted.sort_by(|a, b| disk_path(a).cmp(disk_path(b))); @@ -396,12 +287,6 @@ fn assign_disk_cpus_reserved( let mut used = vec![false; slots.len()]; let mut out: HashMap> = HashMap::new(); - for reserved in reserved_slots { - if let Some(i) = slots.iter().position(|s| s == reserved) { - used[i] = true; - } - } - // Lock in the slots held by surviving open disks before assigning // anything new. Their pin is physically fixed for the lifetime of // the handle, so the fresh assignment must treat those slots as @@ -746,42 +631,6 @@ mod tests { assert_ne!(got[0].1, got[1].1); } - #[test] - fn registry_set_reserves_slots_across_caches() { - let slots = vec![slot(4, None), slot(5, None)]; - let hints = Arc::new(Mutex::new(Vec::new())); - let mut set = DiskRegistrySet::new( - CpuRecorder { - hints: hints.clone(), - }, - slots, - ); - - set.reconcile_ids(["cache-a", "cache-b"]); - - let report_a = set.reconcile_cache("cache-a", &[spec("/cache-a-disk", None)]); - assert_eq!(report_a.added, 1); - assert_eq!(report_a.removed, 0); - assert!(report_a.failures.is_empty()); - - let report_b = set.reconcile_cache("cache-b", &[spec("/cache-b-disk", None)]); - assert_eq!(report_b.added, 1); - assert_eq!(report_b.removed, 0); - assert!(report_b.failures.is_empty()); - - let got = hints.lock().unwrap().clone(); - assert_eq!(got.len(), 2); - assert!(got.iter().all(|(_, cpu)| cpu.is_some())); - assert_ne!(got[0].1, got[1].1, "caches shared a disk CPU slot"); - - let before = set.registries["cache-a"].placement_snapshot(); - let report_a = set.reconcile_cache("cache-a", &[spec("/cache-a-disk", None)]); - assert_eq!(report_a.added, 0); - assert_eq!(report_a.removed, 0); - assert_eq!(set.registries["cache-a"].placement_snapshot(), before); - assert_eq!(hints.lock().unwrap().len(), 2); - } - #[test] fn disks_beyond_slot_count_get_none() { // One slot, three disks: the first (path-sorted) disk claims From 175cd8f970dc7c608c5b2e3a3e957f9b45500b4d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 06:17:18 +0000 Subject: [PATCH 25/56] unbounded-storage: remove legacy p2p request types --- cmd/unbounded-storage/src/bufferpool/mod.rs | 2 +- cmd/unbounded-storage/src/bufferpool/types.rs | 4 - cmd/unbounded-storage/src/p2p/mod.rs | 2 +- cmd/unbounded-storage/src/p2p/types.rs | 112 +----------------- 4 files changed, 4 insertions(+), 116 deletions(-) diff --git a/cmd/unbounded-storage/src/bufferpool/mod.rs b/cmd/unbounded-storage/src/bufferpool/mod.rs index 5db3fd256..3f004d4b5 100644 --- a/cmd/unbounded-storage/src/bufferpool/mod.rs +++ b/cmd/unbounded-storage/src/bufferpool/mod.rs @@ -22,5 +22,5 @@ pub use pipeline::{PipelinedRead, StripePlan}; pub use pool::Pool; pub use stream::{OwnedPageFuture, PageGuard, ReadStream}; pub use traits::{BlockStore, BufferPool, PageCachePolicy, PageStream, Req, Transport}; -pub use types::{BulkRef, Error, PageRef, PoolConfig, StripeKey, TraceCtx}; +pub use types::{BulkRef, Error, PageRef, PoolConfig, StripeKey}; pub use window::WindowedRead; diff --git a/cmd/unbounded-storage/src/bufferpool/types.rs b/cmd/unbounded-storage/src/bufferpool/types.rs index 6f0437ee8..f3fdfe36c 100644 --- a/cmd/unbounded-storage/src/bufferpool/types.rs +++ b/cmd/unbounded-storage/src/bufferpool/types.rs @@ -32,10 +32,6 @@ pub struct BulkRef { pub len: u32, } -/// Opaque tracing handle. The pool does not inspect it. -#[derive(Clone, Debug, Default)] -pub struct TraceCtx; - /// Pool tunables. Most of the design's TODO knobs are still TBD; /// see `designs/bufferpool.md` "TODO(config)". #[derive(Clone, Debug)] diff --git a/cmd/unbounded-storage/src/p2p/mod.rs b/cmd/unbounded-storage/src/p2p/mod.rs index d2098a82a..44756b5d8 100644 --- a/cmd/unbounded-storage/src/p2p/mod.rs +++ b/cmd/unbounded-storage/src/p2p/mod.rs @@ -32,4 +32,4 @@ pub use handler::{ pub use ring::{node_id_from_name, node_to_ring, splitmix64, stripe_to_ring}; pub use routing_handle::{RouteTableHandle, RouteTableSnapshot}; pub use transport::{RoutedStream, RoutedTransport}; -pub use types::{NodeId, P2pReq, PeerEntry, RingId, TopologyTags}; +pub use types::{NodeId, PeerEntry, RingId, TopologyTags}; diff --git a/cmd/unbounded-storage/src/p2p/types.rs b/cmd/unbounded-storage/src/p2p/types.rs index 754cd4756..138d34b05 100644 --- a/cmd/unbounded-storage/src/p2p/types.rs +++ b/cmd/unbounded-storage/src/p2p/types.rs @@ -2,13 +2,9 @@ // Licensed under the MIT License. //! Shared value types for the stripe DHT: ring positions, topology -//! tags, peer entries, and the recursive-routing request. +//! tags, and peer entries. -use std::time::Instant; - -use crate::bufferpool::{Req, StripeKey, TraceCtx}; -use crate::p2p::ring::stripe_to_ring; -use crate::storage::OriginRef; +use crate::bufferpool::StripeKey; /// Opaque node identifier minted by the p2p layer. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] @@ -50,113 +46,9 @@ pub struct PeerEntry { pub tags: TopologyTags, } -/// Recursive-routing request. Unlike the old source-routed -/// placeholder this carries no path; each forwarder computes its -/// own next hop from its local finger table. -#[derive(Clone, Debug)] -pub struct P2pReq { - pub key: StripeKey, - /// Cached `stripe_to_ring(key)` so each forwarder can pick a - /// next hop without rehashing the 32-byte key. The caller is - /// responsible for keeping this consistent with `key`; tests - /// may pass a different mapping for coverage. - pub target: RingId, - pub deadline: Instant, - pub trace: TraceCtx, - pub hops: u32, - /// Origin context for the stripe, populated by a frontend that - /// knows which origin object the stripe came from. Peer - /// handlers ignore it (they serve from their local cache by - /// `key`); it is consulted only when the request falls through - /// to the origin tier and the backend must map `key` back to an - /// origin byte range. `None` for peer-internal forwarding where - /// no origin is known or needed. - pub origin: Option, -} - -impl P2pReq { - pub fn new(key: StripeKey, deadline: Instant, trace: TraceCtx) -> Self { - Self { - key, - target: stripe_to_ring(key), - deadline, - trace, - hops: 0, - origin: None, - } - } - - pub fn new_with_target( - key: StripeKey, - target: RingId, - deadline: Instant, - trace: TraceCtx, - ) -> Self { - Self { - key, - target, - deadline, - trace, - hops: 0, - origin: None, - } - } - - /// Attach origin context, consumed builder-style. Backward - /// compatible: callers that do not know the origin simply skip - /// this and leave `origin` as `None`. - pub fn with_origin(mut self, origin: OriginRef) -> Self { - self.origin = Some(origin); - self - } - - /// Borrow the origin context, if any. - pub fn origin(&self) -> Option<&OriginRef> { - self.origin.as_ref() - } -} - -impl Req for P2pReq { - fn key(&self) -> StripeKey { - self.key - } -} - #[cfg(test)] mod tests { use super::*; - use std::time::Duration; - - #[test] - fn new_populates_fields_with_zero_hops() { - let key = StripeKey([7u8; 32]); - let deadline = Instant::now() + Duration::from_secs(1); - let req = P2pReq::new(key, deadline, TraceCtx::default()); - assert_eq!(req.key, key); - assert_eq!(req.target, stripe_to_ring(key)); - assert_eq!(req.deadline, deadline); - assert_eq!(req.hops, 0); - } - - #[test] - fn req_trait_returns_key() { - let key = StripeKey([3u8; 32]); - let req = P2pReq::new_with_target(key, RingId(0), Instant::now(), TraceCtx::default()); - assert_eq!(::key(&req), key); - } - - #[test] - fn origin_defaults_none_and_round_trips_via_builder() { - let origin = OriginRef::new("primary-s3", "models/llama.bin", 5); - let key = origin.stripe_key(); - let req = P2pReq::new(key, Instant::now(), TraceCtx::default()); - assert_eq!(req.origin(), None); - - let req = req.with_origin(origin.clone()); - assert_eq!(req.origin(), Some(&origin)); - // The key contract is untouched by attaching origin. - assert_eq!(::key(&req), key); - } #[test] fn ring_id_orders_by_raw_u64() { From bcb618ec8302b48ab74e5c4ba99d085153a539bf Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 22:39:14 +0000 Subject: [PATCH 26/56] unbounded-storage: remove unused shard context --- cmd/unbounded-storage/src/runtime/context.rs | 75 -------------------- cmd/unbounded-storage/src/runtime/mod.rs | 6 -- 2 files changed, 81 deletions(-) delete mode 100644 cmd/unbounded-storage/src/runtime/context.rs diff --git a/cmd/unbounded-storage/src/runtime/context.rs b/cmd/unbounded-storage/src/runtime/context.rs deleted file mode 100644 index 2129a0bcd..000000000 --- a/cmd/unbounded-storage/src/runtime/context.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! Per-shard resource bundle handed to frontend/backend builders. -//! -//! [`ShardContext`] carries the cross-cutting, shard-local resources a -//! frontend or backend future needs that are *not* the bufferpool -//! `Pool` itself: the page geometry and the per-shard -//! [`NetworkRing`](crate::ring::NetworkRing). -//! -//! Builders receive `&ShardContext` together with an `Rc` (handed -//! separately so the context need not be generic over the pool's type -//! parameters) and a `&mut ShardLoop` on which they register their -//! `pool.read` / connection futures. The shard bring-up registers the -//! socket ring's [`progress`](crate::ring::NetworkRing::progress) -//! as a [`ShardLoop`](crate::runtime::ShardLoop) tick hook, so a -//! builder only has to enqueue its futures; the loop drives the ring. -//! -//! The whole struct is Linux-gated because it is meaningless without -//! the socket ring, which is itself Linux-only (io_uring). - -use std::cell::RefCell; -use std::rc::Rc; - -use crate::ring::NetworkRing; - -/// Shard-local resources shared by frontend and backend futures. See -/// the module docs for how builders consume it. -pub struct ShardContext { - page_size: usize, - socket: Rc>, -} - -impl ShardContext { - /// Build a context over the shard's page geometry and socket ring. - pub fn new(page_size: usize, socket: Rc>) -> Self { - Self { page_size, socket } - } - - /// Bufferpool page size in bytes for this shard. - pub fn page_size(&self) -> usize { - self.page_size - } - - /// The per-shard io_uring socket ring. Backend/frontend futures - /// borrow it to issue connect / accept / recv / send ops; the shard - /// loop drives its `progress()` via a tick hook. - pub fn socket(&self) -> &Rc> { - &self.socket - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn accessors_return_constructed_values() { - // Ring construction needs a kernel io_uring; if it is - // unavailable in this environment, skip gracefully rather than - // hard-failing the suite. - let socket = match NetworkRing::new(8) { - Ok(s) => Rc::new(RefCell::new(s)), - Err(e) => { - eprintln!("accessors_return_constructed_values: ring unavailable: {e}; skipping"); - return; - } - }; - let ctx = ShardContext::new(4096, socket.clone()); - assert_eq!(ctx.page_size(), 4096); - // The accessor hands back the same Rc the context was built - // with (same allocation). - assert!(Rc::ptr_eq(ctx.socket(), &socket)); - } -} diff --git a/cmd/unbounded-storage/src/runtime/mod.rs b/cmd/unbounded-storage/src/runtime/mod.rs index 80830dd4e..94e193495 100644 --- a/cmd/unbounded-storage/src/runtime/mod.rs +++ b/cmd/unbounded-storage/src/runtime/mod.rs @@ -9,15 +9,9 @@ mod pinned; mod executor; mod shard; -#[cfg(target_os = "linux")] -mod context; - #[cfg(target_os = "linux")] pub use pinned::{PinnedRuntime, WorkerSpec, set_preferred_node}; -#[cfg(target_os = "linux")] -pub use context::ShardContext; - pub use executor::{block_on_cooperative, noop_waker, park_block_on_until, thread_waker}; pub use shard::ShardLoop; From c2d739bc31fbb99fc434e03f214e585eb6662674 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 06:18:08 +0000 Subject: [PATCH 27/56] unbounded-storage: remove borrowed origin dispatch --- cmd/unbounded-storage/src/backend/origin.rs | 41 --------------------- 1 file changed, 41 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/origin.rs b/cmd/unbounded-storage/src/backend/origin.rs index 31a2627a1..f971e24f9 100644 --- a/cmd/unbounded-storage/src/backend/origin.rs +++ b/cmd/unbounded-storage/src/backend/origin.rs @@ -64,25 +64,6 @@ impl OriginBackend { } } -impl Backend for OriginBackend { - type Req = StripeReq; - type Stream<'a> = OriginBorrowedStream<'a>; - - fn bulk_get<'a>( - &'a self, - req: &'a Self::Req, - src: BulkRef, - dsts: &'a [PageRef], - ) -> Self::Stream<'a> { - match self { - OriginBackend::Http(b) => OriginBorrowedStream::Http(b.bulk_get(req, src, dsts)), - OriginBackend::S3(b) => OriginBorrowedStream::S3(b.bulk_get(req, src, dsts)), - OriginBackend::Azure(b) => OriginBorrowedStream::Azure(b.bulk_get(req, src, dsts)), - OriginBackend::Fake(b) => OriginBorrowedStream::Fake(b.bulk_get(req, src, dsts)), - } - } -} - /// Stream produced by [`OriginBackend::fetch_stream`] for registry use. pub enum OriginStream<'a> { Http(::Stream<'static>), @@ -142,25 +123,3 @@ impl PageStream for OriginStream<'_> { } } } - -/// Stream produced by direct [`OriginBackend::bulk_get`] calls. -pub enum OriginBorrowedStream<'a> { - Http(::Stream<'a>), - S3(::Stream<'a>), - Azure(::Stream<'a>), - Fake(::Stream<'a>), -} - -impl PageStream for OriginBorrowedStream<'_> { - fn poll_next( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - match self.get_mut() { - OriginBorrowedStream::Http(s) => Pin::new(s).poll_next(cx), - OriginBorrowedStream::S3(s) => Pin::new(s).poll_next(cx), - OriginBorrowedStream::Azure(s) => Pin::new(s).poll_next(cx), - OriginBorrowedStream::Fake(s) => Pin::new(s).poll_next(cx), - } - } -} From f933a0cd7d452fbebd287431703e87779a6c8813 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 9 Jul 2026 23:44:31 +0000 Subject: [PATCH 28/56] unbounded-storage: release completion capacity before handlers --- .../src/fabric/completion.rs | 79 +++++++++++++++---- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/cmd/unbounded-storage/src/fabric/completion.rs b/cmd/unbounded-storage/src/fabric/completion.rs index 3055cc51a..b082748ae 100644 --- a/cmd/unbounded-storage/src/fabric/completion.rs +++ b/cmd/unbounded-storage/src/fabric/completion.rs @@ -7,9 +7,8 @@ //! `Box::into_raw`s the slot and hands the resulting `*mut c_void` to //! libfabric as the operation's `op_context`. On the progress thread, //! `fi_cq_read` returns the same pointer back inside a -//! `fi_cq_data_entry`; we `Box::from_raw` it, drop the box (releasing -//! libfabric's reference) after calling `complete`, and the awaiting -//! future is woken. +//! `fi_cq_data_entry`; we `Box::from_raw` it, release its registry +//! reservation, run its completion handler, and wake the awaiting future. //! //! The slot's shared state lives in an `Arc` cloned into a //! `CompletionFuture`, so completion and future-drop can race in @@ -25,7 +24,7 @@ use std::future::Future; use std::pin::Pin; -use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; @@ -118,14 +117,15 @@ impl SlotInner { /// dropping the reclaimed box releases libfabric's reference. pub struct CompletionSlot { inner: Arc, - /// Back-reference so the registry's live count drops when this - /// box is reclaimed by the progress thread (or by `cancel_raw` - /// on a synchronous submission failure). + /// Back-reference so the registry reservation is released before a + /// completion handler runs, or when the box is dropped after a + /// synchronous submission failure. registry: Arc, /// Optional handler invoked when the slot completes, before the /// future is woken. Used by the ping responder to re-post its /// recv and emit the pong without involving any future at all. handler: Mutex) + Send>>>, + reservation_live: AtomicBool, } impl CompletionSlot { @@ -148,10 +148,14 @@ impl CompletionSlot { unsafe { Box::from_raw(ptr as *mut CompletionSlot) } } - /// Store a result and wake the awaiting future. If a handler was - /// installed via [`CompletionSlot::set_handler`] it is invoked - /// first; the handler may not panic. + /// Release registry capacity, store a result, and wake the awaiting + /// future. If a handler was installed via + /// [`CompletionSlot::set_handler`] it is invoked after capacity is + /// released and before the future is woken; the handler may not panic. pub fn complete(&self, result: Result) { + // Completion handlers may allocate a replacement slot, notably + // when a receive pool rearms at full registry capacity. + self.release_reservation(); let handler = self.handler.lock().ok().and_then(|mut h| h.take()); if let Some(h) = handler { h(&result); @@ -173,14 +177,20 @@ impl CompletionSlot { *h = Some(Box::new(f)); } } + + fn release_reservation(&self) { + if self.reservation_live.swap(false, Ordering::AcqRel) { + self.registry.live.fetch_sub(1, Ordering::AcqRel); + } + } } impl Drop for CompletionSlot { fn drop(&mut self) { - // Decrement live-slot count exactly once, on the same box - // libfabric handed back. The `CompletionFuture` keeps the - // inner `SlotInner` Arc alive across this drop if needed. - self.registry.live.fetch_sub(1, Ordering::AcqRel); + // Synchronous submission failures drop without completing. A + // completed slot already released its reservation before running + // its handler, so this is idempotent. + self.release_reservation(); } } @@ -212,9 +222,9 @@ pub(crate) struct RegistryShared { } /// Bounded back-pressure on in-flight ops. Capacity is checked at -/// allocate time; the count drops when the boxed slot is freed (by -/// the progress thread reclaiming it, or by `CompletionSlot::cancel` -/// on a synchronous submission failure). +/// allocate time; the count drops when completion begins, before its +/// handler runs, or when an unsubmitted slot is dropped after a +/// synchronous submission failure. pub struct CompletionRegistry { shared: Arc, } @@ -258,6 +268,7 @@ impl CompletionRegistry { inner: inner.clone(), registry: self.shared.clone(), handler: Mutex::new(None), + reservation_live: AtomicBool::new(true), }); let fut = CompletionFuture { inner }; Ok((slot, fut)) @@ -380,6 +391,40 @@ mod tests { assert_eq!(reg.peak_inflight(), 2); } + #[test] + fn completion_handler_can_replace_slot_at_full_capacity() { + let reg = CompletionRegistry::new(1); + let (slot, _fut) = reg.allocate().unwrap(); + let replacement = Arc::new(Mutex::new(None)); + + let handler_reg = Arc::clone(®); + let handler_replacement = Arc::clone(&replacement); + slot.set_handler(move |_| { + let (slot, _fut) = handler_reg + .allocate() + .expect("completed slot releases capacity before handler"); + *handler_replacement.lock().unwrap() = Some(slot); + }); + + let raw = slot.into_raw(); + // SAFETY: raw was just produced by `into_raw`. + let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + reclaimed.complete(Ok(CompletionInfo { + flags: 0, + bytes: 0, + tag: 0, + src_addr: 0, + op_context: raw as usize, + data: 0, + })); + + assert_eq!(reg.live_count(), 1); + drop(reclaimed); + assert_eq!(reg.live_count(), 1); + drop(replacement.lock().unwrap().take()); + assert_eq!(reg.live_count(), 0); + } + #[test] fn wake_before_register_still_resolves_future() { let reg = CompletionRegistry::new(1); From 7e685e0b322683d7c0e703fabb764c43e353fa17 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 00:00:38 +0000 Subject: [PATCH 29/56] unbounded-storage: reuse fabric receive buffers --- cmd/unbounded-storage/ARCHITECTURE.md | 2 + cmd/unbounded-storage/src/fabric/backing.rs | 18 ++-- cmd/unbounded-storage/src/fabric/recvpool.rs | 96 ++++++++++++-------- 3 files changed, 70 insertions(+), 46 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 981afbb05..cd2540426 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -402,6 +402,8 @@ streaming RPC server plus client `Transport`. - The completion machinery (`CompletionFuture`, `CompletionInfo`, `CompletionRegistry`, `CompletionSlot`) bridges libfabric CQ entries to futures. +- Each connection keeps stable receive buffers posted and reuses each buffer's + local MR across successful reposts; completion slots remain one-shot. - **RPC server model** (`rpc.rs`): `start_rpc_server` spawns a fixed pool of `rpc_worker_threads` long-lived OS threads pinned to the shard's worker slot. The connection receive path decodes framed requests and enqueues jobs onto a diff --git a/cmd/unbounded-storage/src/fabric/backing.rs b/cmd/unbounded-storage/src/fabric/backing.rs index ae0e28865..ab9d0c36e 100644 --- a/cmd/unbounded-storage/src/fabric/backing.rs +++ b/cmd/unbounded-storage/src/fabric/backing.rs @@ -113,15 +113,13 @@ impl Fabric { } } -/// A transient local-access memory region covering a single control -/// message buffer. Providers that negotiate `FI_MR_LOCAL` (verbs) +/// A local-access memory region covering one stable control-message +/// buffer. Providers that negotiate `FI_MR_LOCAL` (verbs) /// require every send/recv buffer to carry a `desc` from a registered -/// region; this owns that registration for one in-flight buffer and -/// closes it on `Drop`. The completion handler that frees the buffer -/// also drops the `LocalMr`, so the region lives exactly as long as the -/// buffer is posted. Unlike `register_backing`, the `fid_mr` is *not* -/// retained in the `Fabric`: a per-message control MR is closed as soon -/// as its operation completes rather than at fabric teardown. +/// region; this owns that registration and closes it on `Drop`. Send +/// fallbacks retain it for one operation, while receive pools retain it +/// across successful reposts of the same buffer. Unlike +/// `register_backing`, the `fid_mr` is not retained by `Fabric`. pub(crate) struct LocalMr { mr: *mut ffi::fid_mr, } @@ -195,8 +193,8 @@ impl LocalMrCtx { /// Register `ptr[..len]` for local `access` (`FI_RECV` for a recv /// buffer, `FI_TRANSMIT` for a send buffer). Returns `Ok(None)` when /// the provider does not require a local MR. The returned `LocalMr` - /// must outlive the posted operation: capture it in the completion - /// handler so it is closed when the buffer is freed. + /// must outlive every operation using the buffer: capture it with the + /// buffer until the operation completes or the receive lane retires. pub(crate) fn register( &self, ptr: *mut c_void, diff --git a/cmd/unbounded-storage/src/fabric/recvpool.rs b/cmd/unbounded-storage/src/fabric/recvpool.rs index 91effd42a..15bea6132 100644 --- a/cmd/unbounded-storage/src/fabric/recvpool.rs +++ b/cmd/unbounded-storage/src/fabric/recvpool.rs @@ -8,25 +8,24 @@ //! progress thread runs the slot handler installed here: it parses the //! [`MsgHeader`] prefix, hands the framed body to [`InboundDispatch`] //! for routing (server request or client ack), and then immediately -//! re-arms a fresh recv so the pool depth stays constant. This mirrors +//! re-arms the same buffer on a fresh completion slot so the pool depth +//! stays constant. This mirrors //! the self-reposting request-recv pattern the tagged RDM server used, //! but every buffer now serves all four message kinds and is demuxed by //! the header rather than by tag. //! -//! Buffer ownership: each posted recv owns one heap buffer captured by -//! its completion handler. On success the handler reads the buffer and -//! re-arms (allocating a new buffer for the replacement recv, freeing -//! the old one when the closure returns). On error (including the -//! cancellation libfabric delivers when the endpoint is closed) the -//! handler frees its buffer and does not re-arm, so a closing endpoint -//! drains its pool to zero. +//! Buffer ownership: each posted recv owns one stable heap buffer captured +//! by its completion handler. On success the handler reads the buffer and +//! moves it into the replacement recv. On error (including the cancellation +//! libfabric delivers when the endpoint is closed) the handler frees its +//! buffer and does not re-arm, so a closing endpoint drains its pool to zero. //! //! Local descriptors: providers that negotiate `FI_MR_LOCAL` (verbs) //! require every recv buffer to carry a `desc` from a registered region. -//! Each post registers a transient `LocalMr` for its buffer and hands -//! its `desc` to `fi_recv`; the completion handler holds the `LocalMr` -//! alongside the buffer, so the region is closed when the buffer is -//! freed or re-armed. Providers without `FI_MR_LOCAL` (tcp) negotiate it +//! Each receive lane registers one `LocalMr` for its buffer and hands its +//! `desc` to every `fi_recv` repost; the completion handler moves the +//! `LocalMr` alongside the stable buffer. Providers without `FI_MR_LOCAL` +//! (tcp) negotiate it //! off, so `LocalMrCtx::register` returns `None` and the recv posts with //! `desc = NULL`. @@ -35,7 +34,7 @@ use std::ptr; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use super::backing::LocalMrCtx; +use super::backing::{LocalMr, LocalMrCtx}; use super::completion::{CompletionInfo, CompletionRegistry}; use super::dispatch::{EpPtr, InboundDispatch, ReplyCtx}; use super::error::Result; @@ -55,6 +54,34 @@ struct RecvPoolShared { errors: AtomicU64, } +struct RecvBuffer { + local_mr: Option, + bytes: Box<[u8; RECV_BUF_LEN]>, +} + +impl RecvBuffer { + fn new(local_ctx: &LocalMrCtx) -> Result { + let mut bytes = Box::new([0u8; RECV_BUF_LEN]); + let local_mr = local_ctx.register( + bytes.as_mut_ptr() as *mut c_void, + RECV_BUF_LEN, + ffi::FI_RECV, + )?; + Ok(Self { local_mr, bytes }) + } + + fn ptr(&mut self) -> *mut c_void { + self.bytes.as_mut_ptr() as *mut c_void + } + + fn desc(&self) -> *mut c_void { + self.local_mr + .as_ref() + .map(LocalMr::desc) + .unwrap_or(ptr::null_mut()) + } +} + impl RecvPool { /// Arm up to `depth` receives on `ep`. Each completion routes /// through `dispatch` and re-arms itself. `peer` tags routed @@ -113,31 +140,28 @@ impl RecvPoolShared { /// and buffer are reclaimed and nothing is posted), and `Err` for a /// genuine post failure. fn post_one(self: &Arc) -> Result { - let mut buf: Box<[u8; RECV_BUF_LEN]> = Box::new([0u8; RECV_BUF_LEN]); - let buf_ptr = buf.as_mut_ptr(); - // The future is intentionally dropped: this slot is driven by // its handler, not awaited. The registry slot stays live until // `complete` runs on the progress thread. let (slot, _fut) = self.completions.allocate()?; + let recv = RecvBuffer::new(&self.local_ctx)?; + self.post_buffer(slot, recv) + } - // Transient local MR for the buffer on verbs (`FI_MR_LOCAL`); - // `None` on tcp. Held by the completion handler so it is closed - // when the buffer is freed or re-armed. - let local_mr = - self.local_ctx - .register(buf_ptr as *mut c_void, RECV_BUF_LEN, ffi::FI_RECV)?; - let desc = local_mr - .as_ref() - .map(|m| m.desc()) - .unwrap_or(ptr::null_mut()); + fn repost(self: &Arc, recv: RecvBuffer) -> Result { + let (slot, _fut) = self.completions.allocate()?; + self.post_buffer(slot, recv) + } + fn post_buffer( + self: &Arc, + slot: Box, + mut recv: RecvBuffer, + ) -> Result { + let buf_ptr = recv.ptr(); + let desc = recv.desc(); let shared = Arc::clone(self); slot.set_handler(move |result: &std::result::Result| { - // Keep the buffer's local MR alive until the completion is - // reaped; it is closed when this handler (and its captures) - // is dropped. - let _local_mr = &local_mr; match result { Ok(info) => { // A write-with-immediate (page-landed signal) consumes @@ -147,13 +171,13 @@ impl RecvPoolShared { if info.flags & ffi::FI_REMOTE_CQ_DATA != 0 { shared.handle_remote_write(info.data); } else { - shared.handle_message(&buf[..], info.bytes); + shared.handle_message(&recv.bytes[..], info.bytes); } - // Re-arm to keep the pool depth constant. A failure - // or transient EAGAIN here shrinks the pool by one; - // record it but do not unwind on the progress - // thread. Subsequent completions re-arm again. - if !matches!(shared.post_one(), Ok(true)) { + // Re-arm the stable buffer and local MR on a fresh + // completion slot. A failure or transient EAGAIN here + // shrinks the pool by one; record it but do not unwind + // on the progress thread. + if !matches!(shared.repost(recv), Ok(true)) { shared.errors.fetch_add(1, Ordering::Relaxed); } } From f813dc9a16723cd543a28bab4aedfd5777908d40 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 00:15:47 +0000 Subject: [PATCH 30/56] unbounded-storage: distribute fabric worker threads --- cmd/unbounded-storage/ARCHITECTURE.md | 4 +- cmd/unbounded-storage/README.md | 6 +-- cmd/unbounded-storage/src/fabric/config.rs | 29 +++++++++- cmd/unbounded-storage/src/fabric/progress.rs | 39 ++++++++++---- cmd/unbounded-storage/src/fabric/rpc.rs | 14 ++--- cmd/unbounded-storage/src/fabric_group.rs | 57 ++++++++++++-------- 6 files changed, 102 insertions(+), 47 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index cd2540426..a7ddd8865 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -642,8 +642,8 @@ required (kTLS receive on TLS 1.3). |-------|-----------|-------| | Shard | One pinned OS thread per `ServingShard` | Owns `!Send` pool, transport, frontend, RPC handler | | Shard loop | Cooperative tick hooks, noop waker | Busy-poll active, sleep 100us idle | -| Fabric progress | One pinned thread per CQ | Self-driving | -| Fabric RPC serve | Fixed worker pool per shard | Bounded job queue, real-waker completion waits | +| Fabric progress | Fixed pool per fabric unit | Pinned across reserved workers, CQs distributed by NUMA | +| Fabric RPC serve | Fixed worker pool per fabric unit | Bounded job queue, real-waker completion waits | | Storage engine | One pinned storage core per disk | Reached only via `PageChannel` mpsc | | Config watcher | `notify` thread + main loop | Reconciles peers/disks live | diff --git a/cmd/unbounded-storage/README.md b/cmd/unbounded-storage/README.md index 9cfbefe70..029c13a97 100644 --- a/cmd/unbounded-storage/README.md +++ b/cmd/unbounded-storage/README.md @@ -164,10 +164,10 @@ memory_total_bytes = 134217728 # u64 bytes (no K/M/G suffix). Total backing po # evenly across serving shards. 0 -> 128 MiB. [startup.fabric] -progress_threads = 2 # libfabric progress threads per shard. +progress_threads = 2 # libfabric progress threads per fabric unit. progress_poll_us = 10 # progress-thread busy-poll budget (us). -rpc_worker_threads = 4 # fabric RPC worker threads per shard. -max_inflight = 1024 # max in-flight fabric ops per shard (back-pressure). +rpc_worker_threads = 4 # RPC worker threads per fabric unit. +max_inflight = 1024 # max in-flight ops per fabric unit (back-pressure). [startup.fabric.binds.tcp] addr = "0.0.0.0:0" # fabric listen address; :0 picks a free port. diff --git a/cmd/unbounded-storage/src/fabric/config.rs b/cmd/unbounded-storage/src/fabric/config.rs index 617c80584..e1edb757b 100644 --- a/cmd/unbounded-storage/src/fabric/config.rs +++ b/cmd/unbounded-storage/src/fabric/config.rs @@ -93,7 +93,9 @@ pub struct FabricConfig { /// to bound idle CPU. Default is 10. pub progress_poll_us: u32, pub runtime: Arc, - pub worker_idx: WorkerIdx, + /// Runtime workers available to this fabric unit. Progress and RPC + /// pools independently distribute their threads over this list. + pub worker_indices: Vec, pub numa: Option, /// This node's own fabric identity, sent as the connection-manager /// private data on every outbound dial so the accepting peer learns @@ -107,6 +109,9 @@ impl FabricConfig { if self.progress_threads < 1 { return Err(FabricError::BadConfig("progress_threads must be >= 1")); } + if self.worker_indices.is_empty() { + return Err(FabricError::BadConfig("worker_indices must not be empty")); + } if self.max_inflight == 0 { return Err(FabricError::BadConfig("max_inflight must be > 0")); } @@ -132,6 +137,10 @@ impl FabricConfig { } Ok(()) } + + pub(crate) fn worker_for_thread(&self, ordinal: usize) -> WorkerIdx { + self.worker_indices[ordinal % self.worker_indices.len()] + } } /// Apply this crate's libfabric `tcp` provider tuning as process-wide @@ -199,7 +208,7 @@ pub fn defaults_for( progress_threads: 2, progress_poll_us: 10, runtime, - worker_idx, + worker_indices: vec![worker_idx], numa: None, self_peer: PeerId(0), } @@ -261,6 +270,22 @@ mod tests { } } + #[test] + fn validate_rejects_empty_worker_list() { + let mut c = defaults_for("eth0", rt(), WorkerIdx(0)); + c.worker_indices.clear(); + assert!(matches!(c.validate(), Err(FabricError::BadConfig(_)))); + } + + #[test] + fn worker_assignment_round_robins() { + let mut c = defaults_for("eth0", rt(), WorkerIdx(0)); + c.worker_indices = vec![WorkerIdx(3), WorkerIdx(5)]; + assert_eq!(c.worker_for_thread(0), WorkerIdx(3)); + assert_eq!(c.worker_for_thread(1), WorkerIdx(5)); + assert_eq!(c.worker_for_thread(2), WorkerIdx(3)); + } + #[test] fn validate_rejects_zero_max_inflight() { let mut c = defaults_for("eth0", rt(), WorkerIdx(0)); diff --git a/cmd/unbounded-storage/src/fabric/progress.rs b/cmd/unbounded-storage/src/fabric/progress.rs index b8ed5a515..ca4a711b7 100644 --- a/cmd/unbounded-storage/src/fabric/progress.rs +++ b/cmd/unbounded-storage/src/fabric/progress.rs @@ -130,17 +130,15 @@ unsafe impl Send for ProgressGroup {} unsafe impl Sync for ProgressGroup {} impl ProgressGroup { - /// Spawn the progress pool. Creates `cfg.progress_threads` (at least - /// one) threads pinned to `cfg.worker_idx`, each labeled with the - /// fabric's NUMA node (`cfg.numa`, defaulting to 0). In a daemon - /// that runs one fabric per NUMA node this yields one progress - /// thread per NUMA across the process. + /// Spawn the progress pool, distributing threads over the fabric + /// unit's reserved workers. pub(crate) fn new(cfg: &FabricConfig) -> Result { let count = (cfg.progress_threads as usize).max(1); - let numa = cfg.numa.unwrap_or(0); let mut threads = Vec::with_capacity(count); for i in 0..count { + let worker_idx = cfg.worker_for_thread(i); + let numa = cfg.runtime.numa_of(worker_idx).or(cfg.numa).unwrap_or(0); let registry = Arc::new(CqRegistry::new()); let shutdown = Arc::new(AtomicBool::new(false)); let transient_errors = Arc::new(AtomicU64::new(0)); @@ -154,7 +152,7 @@ impl ProgressGroup { let name = format!("fabric-progress-{numa}-{i}"); let handle = cfg.runtime.spawn_pinned( - cfg.worker_idx, + worker_idx, &name, Box::new(move || { progress_loop( @@ -235,12 +233,16 @@ impl ProgressGroup { } fn pick_thread(&self, numa: Option) -> usize { + let start = self.next.fetch_add(1, Ordering::Relaxed) % self.threads.len(); if let Some(n) = numa { - if let Some(i) = self.threads.iter().position(|t| t.numa == n) { - return i; + for offset in 0..self.threads.len() { + let idx = (start + offset) % self.threads.len(); + if self.threads[idx].numa == n { + return idx; + } } } - self.next.fetch_add(1, Ordering::Relaxed) % self.threads.len() + start } /// Total transient (non-EAGAIN, non-EAVAIL) error count across all @@ -421,4 +423,21 @@ mod tests { assert_eq!(total, 0); drop(group); } + + #[test] + fn matching_numa_registrations_round_robin() { + let cfg = test_cfg(2); + let group = ProgressGroup::new(&cfg).expect("group"); + group.stop(); + + let a = 0x1234 as *mut ffi::fid_cq; + let b = 0x5678 as *mut ffi::fid_cq; + group.register(Some(0), a); + group.register(Some(0), b); + + let first = group.threads[0].registry.snapshot(); + let second = group.threads[1].registry.snapshot(); + assert!(first.len() == 1 && first[0] == CqPtr(a)); + assert!(second.len() == 1 && second[0] == CqPtr(b)); + } } diff --git a/cmd/unbounded-storage/src/fabric/rpc.rs b/cmd/unbounded-storage/src/fabric/rpc.rs index a2f63f1a6..5c4570bed 100644 --- a/cmd/unbounded-storage/src/fabric/rpc.rs +++ b/cmd/unbounded-storage/src/fabric/rpc.rs @@ -28,9 +28,9 @@ //! `dest_addr = FI_ADDR_UNSPEC`. //! //! **Worker model**: a fixed pool of `rpc_worker_threads` long-lived -//! OS threads is spawned at `start_rpc_server`, each pinned to the -//! shard's `worker_idx`. Inbound requests are demultiplexed on the -//! progress thread by the connection's receive pool, handed to the +//! OS threads is spawned at `start_rpc_server`, distributed over the +//! fabric unit's reserved workers. Inbound requests are demultiplexed on +//! the progress thread by the connection's receive pool, handed to the //! installed [`RequestSink`], decoded, and enqueued onto a bounded //! [`JobQueue`](super::rpc_queue::JobQueue) rather than spawning a //! thread per request. A pool worker pulls the job, drives the handler @@ -362,13 +362,13 @@ impl Fabric { // Spawn the persistent worker pool before installing the request // sink so a request that lands immediately has a consumer - // waiting. Pin to this shard's worker index so handler - // scratch/MR access stays NUMA-local. + // waiting. Distribute the pool over this fabric unit's reserved + // workers so handler scratch/MR access stays NUMA-local. let runtime = shared.fabric.cfg.runtime.clone(); - let worker_idx = shared.fabric.cfg.worker_idx; let pool_size = shared.fabric.cfg.rpc_worker_threads.max(1); let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { + for i in 0..pool_size { + let worker_idx = shared.fabric.cfg.worker_for_thread(i); let queue = shared.queue.clone(); workers.push(runtime.spawn_pinned( worker_idx, diff --git a/cmd/unbounded-storage/src/fabric_group.rs b/cmd/unbounded-storage/src/fabric_group.rs index 7b64997c4..4978111b1 100644 --- a/cmd/unbounded-storage/src/fabric_group.rs +++ b/cmd/unbounded-storage/src/fabric_group.rs @@ -51,8 +51,8 @@ pub struct FabricUnitSpec { pub unit_idx: usize, pub device_name: String, pub provider: Provider, - /// Worker the fabric's progress and RPC threads pin to. - pub worker_idx: WorkerIdx, + /// Workers over which the fabric's progress and RPC threads are spread. + pub worker_indices: Vec, pub numa: Option, /// Serving-shard indices that share this endpoint. pub shards_assigned: Vec, @@ -92,9 +92,8 @@ pub struct FabricUnitAddress { /// fabric endpoint on that fixed port; binding one endpoint per shard /// would make every shard past the first collide on the port /// (`fi_endpoint` -> EADDRINUSE) once `serving_cores > 1`. -/// - verbs: one endpoint per distinct HCA device, pinned to the first -/// worker of the matching nic-worker group and shared by every shard -/// assigned that device. +/// - verbs: one endpoint per distinct HCA device, using all workers in +/// the matching nic-worker group and shared by every assigned shard. pub fn plan_fabric_units( serving_shards: &[ServingShard], shard_devices: &[Option], @@ -126,7 +125,7 @@ fn plan_tcp_units(serving_shards: &[ServingShard]) -> FabricPlan { unit_idx: 0, device_name: "lo".to_string(), provider: Provider::Tcp, - worker_idx: WorkerIdx(0), + worker_indices: vec![WorkerIdx(0)], numa: None, shards_assigned, // One data backing per assigned shard plus the one shared scratch. @@ -160,13 +159,13 @@ fn plan_verbs_units( let device = dev.clone().unwrap_or_else(|| "lo".to_string()); let unit_idx = *device_to_unit.entry(device.clone()).or_insert_with(|| { let idx = units.len(); - let (worker_idx, numa) = - verbs_worker_for_device(&device, serving_count, nic_workers, hca_dev_names); + let (worker_indices, numa) = + verbs_workers_for_device(&device, serving_count, nic_workers, hca_dev_names); units.push(FabricUnitSpec { unit_idx: idx, device_name: device.clone(), provider: Provider::from_device_name(&device), - worker_idx, + worker_indices, numa, shards_assigned: Vec::new(), expected_mr: 0, @@ -187,17 +186,17 @@ fn plan_verbs_units( } } -/// Worker index (and NUMA) a verbs endpoint on `device` pins to: the -/// first worker of the nic-worker group whose HCA matches `device`. The +/// Worker indices (and NUMA) available to a verbs endpoint on `device`. +/// The /// runtime lays serving shards out at `0..serving_count` and the /// nic-worker groups after them, flattened in order, so a group's base /// is `serving_count` plus the worker counts of all earlier groups. -fn verbs_worker_for_device( +fn verbs_workers_for_device( device: &str, serving_count: usize, nic_workers: &[NicWorkerGroup], hca_dev_names: &[String], -) -> (WorkerIdx, Option) { +) -> (Vec, Option) { let mut flat_base = 0usize; for group in nic_workers { @@ -205,7 +204,10 @@ fn verbs_worker_for_device( .get(group.hca) .is_some_and(|name| name == device); if matches && !group.workers.is_empty() { - return (WorkerIdx((serving_count + flat_base) as u16), group.numa); + let workers = (0..group.workers.len()) + .map(|offset| WorkerIdx((serving_count + flat_base + offset) as u16)) + .collect(); + return (workers, group.numa); } flat_base += group.workers.len(); } @@ -213,7 +215,7 @@ fn verbs_worker_for_device( // No matching nic-worker group. Unreachable when `shard_devices` // came from `assign_shard_devices` over these same `nic_workers`; // fall back to worker 0 so the unit still pins somewhere valid. - (WorkerIdx(0), None) + (vec![WorkerIdx(0)], None) } /// A shard phase-A publication needed by shared RPC workers. @@ -464,9 +466,15 @@ fn build_unit( peers: &[RuntimePeer], self_peer: PeerId, ) -> Result { - let worker = spec.worker_idx.0; - - let mut cfg = fabric::defaults_for(spec.device_name.clone(), runtime.clone(), spec.worker_idx); + let worker_idx = spec + .worker_indices + .first() + .copied() + .ok_or_else(|| format!("fabric unit {} has no workers", spec.unit_idx))?; + let worker = worker_idx.0; + + let mut cfg = fabric::defaults_for(spec.device_name.clone(), runtime.clone(), worker_idx); + cfg.worker_indices = spec.worker_indices.clone(); cfg.provider = spec.provider; cfg.listen = true; cfg.listen_addr = Some( @@ -537,7 +545,7 @@ fn build_unit( page_size, shards_assigned: spec.shards_assigned.clone(), cache_directories: cache_directories.clone(), - worker_idx: spec.worker_idx, + worker_idx, device_name: spec.device_name.clone(), rdma: spec.provider == Provider::Verbs, self_addr, @@ -666,7 +674,7 @@ mod tests { let unit = &plan.units[0]; assert_eq!(unit.device_name, "lo"); assert_eq!(unit.provider, Provider::Tcp); - assert_eq!(unit.worker_idx, WorkerIdx(0)); + assert_eq!(unit.worker_indices, vec![WorkerIdx(0)]); // No NUMA affinity, so shards across nodes can register against it. assert_eq!(unit.numa, None); assert_eq!(unit.shards_assigned, vec![0, 1, 2]); @@ -709,7 +717,7 @@ mod tests { let u0 = &plan.units[0]; assert_eq!(u0.device_name, "mlx5_0"); assert_eq!(u0.provider, Provider::Verbs); - assert_eq!(u0.worker_idx, WorkerIdx(4)); // serving_count(4) + base(0) + assert_eq!(u0.worker_indices, vec![WorkerIdx(4), WorkerIdx(5)]); assert_eq!(u0.numa, Some(0)); assert_eq!(u0.shards_assigned, vec![0, 2]); assert_eq!(u0.expected_mr, 3); // 2 shards + 1 scratch @@ -717,7 +725,7 @@ mod tests { let u1 = &plan.units[1]; assert_eq!(u1.device_name, "mlx5_1"); assert_eq!(u1.provider, Provider::Verbs); - assert_eq!(u1.worker_idx, WorkerIdx(6)); // serving_count(4) + base(2) + assert_eq!(u1.worker_indices, vec![WorkerIdx(6)]); assert_eq!(u1.numa, Some(1)); assert_eq!(u1.shards_assigned, vec![1, 3]); assert_eq!(u1.expected_mr, 3); @@ -734,7 +742,10 @@ mod tests { assert_eq!(plan.units.len(), 1); assert_eq!(plan.shard_to_unit, vec![0, 0]); - assert_eq!(plan.units[0].worker_idx, WorkerIdx(2)); // serving_count(2) + base(0) + assert_eq!( + plan.units[0].worker_indices, + vec![WorkerIdx(2), WorkerIdx(3), WorkerIdx(4), WorkerIdx(5)] + ); assert_eq!(plan.units[0].shards_assigned, vec![0, 1]); assert_eq!(plan.units[0].expected_mr, 3); // 2 shards + 1 scratch } From 4f6a9f89427e79ec39a5cb2992fdb8a88e09ceaa Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 00:45:02 +0000 Subject: [PATCH 31/56] unbounded-storage: make rpc admission panic-safe --- cmd/unbounded-storage/src/fabric/mod.rs | 1 + cmd/unbounded-storage/src/fabric/rpc.rs | 24 ++---- .../src/fabric/rpc_admission.rs | 85 +++++++++++++++++++ cmd/unbounded-storage/src/fabric/rpc_queue.rs | 21 ++++- 4 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 cmd/unbounded-storage/src/fabric/rpc_admission.rs diff --git a/cmd/unbounded-storage/src/fabric/mod.rs b/cmd/unbounded-storage/src/fabric/mod.rs index 176ec4545..3af94a2c5 100644 --- a/cmd/unbounded-storage/src/fabric/mod.rs +++ b/cmd/unbounded-storage/src/fabric/mod.rs @@ -24,6 +24,7 @@ mod pool_handler; mod progress; mod recvpool; mod rpc; +mod rpc_admission; mod rpc_queue; pub(crate) mod scratch; mod sendpool; diff --git a/cmd/unbounded-storage/src/fabric/rpc.rs b/cmd/unbounded-storage/src/fabric/rpc.rs index 5c4570bed..8f848c373 100644 --- a/cmd/unbounded-storage/src/fabric/rpc.rs +++ b/cmd/unbounded-storage/src/fabric/rpc.rs @@ -67,6 +67,7 @@ use super::error::{FabricError, Result as FabResult}; use super::fabric::{Fabric, FabricInner}; use super::ffi; use super::handler::{FabricPage, Handler, HandlerStream, PageRelease, PageSource}; +use super::rpc_admission::RpcAdmission; use super::rpc_queue::{Job, JobQueue}; use super::wire::{MsgHeader, MsgKind}; @@ -236,7 +237,7 @@ struct ServerShared { /// `max_inflight` as the server-side back-pressure limit: a request /// that would exceed it is rejected with a "server overloaded" /// `ERROR_ACK` instead of being enqueued. - inflight: AtomicU64, + admission: Arc, } /// Inbound-request sink installed on the dispatch table. Decodes each @@ -276,29 +277,20 @@ where // a fast "server overloaded" ack instead of being enqueued. This // runs on the progress thread, so the rejection ack is // fire-and-forget (see `reject_overloaded`). - let max_inflight = shared.fabric.cfg.max_inflight as u64; - let prev = shared.inflight.fetch_add(1, Ordering::AcqRel); - if prev >= max_inflight { - shared.inflight.fetch_sub(1, Ordering::AcqRel); + let Some(permit) = shared.admission.try_acquire() else { crate::metrics::fabric_rpc_served(crate::metrics::Outcome::Err); let _ = reject_overloaded(shared, reply.ep, request_id); return; - } - crate::metrics::fabric_inflight_delta(1); + }; let shared_for_job = shared.clone(); let handler_for_job = self.handler.clone(); let job: Job = Box::new(move || { + let _permit = permit; run_worker::(shared_for_job.clone(), handler_for_job, header, req, reply); - shared_for_job.inflight.fetch_sub(1, Ordering::AcqRel); - crate::metrics::fabric_inflight_delta(-1); }); - if shared.queue.push(job).is_err() { - // Queue closed (server shutting down): the job never runs, so - // release the in-flight reservation it would have decremented. - shared.inflight.fetch_sub(1, Ordering::AcqRel); - crate::metrics::fabric_inflight_delta(-1); - } + // A closed queue returns the job; dropping it releases its permit. + drop(shared.queue.push(job)); } } @@ -357,7 +349,7 @@ impl Fabric { page_size, queue: Arc::new(JobQueue::new()), shutdown: AtomicBool::new(false), - inflight: AtomicU64::new(0), + admission: RpcAdmission::new(self.inner_arc().cfg.max_inflight), }); // Spawn the persistent worker pool before installing the request diff --git a/cmd/unbounded-storage/src/fabric/rpc_admission.rs b/cmd/unbounded-storage/src/fabric/rpc_admission.rs new file mode 100644 index 000000000..cac698b97 --- /dev/null +++ b/cmd/unbounded-storage/src/fabric/rpc_admission.rs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Panic-safe admission accounting for queued and running RPC requests. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub(crate) struct RpcAdmission { + limit: u64, + inflight: AtomicU64, +} + +impl RpcAdmission { + pub(crate) fn new(limit: usize) -> Arc { + Arc::new(Self { + limit: limit as u64, + inflight: AtomicU64::new(0), + }) + } + + pub(crate) fn try_acquire(self: &Arc) -> Option { + let previous = self.inflight.fetch_add(1, Ordering::AcqRel); + if previous >= self.limit { + self.inflight.fetch_sub(1, Ordering::AcqRel); + return None; + } + + crate::metrics::fabric_inflight_delta(1); + Some(InflightPermit { + admission: self.clone(), + }) + } + + #[cfg(test)] + fn inflight(&self) -> u64 { + self.inflight.load(Ordering::Acquire) + } +} + +pub(crate) struct InflightPermit { + admission: Arc, +} + +impl Drop for InflightPermit { + fn drop(&mut self) { + self.admission.inflight.fetch_sub(1, Ordering::AcqRel); + crate::metrics::fabric_inflight_delta(-1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn permit_releases_capacity_exactly_once() { + let admission = RpcAdmission::new(1); + let permit = admission.try_acquire().expect("first request admitted"); + + assert!(admission.try_acquire().is_none()); + assert_eq!(admission.inflight(), 1); + + drop(permit); + assert_eq!(admission.inflight(), 0); + assert!(admission.try_acquire().is_some()); + } + + #[test] + fn permit_releases_capacity_during_unwind() { + let admission = RpcAdmission::new(1); + let unwind_admission = admission.clone(); + + let result = std::panic::catch_unwind(move || { + let _permit = unwind_admission + .try_acquire() + .expect("request admitted before panic"); + panic!("test panic"); + }); + + assert!(result.is_err()); + assert_eq!(admission.inflight(), 0); + assert!(admission.try_acquire().is_some()); + } +} diff --git a/cmd/unbounded-storage/src/fabric/rpc_queue.rs b/cmd/unbounded-storage/src/fabric/rpc_queue.rs index 16c681bbb..092a8c7db 100644 --- a/cmd/unbounded-storage/src/fabric/rpc_queue.rs +++ b/cmd/unbounded-storage/src/fabric/rpc_queue.rs @@ -50,7 +50,7 @@ impl JobQueue { } /// Enqueue `job`. Returns `Err(job)` if the queue is closed so the - /// caller can unwind any accounting it did for the rejected work. + /// caller can drop the work and its captured resources. pub(crate) fn push(&self, job: Job) -> Result<(), Job> { let mut inner = self.inner.lock().expect("job queue poisoned"); if inner.closed { @@ -120,6 +120,25 @@ mod tests { assert!(rejected.is_err(), "closed queue must reject pushes"); } + #[test] + fn rejected_job_drops_captured_resources() { + struct DropCount(Arc); + + impl Drop for DropCount { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + let q = JobQueue::new(); + let drops = Arc::new(AtomicU32::new(0)); + let guard = DropCount(drops.clone()); + q.close(); + + drop(q.push(Box::new(move || drop(guard)))); + assert_eq!(drops.load(Ordering::SeqCst), 1); + } + #[test] fn close_drains_remaining_then_returns_none() { let q = JobQueue::new(); From 44b01281f62fcd9bc32b68dca84e1ec58872b2a2 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 00:28:30 +0000 Subject: [PATCH 32/56] unbounded-storage: simplify network ring ownership --- cmd/unbounded-storage/src/backend/registry.rs | 3 +- .../src/frontend/http_serve.rs | 3 +- .../src/frontend/s3_serve.rs | 3 +- cmd/unbounded-storage/src/main.rs | 40 +++---- cmd/unbounded-storage/src/ring/network.rs | 108 +++++++++--------- cmd/unbounded-storage/src/ring/tls_recv.rs | 8 +- 6 files changed, 76 insertions(+), 89 deletions(-) diff --git a/cmd/unbounded-storage/src/backend/registry.rs b/cmd/unbounded-storage/src/backend/registry.rs index 654418f36..1987d4db4 100644 --- a/cmd/unbounded-storage/src/backend/registry.rs +++ b/cmd/unbounded-storage/src/backend/registry.rs @@ -315,7 +315,6 @@ impl PageStream for RegistryFetchStream<'_> { #[cfg(test)] mod tests { - use std::cell::RefCell; use std::ptr; use std::rc::Rc; use std::task::{RawWaker, RawWakerVTable, Waker}; @@ -353,7 +352,7 @@ mod tests { } fn registry(specs: &[BackendSpec]) -> BackendRegistry { - let ring = Rc::new(RefCell::new(NetworkRing::new(8).expect("network ring"))); + let ring = Rc::new(NetworkRing::new(8).expect("network ring")); BackendRegistry::new(specs, NetHandle::new(ring), 4096, ptr::null_mut()) .expect("seed registry") } diff --git a/cmd/unbounded-storage/src/frontend/http_serve.rs b/cmd/unbounded-storage/src/frontend/http_serve.rs index 8e14460dc..dae619f82 100644 --- a/cmd/unbounded-storage/src/frontend/http_serve.rs +++ b/cmd/unbounded-storage/src/frontend/http_serve.rs @@ -984,7 +984,6 @@ mod tests { use crate::frontend::range::StripeSlice; use crate::http::{request_is_bodyless, request_wants_keep_alive}; use crate::storage::disks::CacheDirectorySet; - use std::cell::RefCell; fn spec(id: &str, addr: &str) -> FrontendSpec { FrontendSpec { @@ -1356,7 +1355,7 @@ mod tests { fn driver_idle_progress_returns_false_without_clients() { // Needs a real socket ring; skip gracefully when unavailable. let ring = match crate::ring::NetworkRing::new(16) { - Ok(r) => Rc::new(RefCell::new(r)), + Ok(r) => Rc::new(r), Err(e) => { eprintln!("driver_idle_progress: ring unavailable: {e}; skipping"); return; diff --git a/cmd/unbounded-storage/src/frontend/s3_serve.rs b/cmd/unbounded-storage/src/frontend/s3_serve.rs index aa32e94fa..40c03a89a 100644 --- a/cmd/unbounded-storage/src/frontend/s3_serve.rs +++ b/cmd/unbounded-storage/src/frontend/s3_serve.rs @@ -694,7 +694,6 @@ mod tests { use super::*; use crate::bufferpool::{PipelinedRead, ReadStream, WindowedRead}; use crate::config::{HttpFrontendConfig, S3FrontendConfig}; - use std::cell::RefCell; fn spec(id: &str, addr: &str) -> FrontendSpec { FrontendSpec { @@ -858,7 +857,7 @@ mod tests { fn driver_idle_progress_returns_false_without_clients() { // Needs a real socket ring; skip gracefully when unavailable. let ring = match crate::ring::NetworkRing::new(16) { - Ok(r) => Rc::new(RefCell::new(r)), + Ok(r) => Rc::new(r), Err(e) => { eprintln!("driver_idle_progress: ring unavailable: {e}; skipping"); return; diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index be3991d9d..5fbc83c22 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -600,7 +600,7 @@ fn run_shard( // register while `backing` is still owned, before `Pool::new` moves // it. let socket = match NetworkRing::new(256) { - Ok(s) => Rc::new(RefCell::new(s)), + Ok(s) => Rc::new(s), Err(e) => { let _ = tx.send(ShardReady::Failed(format!( "worker={}: NetworkRing::new: {e}", @@ -609,7 +609,7 @@ fn run_shard( return; } }; - if let Err(e) = socket.borrow().register_backing(&backing) { + if let Err(e) = socket.register_backing(&backing) { let _ = tx.send(ShardReady::Failed(format!( "worker={}: socket register_backing: {e}", widx.0, @@ -742,10 +742,7 @@ fn run_shard( // rings receive into their own scratch backing and keep the blocking // drain fallback. Installed before serving begins so every cancelled // RECV is covered. - unbounded_storage::backend::install_recv_quarantine( - &socket.borrow(), - pool.recv_quarantine_handle(), - ); + unbounded_storage::backend::install_recv_quarantine(&socket, pool.recv_quarantine_handle()); // The RPC server that serves peer cache-hits and forwards Chord hops // is brought up and owned by the `FabricGroup` (shared across every @@ -758,12 +755,7 @@ fn run_shard( let mut shard_loop = ShardLoop::new(); { let socket = socket.clone(); - // `progress()` takes `&self`, so this must be a *shared* borrow: - // the origin backend holds `socket.borrow()` across its recv/send - // awaits (see `backend::http`), so a `borrow_mut()` here would hit - // a `BorrowMutError` panic whenever a cache-miss fetch is in - // flight across a shard tick. Shared borrows coexist. - shard_loop.add_tick_hook(move || socket.borrow().progress()); + shard_loop.add_tick_hook(move || socket.progress()); } // Cross-shard fan-out channel. This shard publishes the sender half @@ -816,19 +808,17 @@ fn run_shard( if peer.shard_index == own_shard_index { continue; } - let buf_index = match socket - .borrow() - .register_region_indexed(peer.backing_base as *mut u8, peer.backing_len) - { - Ok(idx) => idx, - Err(e) => { - phaseb_guard.report_failed(format!( - "worker={}: register peer shard {} backing: {e}", - widx.0, peer.shard_index, - )); - return; - } - }; + let buf_index = + match socket.register_region_indexed(peer.backing_base as *mut u8, peer.backing_len) { + Ok(idx) => idx, + Err(e) => { + phaseb_guard.report_failed(format!( + "worker={}: register peer shard {} backing: {e}", + widx.0, peer.shard_index, + )); + return; + } + }; routed[peer.shard_index] = Some(FanoutPeer { channel: peer.channel.clone(), buf_index, diff --git a/cmd/unbounded-storage/src/ring/network.rs b/cmd/unbounded-storage/src/ring/network.rs index 0fd84e456..388964446 100644 --- a/cmd/unbounded-storage/src/ring/network.rs +++ b/cmd/unbounded-storage/src/ring/network.rs @@ -441,18 +441,18 @@ impl NetworkRing { /// `&'a NetworkRing`, so they are not `'static`. The serving frontend /// multiplexes many long-lived per-connection futures the shard loop /// polls each tick, so it needs `'static` futures. `NetHandle` owns an -/// `Rc>` clone instead of borrowing the ring: each +/// `Rc` clone instead of borrowing the ring: each /// method parks on the ring's submission backpressure before submitting, /// then returns a future that owns its own ring clone plus the op's -/// `Rc` and re-borrows the ring inside `poll`/`drop`. +/// `Rc`. #[derive(Clone)] pub struct NetHandle { - ring: Rc>, + ring: Rc, } impl NetHandle { /// Wrap a shared network ring. - pub fn new(ring: Rc>) -> Self { + pub fn new(ring: Rc) -> Self { Self { ring } } @@ -462,12 +462,12 @@ impl NetHandle { /// called during bring-up before any I/O is in flight (see /// [`NetworkRing::register_region_indexed`]). pub fn register_peer_region(&self, base: *mut u8, len: usize) -> io::Result { - self.ring.borrow().register_region_indexed(base, len) + self.ring.register_region_indexed(base, len) } - /// Shared access to the underlying ring cell, for sibling-module ops + /// Shared access to the underlying ring, for sibling-module ops /// (see `tls_recv`) that submit on this handle's ring. - pub(crate) fn ring_cell(&self) -> &Rc> { + pub(crate) fn ring_cell(&self) -> &Rc { &self.ring } @@ -493,7 +493,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ud = r.core.alloc_user_data(); let sqe = opcode::Accept::new( types::Fd(listen_fd), @@ -523,7 +523,7 @@ impl NetHandle { let addr = Box::new(addr); OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ud = r.core.alloc_user_data(); let sqe = opcode::Connect::new(types::Fd(fd), addr.as_ptr(), addr.socklen()) .build() @@ -548,7 +548,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = buf.as_ptr(); let len = buf.len() as u32; let ud = r.core.alloc_user_data(); @@ -576,7 +576,7 @@ impl NetHandle { let buf = Rc::new(RefCell::new(vec![0u8; max_len])); OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = buf.borrow_mut().as_mut_ptr(); let len = max_len as u32; let ud = r.core.alloc_user_data(); @@ -614,7 +614,7 @@ impl NetHandle { } OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = r.fixed_ptr(buf_index, page_byte_offset)?; let ud = r.core.alloc_user_data(); let sqe = opcode::SendZc::new(types::Fd(fd), ptr, len as u32) @@ -649,7 +649,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = r.fixed_ptr(buf_index, page_byte_offset)?; let ud = r.core.alloc_user_data(); let sqe = opcode::SendZc::new(types::Fd(fd), ptr, len as u32) @@ -680,7 +680,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = r.fixed_ptr(buf_index, page_byte_offset)?; let ud = r.core.alloc_user_data(); let sqe = opcode::Recv::new(types::Fd(fd), ptr as *mut u8, len as u32) @@ -706,12 +706,11 @@ impl NetHandle { } /// `'static` op future for the [`NetHandle`] path. Owns an -/// `Rc>` clone instead of borrowing the ring, so it -/// can be stored across shard-loop ticks. The owned slot carries the -/// completion state; the ring borrow is taken transiently in `poll` (to -/// pump `progress`) and `drop` (to best-effort cancel). +/// `Rc` clone instead of borrowing the ring, so it can be +/// stored across shard-loop ticks. The owned slot carries the completion +/// state. pub(crate) struct OwnedNetFut { - ring: Rc>, + ring: Rc, user_data: u64, slot: Rc, /// When `Some(offset)`, this future is a fixed-buffer RECV whose @@ -730,11 +729,11 @@ pub(crate) struct OwnedNetFut { } pub(crate) struct OwnedSubmitSlot { - ring: Rc>, + ring: Rc, } impl OwnedSubmitSlot { - pub(crate) fn new(ring: Rc>) -> Self { + pub(crate) fn new(ring: Rc) -> Self { Self { ring } } } @@ -743,14 +742,15 @@ impl Future for OwnedSubmitSlot { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - let ring = self.ring.borrow(); - let mut submit = SubmitSlot { core: &ring.core }; + let mut submit = SubmitSlot { + core: &self.ring.core, + }; Pin::new(&mut submit).poll(cx) } } impl OwnedNetFut { - pub(crate) fn new(ring: Rc>, user_data: u64, slot: Rc) -> Self { + pub(crate) fn new(ring: Rc, user_data: u64, slot: Rc) -> Self { Self { ring, user_data, @@ -793,7 +793,7 @@ impl Future for OwnedNetFut { if this.slot.is_done() { return Poll::Ready(this.slot.result()); } - this.ring.borrow().progress(); + this.ring.progress(); if this.slot.is_done() { return Poll::Ready(this.slot.result()); } @@ -805,20 +805,20 @@ impl Future for OwnedNetFut { impl Drop for OwnedNetFut { fn drop(&mut self) { if !self.slot.is_done() { - if let Ok(ring) = self.ring.try_borrow() { - match self.fixed_recv_offset { - Some(off) => { - ring.core.cancel_fixed_recv(self.user_data, off, &self.slot); - } - None if self.abandon_send_zc => { - ring.core.abandon_send_zc( - self.user_data, - self.send_zc_source_offset, - &self.slot, - ); - } - None => ring.core.cancel(self.user_data), + match self.fixed_recv_offset { + Some(off) => { + self.ring + .core + .cancel_fixed_recv(self.user_data, off, &self.slot); } + None if self.abandon_send_zc => { + self.ring.core.abandon_send_zc( + self.user_data, + self.send_zc_source_offset, + &self.slot, + ); + } + None => self.ring.core.cancel(self.user_data), } } } @@ -1127,7 +1127,7 @@ mod tests { return; } - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { @@ -1204,7 +1204,7 @@ mod tests { return; } - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { @@ -1231,7 +1231,7 @@ mod tests { Poll::Pending => {} } assert_eq!( - ring.borrow().core.in_flight(), + ring.core.in_flight(), 1, "the RECV should be outstanding after the first poll", ); @@ -1239,7 +1239,7 @@ mod tests { // Dropping the parked future must block until the RECV is reaped. drop(fut); assert_eq!( - ring.borrow().core.in_flight(), + ring.core.in_flight(), 0, "dropping a draining recv_fixed must reap the in-flight op", ); @@ -1300,7 +1300,7 @@ mod tests { events: Rc::clone(&events), })); - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { @@ -1325,7 +1325,7 @@ mod tests { Poll::Pending => {} } assert_eq!( - ring.borrow().core.in_flight(), + ring.core.in_flight(), 1, "the RECV should be outstanding after the first poll", ); @@ -1335,7 +1335,7 @@ mod tests { // yet reclaimed. drop(fut); assert_eq!( - ring.borrow().core.in_flight(), + ring.core.in_flight(), 1, "dropping a quarantined recv_fixed must not block to drain", ); @@ -1348,8 +1348,8 @@ mod tests { // Pump progress() until the cancelled RECV's CQE is reaped. Only // then is the page reclaimed back to the free list. let mut spins = 0u32; - while ring.borrow().core.in_flight() != 0 { - ring.borrow().progress(); + while ring.core.in_flight() != 0 { + ring.progress(); spins += 1; assert!(spins < 5_000_000, "cancelled RECV was never reaped"); } @@ -1370,7 +1370,7 @@ mod tests { #[test] fn handle_send_recv_roundtrip() { let ring = match NetworkRing::new(16) { - Ok(r) => Rc::new(RefCell::new(r)), + Ok(r) => Rc::new(r), Err(e) => { eprintln!("handle_send_recv_roundtrip: ring unavailable: {e}; skipping"); return; @@ -1428,7 +1428,7 @@ mod tests { #[test] fn handle_submission_parks_when_ring_depth_is_full() { let ring = match NetworkRing::new(1) { - Ok(r) => Rc::new(RefCell::new(r)), + Ok(r) => Rc::new(r), Err(e) => { eprintln!("handle_backpressure: ring unavailable: {e}; skipping"); return; @@ -1455,7 +1455,7 @@ mod tests { Poll::Ready(other) => panic!("first recv unexpectedly completed: {other:?}"), Poll::Pending => {} } - assert_eq!(ring.borrow().core.in_flight(), 1); + assert_eq!(ring.core.in_flight(), 1); let mut second = Box::pin(handle.recv(b, 64)); match second.as_mut().poll(&mut cx) { @@ -1463,7 +1463,7 @@ mod tests { Poll::Pending => {} } assert_eq!( - ring.borrow().core.in_flight(), + ring.core.in_flight(), 1, "parked submitter must not push past ring depth", ); @@ -1541,7 +1541,7 @@ mod tests { return; } - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { @@ -1592,7 +1592,7 @@ mod tests { return; } }; - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { @@ -1637,7 +1637,7 @@ mod tests { return; } }; - let ring = Rc::new(RefCell::new(ring)); + let ring = Rc::new(ring); let handle = NetHandle::new(Rc::clone(&ring)); let Some((a, b)) = tcp_loopback_pair() else { diff --git a/cmd/unbounded-storage/src/ring/tls_recv.rs b/cmd/unbounded-storage/src/ring/tls_recv.rs index 19ad65211..b34a2e423 100644 --- a/cmd/unbounded-storage/src/ring/tls_recv.rs +++ b/cmd/unbounded-storage/src/ring/tls_recv.rs @@ -44,7 +44,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ud = r.core.alloc_user_data(); let sqe = opcode::PollAdd::new(types::Fd(fd), flags) .build() @@ -72,7 +72,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot, state) = { - let r = ring.borrow(); + let r = ring.as_ref(); let state = RecvMsgState::new_heap(max_len); let msg_ptr = state.borrow().msghdr_ptr(); let ud = r.core.alloc_user_data(); @@ -115,7 +115,7 @@ impl NetHandle { async move { OwnedSubmitSlot::new(Rc::clone(&ring)).await; let (ud, slot, state) = { - let r = ring.borrow(); + let r = ring.as_ref(); let ptr = r.fixed_ptr(0, page_byte_offset)?; let state = RecvMsgState::new(ptr as *mut u8, len); let msg_ptr = state.borrow().msghdr_ptr(); @@ -143,7 +143,7 @@ impl NetHandle { // graceful close_notify from a fatal alert. The op has // completed, so the kernel-written bytes are stable. let alert_desc = if record_type == TLS_RECORD_TYPE_ALERT && n >= 2 { - let ptr = ring.borrow().fixed_ptr(0, page_byte_offset)?; + let ptr = ring.fixed_ptr(0, page_byte_offset)?; // SAFETY: `ptr` addresses the registered page the kernel // just decrypted at least two alert bytes into. Some(unsafe { *ptr.add(1) }) From 1ebe72d3c36b0fdbf715c5824a8b05e88684117c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 00:58:32 +0000 Subject: [PATCH 33/56] unbounded-storage: simplify fabric ack delivery --- cmd/unbounded-storage/ARCHITECTURE.md | 7 +- cmd/unbounded-storage/src/fabric/dispatch.rs | 8 +- cmd/unbounded-storage/src/fabric/transport.rs | 273 ++++++++++++------ 3 files changed, 186 insertions(+), 102 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index a7ddd8865..4ea36191e 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -412,9 +412,10 @@ streaming RPC server plus client `Transport`. submits libfabric writes/sends, and parks on completion futures with a real thread waker. Wire framing uses an 8-byte `MsgHeader` prefix with a message kind and request id. The client sends a bincode `RequestHeader` plus request - body; the server `fi_write`s each page into the client's destination MR and - sends one `PageAck` per page. A short success sends `RESPONSE_END`; any error - sends `ERROR_ACK`. `RpcServerHandle::drop` uninstalls the request sink, closes + body. Verbs uses write-with-immediate to deliver typed page ordinals; the TCP + fallback follows each `fi_write` with a framed bincode `PageAck`. A short + success sends `RESPONSE_END`; any error sends `ERROR_ACK`. + `RpcServerHandle::drop` uninstalls the request sink, closes the queue, signals shutdown, and joins the workers. - `Handler`/`HandlerStream` is the server-side resolution trait; `PoolHandler`/`PoolHandlerStream` serve locally-resident pages. diff --git a/cmd/unbounded-storage/src/fabric/dispatch.rs b/cmd/unbounded-storage/src/fabric/dispatch.rs index 203d560fe..19c6a27dd 100644 --- a/cmd/unbounded-storage/src/fabric/dispatch.rs +++ b/cmd/unbounded-storage/src/fabric/dispatch.rs @@ -73,7 +73,7 @@ pub(crate) trait AckSink: Send + Sync { /// Deliver one inbound control message (`RESPONSE_END` or /// `ERROR_ACK`) for this request. `body` is the message payload /// after the 8-byte header. - fn deliver(&self, kind: MsgKind, body: &[u8]); + fn deliver(&self, kind: MsgKind, body: Vec); /// Deliver one page-landed signal carried by an RDMA /// write-with-immediate completion (no framed message): `ordinal` @@ -237,7 +237,7 @@ impl InboundDispatch { MsgKind::PageAck | MsgKind::ResponseEnd | MsgKind::ErrorAck => { let sink = self.streams.lock().unwrap().get(&request_id).cloned(); match sink { - Some(sink) => sink.deliver(kind, &body), + Some(sink) => sink.deliver(kind, body), None => { // A late ack for a request whose stream has // already completed and unregistered. Benign. @@ -278,8 +278,8 @@ mod tests { pages: StdMutex>, } impl AckSink for RecordingAck { - fn deliver(&self, kind: MsgKind, body: &[u8]) { - self.got.lock().unwrap().push((kind, body.to_vec())); + fn deliver(&self, kind: MsgKind, body: Vec) { + self.got.lock().unwrap().push((kind, body)); } fn deliver_page(&self, ordinal: u32) { self.pages.lock().unwrap().push(ordinal); diff --git a/cmd/unbounded-storage/src/fabric/transport.rs b/cmd/unbounded-storage/src/fabric/transport.rs index af9a01c3d..bd0ab02c7 100644 --- a/cmd/unbounded-storage/src/fabric/transport.rs +++ b/cmd/unbounded-storage/src/fabric/transport.rs @@ -219,52 +219,44 @@ fn forward_dest_mr(mr: MrHandle, dest_offset: usize) -> MrHandle { /// delivers acks from the progress thread. Implements [`AckSink`] so /// the dispatch can route replies straight into the queue. pub(crate) struct StreamShared { - /// FIFO of reply messages arriving from the dispatch. - queue: Mutex>, - /// Waker registered by the stream when it last returned Pending. - waker: Mutex>, + delivery: Mutex, } impl StreamShared { fn push(&self, item: RecvCompletion) { - if let Ok(mut q) = self.queue.lock() { - q.push_back(item); - } - if let Ok(mut w) = self.waker.lock() { - if let Some(w) = w.take() { - w.wake(); - } + let waker = { + let mut delivery = self.delivery.lock().expect("stream delivery poisoned"); + delivery.queue.push_back(item); + delivery.waker.take() + }; + if let Some(waker) = waker { + waker.wake(); } } } impl AckSink for StreamShared { - fn deliver(&self, kind: MsgKind, body: &[u8]) { - self.push(RecvCompletion { + fn deliver(&self, kind: MsgKind, body: Vec) { + self.push(RecvCompletion::Message { kind, - payload: body.to_vec(), + payload: body, }); } fn deliver_page(&self, ordinal: u32) { - // A page-landed write-with-immediate carries no framed body. - // Re-encode it as the internal `PAGE_ACK` event so the poll - // loop's existing ack bookkeeping (bounds, dedup, end-of-stream) - // handles it verbatim. A serialize failure is impossible for a - // fixed-size struct; fall back to an empty body, which the poll - // loop rejects as a malformed ack rather than panicking on the - // progress thread. - let payload = bincode::serialize(&PageAck { page_idx: ordinal }).unwrap_or_default(); - self.push(RecvCompletion { - kind: MsgKind::PageAck, - payload, - }); + self.push(RecvCompletion::PageLanded(ordinal)); } } -struct RecvCompletion { - kind: MsgKind, - payload: Vec, +#[derive(Default)] +struct DeliveryState { + queue: VecDeque, + waker: Option, +} + +enum RecvCompletion { + Message { kind: MsgKind, payload: Vec }, + PageLanded(u32), } /// Stream returned by `FabricTransport::bulk_get`. On first poll it @@ -304,6 +296,7 @@ enum StreamState<'a> { /// the write-with-immediate path; Drop releases it. page_handle: u16, acked: Vec, + acked_count: usize, ended: bool, emitted_error: bool, log: Option, @@ -408,6 +401,7 @@ impl<'a, R> PageStream for FabricBulkStream<'a, R> { request_id, page_handle, acked: vec![false; dsts.len()], + acked_count: 0, ended: false, emitted_error: false, log: Some(log), @@ -427,6 +421,7 @@ impl<'a, R> PageStream for FabricBulkStream<'a, R> { request_id: _, page_handle: _, acked, + acked_count, ended, emitted_error, log, @@ -434,84 +429,108 @@ impl<'a, R> PageStream for FabricBulkStream<'a, R> { if *ended { return Poll::Ready(None); } - // Register waker first to avoid a lost wakeup. - if let Ok(mut w) = shared.waker.lock() { - *w = Some(cx.waker().clone()); - } - let item = shared.queue.lock().ok().and_then(|mut q| q.pop_front()); + let item = { + let mut delivery = + shared.delivery.lock().expect("stream delivery poisoned"); + match delivery.queue.pop_front() { + Some(item) => Some(item), + None => { + delivery.waker = Some(cx.waker().clone()); + None + } + } + }; match item { None => return Poll::Pending, - Some(RecvCompletion { kind, payload }) => match kind { - MsgKind::PageAck => { - let ack: PageAck = match bincode::deserialize(&payload) { - Ok(v) => v, - Err(_) => { + Some(item) => { + let (kind, payload, page_idx) = match item { + RecvCompletion::Message { kind, payload } => { + let page_idx = if kind == MsgKind::PageAck { + let ack: PageAck = match bincode::deserialize(&payload) { + Ok(v) => v, + Err(_) => { + *ended = true; + if let Some(mut l) = log.take() { + l.finish_err("malformed PAGE_ACK"); + } + return Poll::Ready(Some(Err( + PoolError::transport(FabricError::BadConfig( + "malformed PAGE_ACK", + )), + ))); + } + }; + Some(ack.page_idx) + } else { + None + }; + (kind, payload, page_idx) + } + RecvCompletion::PageLanded(page_idx) => { + (MsgKind::PageAck, Vec::new(), Some(page_idx)) + } + }; + match kind { + MsgKind::PageAck => { + let page_idx = page_idx.expect("page event missing ordinal"); + let idx = page_idx as usize; + if idx >= dsts.len() { *ended = true; if let Some(mut l) = log.take() { - l.finish_err("malformed PAGE_ACK"); + l.finish_err("PAGE_ACK index out of range"); } return Poll::Ready(Some(Err(PoolError::transport( - FabricError::BadConfig("malformed PAGE_ACK"), + FabricError::BadConfig("PAGE_ACK index out of range"), )))); } - }; - let idx = ack.page_idx as usize; - if idx >= dsts.len() { - *ended = true; - if let Some(mut l) = log.take() { - l.finish_err("PAGE_ACK index out of range"); + if acked[idx] { + *ended = true; + if let Some(mut l) = log.take() { + l.finish_err("duplicate PAGE_ACK index"); + } + return Poll::Ready(Some(Err(PoolError::transport( + FabricError::BadConfig("duplicate PAGE_ACK index"), + )))); } - return Poll::Ready(Some(Err(PoolError::transport( - FabricError::BadConfig("PAGE_ACK index out of range"), - )))); + acked[idx] = true; + *acked_count += 1; + // Full success carries no RESPONSE_END; + // the final distinct ack ends the stream. + if *acked_count == dsts.len() { + *ended = true; + if let Some(mut l) = log.take() { + l.field("acked", *acked_count).finish_ok(); + } + } + return Poll::Ready(Some(Ok(dsts[idx]))); } - if acked[idx] { + MsgKind::ResponseEnd => { *ended = true; if let Some(mut l) = log.take() { - l.finish_err("duplicate PAGE_ACK index"); + l.field("acked", *acked_count).finish_ok(); } - return Poll::Ready(Some(Err(PoolError::transport( - FabricError::BadConfig("duplicate PAGE_ACK index"), - )))); + return Poll::Ready(None); } - acked[idx] = true; - // Full success carries no RESPONSE_END; - // the last in-order ack ends the stream. - if acked.iter().all(|a| *a) { + MsgKind::ErrorAck => { *ended = true; + if *emitted_error { + return Poll::Ready(None); + } + *emitted_error = true; + let ack = bincode::deserialize::(&payload) + .unwrap_or_else(|_| { + ErrorAck::generic("server error (undecodable)") + }); if let Some(mut l) = log.take() { - l.field("acked", acked.len()).finish_ok(); + l.finish_err(&ack.message); } + return Poll::Ready(Some(Err(error_ack_to_pool_error(ack)))); } - return Poll::Ready(Some(Ok(dsts[idx]))); + // A REQUEST is never routed to a client ack + // sink; ignore any stray delivery. + MsgKind::Request => continue, } - MsgKind::ResponseEnd => { - *ended = true; - if let Some(mut l) = log.take() { - l.field("acked", acked.iter().filter(|a| **a).count()) - .finish_ok(); - } - return Poll::Ready(None); - } - MsgKind::ErrorAck => { - *ended = true; - if *emitted_error { - return Poll::Ready(None); - } - *emitted_error = true; - let ack = bincode::deserialize::(&payload) - .unwrap_or_else(|_| { - ErrorAck::generic("server error (undecodable)") - }); - if let Some(mut l) = log.take() { - l.finish_err(&ack.message); - } - return Poll::Ready(Some(Err(error_ack_to_pool_error(ack)))); - } - // A REQUEST is never routed to a client ack - // sink; ignore any stray delivery. - MsgKind::Request => continue, - }, + } } } } @@ -604,8 +623,7 @@ fn launch( let (ep, _addr) = fabric.resolve_peer(peer)?; let shared = Arc::new(StreamShared { - queue: Mutex::new(VecDeque::new()), - waker: Mutex::new(None), + delivery: Mutex::new(DeliveryState::default()), }); // Register the ack sink BEFORE sending so no reply can race ahead @@ -737,17 +755,18 @@ mod tests { fn active_stream<'a>(dsts: &'a [PageRef], acked: Vec) -> FabricBulkStream<'a, ()> { assert_eq!(acked.len(), dsts.len()); + let acked_count = acked.iter().filter(|value| **value).count(); FabricBulkStream { state: StreamState::Active { shared: Arc::new(StreamShared { - queue: Mutex::new(VecDeque::new()), - waker: Mutex::new(None), + delivery: Mutex::new(DeliveryState::default()), }), dsts, dispatch: InboundDispatch::new(), request_id: 1, page_handle: 0, acked, + acked_count, ended: false, emitted_error: false, log: None, @@ -761,7 +780,7 @@ mod tests { panic!("expected active stream"); }; let payload = bincode::serialize(&PageAck { page_idx }).expect("serialize page ack"); - shared.push(RecvCompletion { + shared.push(RecvCompletion::Message { kind: MsgKind::PageAck, payload, }); @@ -771,7 +790,7 @@ mod tests { let StreamState::Active { shared, .. } = &stream.state else { panic!("expected active stream"); }; - shared.push(RecvCompletion { + shared.push(RecvCompletion::Message { kind: MsgKind::ResponseEnd, payload: Vec::new(), }); @@ -786,7 +805,7 @@ mod tests { message: message.to_string(), }) .expect("serialize error ack"); - shared.push(RecvCompletion { + shared.push(RecvCompletion::Message { kind: MsgKind::ErrorAck, payload, }); @@ -1005,6 +1024,37 @@ mod tests { } } + #[test] + fn immediate_duplicate_has_same_error_as_framed_ack() { + let dsts = [ + PageRef { + page_idx: 0, + offset: 0, + len: 4096, + }, + PageRef { + page_idx: 1, + offset: 0, + len: 4096, + }, + ]; + let mut stream = active_stream(&dsts, vec![false, false]); + push_page_landed(&mut stream, 1); + push_page_landed(&mut stream, 1); + + assert!(matches!( + poll_once(&mut stream), + Poll::Ready(Some(Ok(PageRef { page_idx: 1, .. }))) + )); + match poll_once(&mut stream) { + Poll::Ready(Some(Err(PoolError::Transport(err)))) => assert!( + err.to_string().contains("duplicate PAGE_ACK index"), + "unexpected error: {err}", + ), + other => panic!("expected transport error, got {other:?}"), + } + } + #[test] fn response_end_after_short_success_returns_eof() { let dsts = [ @@ -1102,6 +1152,39 @@ mod tests { )); } + #[test] + fn out_of_order_acks_yield_arrival_order_and_end_at_exact_count() { + let dsts = [ + PageRef { + page_idx: 0, + offset: 0, + len: 4096, + }, + PageRef { + page_idx: 1, + offset: 0, + len: 4096, + }, + PageRef { + page_idx: 2, + offset: 0, + len: 4096, + }, + ]; + let mut stream = active_stream(&dsts, vec![false; 3]); + push_page_ack(&mut stream, 2); + push_page_landed(&mut stream, 0); + push_page_ack(&mut stream, 1); + + for page_idx in [2, 0, 1] { + assert!(matches!( + poll_once(&mut stream), + Poll::Ready(Some(Ok(PageRef { page_idx: actual, .. }))) if actual == page_idx + )); + } + assert!(matches!(poll_once(&mut stream), Poll::Ready(None))); + } + #[test] fn page_landed_immediate_yields_page_and_ends_stream() { // The write-with-immediate path delivers page ordinals through From ac0db6eef5f539187a7b83aa6492a6fd176f69a7 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 01:16:06 +0000 Subject: [PATCH 34/56] unbounded-storage: scope storage ring installation --- cmd/unbounded-storage/src/ring/mod.rs | 6 +- cmd/unbounded-storage/src/ring/registry.rs | 61 ++++++++++++++++--- .../src/storage/blockdev/core_local.rs | 2 +- .../src/storage/blockdev/tests.rs | 22 ++++--- .../src/storage/blockdev/uring.rs | 2 +- .../src/storage/disks/uring.rs | 15 ++--- 6 files changed, 77 insertions(+), 31 deletions(-) diff --git a/cmd/unbounded-storage/src/ring/mod.rs b/cmd/unbounded-storage/src/ring/mod.rs index 2c39e962e..a902885b3 100644 --- a/cmd/unbounded-storage/src/ring/mod.rs +++ b/cmd/unbounded-storage/src/ring/mod.rs @@ -5,7 +5,7 @@ //! //! [`RingCore`](core::RingCore) is the single low-level engine; the two //! facades over it are [`StorageRing`] (block read/write, IOPOLL-capable) -//! and [`NetworkRing`] (sockets, no IOPOLL). [`set_current_storage_ring`] +//! and [`NetworkRing`] (sockets, no IOPOLL). [`install_current_storage_ring`] //! and friends expose a thread-local registry of the current shard's //! [`StorageRing`]. @@ -21,7 +21,7 @@ pub use network::{ TLS_RECORD_TYPE_APPLICATION_DATA, TLS_RECORD_TYPE_HANDSHAKE, }; pub use registry::{ - clear_current_storage_ring, current_storage_ring, set_current_storage_ring, - with_current_storage_ring, + CurrentStorageRingGuard, clear_current_storage_ring, current_storage_ring, + install_current_storage_ring, with_current_storage_ring, }; pub use storage::{StorageRing, StorageRingConfig}; diff --git a/cmd/unbounded-storage/src/ring/registry.rs b/cmd/unbounded-storage/src/ring/registry.rs index f7ad58cc7..6d9b179c6 100644 --- a/cmd/unbounded-storage/src/ring/registry.rs +++ b/cmd/unbounded-storage/src/ring/registry.rs @@ -17,10 +17,25 @@ thread_local! { static CURRENT: RefCell>> = const { RefCell::new(None) }; } -/// Install `ring` as the current thread's storage ring, replacing any -/// previous one. -pub fn set_current_storage_ring(ring: Rc) { - CURRENT.with(|c| *c.borrow_mut() = Some(ring)); +/// Restores the previous thread-local storage ring when an installation +/// scope ends. +pub struct CurrentStorageRingGuard { + previous: Option>, +} + +impl Drop for CurrentStorageRingGuard { + fn drop(&mut self) { + CURRENT.with(|current| { + *current.borrow_mut() = self.previous.take(); + }); + } +} + +/// Install `ring` as the current thread's storage ring until the returned +/// guard is dropped. +pub fn install_current_storage_ring(ring: Rc) -> CurrentStorageRingGuard { + let previous = CURRENT.with(|current| current.borrow_mut().replace(ring)); + CurrentStorageRingGuard { previous } } /// Clear the current thread's storage ring, if any. @@ -49,17 +64,49 @@ mod tests { use super::*; #[test] - fn set_with_clear_round_trip() { + fn install_scope_restores_empty_registry() { // No ring installed initially. assert!(with_current_storage_ring(|r| r.queue_depth()).is_none()); let ring = Rc::new(StorageRing::new(StorageRingConfig::test_local()).expect("ring")); - set_current_storage_ring(Rc::clone(&ring)); + let guard = install_current_storage_ring(Rc::clone(&ring)); let qd = with_current_storage_ring(|r| r.queue_depth()); assert_eq!(qd, Some(StorageRingConfig::test_local().queue_depth)); - clear_current_storage_ring(); + drop(guard); assert!(with_current_storage_ring(|r| r.queue_depth()).is_none()); } + + #[test] + fn nested_install_restores_previous_ring() { + clear_current_storage_ring(); + let outer = Rc::new(StorageRing::new(StorageRingConfig::test_local()).expect("outer")); + let inner = Rc::new(StorageRing::new(StorageRingConfig::default()).expect("inner")); + let outer_guard = install_current_storage_ring(Rc::clone(&outer)); + + { + let _inner_guard = install_current_storage_ring(Rc::clone(&inner)); + let current = current_storage_ring().expect("inner installed"); + assert!(Rc::ptr_eq(¤t, &inner)); + } + + let current = current_storage_ring().expect("outer restored"); + assert!(Rc::ptr_eq(¤t, &outer)); + drop(outer_guard); + assert!(current_storage_ring().is_none()); + } + + #[test] + fn install_is_restored_during_unwind() { + clear_current_storage_ring(); + let result = std::panic::catch_unwind(|| { + let ring = Rc::new(StorageRing::new(StorageRingConfig::test_local()).expect("ring")); + let _guard = install_current_storage_ring(ring); + panic!("test unwind"); + }); + + assert!(result.is_err()); + assert!(current_storage_ring().is_none()); + } } diff --git a/cmd/unbounded-storage/src/storage/blockdev/core_local.rs b/cmd/unbounded-storage/src/storage/blockdev/core_local.rs index ef19bdc02..00d6f0906 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/core_local.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/core_local.rs @@ -11,7 +11,7 @@ //! the engine that holds it can be published to any thread, but every //! [`BlockDevice`] call only succeeds on the storage core whose ring //! has been installed (via -//! [`set_current_storage_ring`](crate::ring::set_current_storage_ring)). +//! [`install_current_storage_ring`](crate::ring::install_current_storage_ring)). //! Off-core calls fail with `Err(Io(ENXIO))` rather than touching a //! foreign ring. diff --git a/cmd/unbounded-storage/src/storage/blockdev/tests.rs b/cmd/unbounded-storage/src/storage/blockdev/tests.rs index 48e0d9357..ec2b3d6fe 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/tests.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/tests.rs @@ -127,7 +127,7 @@ mod uring_tests { use super::super::{BlockDevice, CoreLocalDevice, OpenDisk, UringDevice}; use crate::ring::{ - StorageRing, StorageRingConfig, clear_current_storage_ring, set_current_storage_ring, + CurrentStorageRingGuard, StorageRing, StorageRingConfig, install_current_storage_ring, }; use crate::runtime::noop_waker; use crate::storage::types::{Error, Lba}; @@ -167,21 +167,16 @@ mod uring_tests { /// /// A [`CoreLocalDevice`] resolves its ring from the thread-local /// registry, so a ring must be installed before any device I/O. On - /// drop the thread-local is cleared first (so a later test reusing + /// drop the thread-local is restored first (so a later test reusing /// this pooled thread never observes a stale ring), then `ring` /// drops - unregistering the disk fd - and finally `_file` closes /// it. struct Installed { + _ring_install: CurrentStorageRingGuard, ring: Rc, _file: std::fs::File, } - impl Drop for Installed { - fn drop(&mut self) { - clear_current_storage_ring(); - } - } - /// Open `path` through the production [`UringDevice::open`] path with /// the tmpfile-friendly `test_local` ring config (no IOPOLL, no /// O_DIRECT), install the returned ring, and hand back the @@ -191,8 +186,15 @@ mod uring_tests { UringDevice::open(path, StorageRingConfig::test_local(), false, page_size) .expect("open uring device"); let ring = Rc::new(ring); - set_current_storage_ring(ring.clone()); - (device, Installed { ring, _file: file }) + let ring_install = install_current_storage_ring(ring.clone()); + ( + device, + Installed { + _ring_install: ring_install, + ring, + _file: file, + }, + ) } /// Drive a single block-device future to completion, pumping the diff --git a/cmd/unbounded-storage/src/storage/blockdev/uring.rs b/cmd/unbounded-storage/src/storage/blockdev/uring.rs index 8ed49986b..16780a2a1 100644 --- a/cmd/unbounded-storage/src/storage/blockdev/uring.rs +++ b/cmd/unbounded-storage/src/storage/blockdev/uring.rs @@ -54,7 +54,7 @@ impl UringDevice { /// capacity probe, file registration. The returned [`OpenDisk`] /// hands the ring back un-wrapped; the caller is responsible for /// installing it into the thread-local registry - /// ([`set_current_storage_ring`](crate::ring::set_current_storage_ring)) + /// ([`install_current_storage_ring`](crate::ring::install_current_storage_ring)) /// on the storage-core thread, and for keeping [`OpenDisk::file`] /// alive for as long as the ring's registered file table addresses /// it. diff --git a/cmd/unbounded-storage/src/storage/disks/uring.rs b/cmd/unbounded-storage/src/storage/disks/uring.rs index b610af17d..d8dd8496a 100644 --- a/cmd/unbounded-storage/src/storage/disks/uring.rs +++ b/cmd/unbounded-storage/src/storage/disks/uring.rs @@ -15,7 +15,7 @@ //! registered `Fixed` index and disk geometry that resolves the ring //! from the thread-local registry at call time, //! 2. install the ring into the registry via -//! [`set_current_storage_ring`] so the device (and the engine built +//! [`install_current_storage_ring`] so the device (and the engine built //! on it) reach the ring without it being threaded through every //! signature, //! 3. drive [`StorageEngine::open`] interleaved with ring progress so @@ -30,7 +30,7 @@ //! sets the stop flag and joins. Once shutdown is observed the thread //! stops admitting new page ops from the channel, calls //! [`StorageEngine::close_mutator`], fails any in-flight or queued page -//! ops, waits for in-flight ring I/O to drain, clears the thread-local +//! ops, waits for in-flight ring I/O to drain, restores the thread-local //! ring, and drops the ring on its own stack. Not admitting new work //! after the mutator is closing is what keeps the join from hanging //! (see [`run_core_loop`]). @@ -56,7 +56,7 @@ use std::thread::{self, JoinHandle}; use std::time::Duration; use crate::config::schema::DiskSpec; -use crate::ring::{StorageRingConfig, clear_current_storage_ring, set_current_storage_ring}; +use crate::ring::{StorageRingConfig, install_current_storage_ring}; use crate::runtime::{PinnedRuntime, WorkerSpec, noop_waker}; use crate::storage::blockdev::{ BlockDevice, CoreLocalDevice, OpenDisk, UringDevice, provision_file, @@ -217,7 +217,7 @@ fn run_storage_core( // This MUST run on this pinned storage-core thread: the device // resolves its ring from the thread-local registry, so installing it // anywhere else would strand every off-thread I/O with ENXIO. - set_current_storage_ring(ring.clone()); + let ring_install = install_current_storage_ring(ring.clone()); let waker = noop_waker(); let mut cx = Context::from_waker(&waker); @@ -230,19 +230,16 @@ fn run_storage_core( Poll::Ready(Ok(eng)) => break Arc::new(eng), Poll::Ready(Err(e)) => { let _ = ready_tx.send(Err(format!("engine open: {e}"))); - clear_current_storage_ring(); return; } Poll::Pending => {} } if let Err(e) = ring.progress() { let _ = ready_tx.send(Err(format!("ring progress during open: {e}"))); - clear_current_storage_ring(); return; } if stop.load(Ordering::Acquire) { let _ = ready_tx.send(Err("shutdown requested during open".into())); - clear_current_storage_ring(); return; } thread::sleep(Duration::from_micros(100)); @@ -275,10 +272,10 @@ fn run_storage_core( ); // Phase 6: tear down. Drop the mutator future first so any borrow - // of the engine is released, then clear the thread-local ring. + // of the engine is released, then restore the thread-local ring. drop(mutator_fut); drop(service); - clear_current_storage_ring(); + drop(ring_install); } /// The progress surface of a storage ring the core loop drives: push From 5186a1021a78d0384de5ba6dab327c87692e27df Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 01:09:05 +0000 Subject: [PATCH 35/56] unbounded-storage: simplify completion state --- .../src/fabric/completion.rs | 106 ++++++------------ cmd/unbounded-storage/src/fabric/progress.rs | 4 +- cmd/unbounded-storage/src/fabric/recvpool.rs | 2 +- cmd/unbounded-storage/src/fabric/rpc.rs | 4 +- cmd/unbounded-storage/src/fabric/sendpool.rs | 4 +- 5 files changed, 39 insertions(+), 81 deletions(-) diff --git a/cmd/unbounded-storage/src/fabric/completion.rs b/cmd/unbounded-storage/src/fabric/completion.rs index b082748ae..e28a0ca59 100644 --- a/cmd/unbounded-storage/src/fabric/completion.rs +++ b/cmd/unbounded-storage/src/fabric/completion.rs @@ -24,61 +24,12 @@ use std::future::Future; use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use super::error::{FabricError, Result}; -const WAKER_NONE: u8 = 0; -const WAKER_REGISTERED: u8 = 1; -const WAKER_WOKEN: u8 = 3; - -/// Small register-store-wake state machine. One-shot semantics: a -/// single completer wakes a single awaiter exactly once. -struct AtomicWaker { - state: AtomicU8, - waker: Mutex>, -} - -impl AtomicWaker { - fn new() -> Self { - Self { - state: AtomicU8::new(WAKER_NONE), - waker: Mutex::new(None), - } - } - - fn register(&self, w: &Waker) { - if self.state.load(Ordering::Acquire) == WAKER_WOKEN { - w.wake_by_ref(); - return; - } - if let Ok(mut slot) = self.waker.lock() { - *slot = Some(w.clone()); - } - let prev = self.state.swap(WAKER_REGISTERED, Ordering::AcqRel); - if prev == WAKER_WOKEN { - if let Ok(mut slot) = self.waker.lock() { - if let Some(w) = slot.take() { - w.wake(); - } - } - } - } - - fn wake(&self) { - let prev = self.state.swap(WAKER_WOKEN, Ordering::AcqRel); - if prev == WAKER_REGISTERED { - if let Ok(mut slot) = self.waker.lock() { - if let Some(w) = slot.take() { - w.wake(); - } - } - } - } -} - /// Result a `CompletionSlot` carries back to its future. Per-op /// metadata is encoded in `bytes`, `flags`, `tag` and `src_addr` /// directly out of the `fi_cq_tagged_entry` and `fi_cq_readfrom`. @@ -94,16 +45,22 @@ pub struct CompletionInfo { pub data: u64, } +struct CompletionState { + result: Option>, + waker: Option, +} + pub(crate) struct SlotInner { - result: Mutex>>, - waker: AtomicWaker, + state: Mutex, } impl SlotInner { fn new() -> Self { Self { - result: Mutex::new(None), - waker: AtomicWaker::new(), + state: Mutex::new(CompletionState { + result: None, + waker: None, + }), } } } @@ -124,7 +81,7 @@ pub struct CompletionSlot { /// Optional handler invoked when the slot completes, before the /// future is woken. Used by the ping responder to re-post its /// recv and emit the pong without involving any future at all. - handler: Mutex) + Send>>>, + handler: Option) + Send>>, reservation_live: AtomicBool, } @@ -152,30 +109,31 @@ impl CompletionSlot { /// future. If a handler was installed via /// [`CompletionSlot::set_handler`] it is invoked after capacity is /// released and before the future is woken; the handler may not panic. - pub fn complete(&self, result: Result) { + pub fn complete(&mut self, result: Result) { // Completion handlers may allocate a replacement slot, notably // when a receive pool rearms at full registry capacity. self.release_reservation(); - let handler = self.handler.lock().ok().and_then(|mut h| h.take()); - if let Some(h) = handler { + if let Some(h) = self.handler.take() { h(&result); } - if let Ok(mut slot) = self.inner.result.lock() { - *slot = Some(result); + let waker = { + let mut state = self.inner.state.lock().expect("completion state mutex"); + state.result = Some(result); + state.waker.take() + }; + if let Some(waker) = waker { + waker.wake(); } - self.inner.waker.wake(); } /// Install a one-shot handler that fires when the slot completes. /// Used by self-managed slots (e.g. the ping responder) that do /// not have a future awaiting the outcome. - pub fn set_handler(&self, f: F) + pub fn set_handler(&mut self, f: F) where F: FnOnce(&Result) + Send + 'static, { - if let Ok(mut h) = self.handler.lock() { - *h = Some(Box::new(f)); - } + self.handler = Some(Box::new(f)); } fn release_reservation(&self) { @@ -205,11 +163,11 @@ pub struct CompletionFuture { impl Future for CompletionFuture { type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.inner.waker.register(cx.waker()); - let mut result = self.inner.result.lock().expect("completion slot mutex"); - if let Some(r) = result.take() { + let mut state = self.inner.state.lock().expect("completion state mutex"); + if let Some(r) = state.result.take() { Poll::Ready(r) } else { + state.waker = Some(cx.waker().clone()); Poll::Pending } } @@ -267,7 +225,7 @@ impl CompletionRegistry { let slot = Box::new(CompletionSlot { inner: inner.clone(), registry: self.shared.clone(), - handler: Mutex::new(None), + handler: None, reservation_live: AtomicBool::new(true), }); let fut = CompletionFuture { inner }; @@ -311,7 +269,7 @@ mod tests { let raw = slot.into_raw(); // SAFETY: raw was just produced by `into_raw`. - let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + let mut reclaimed = unsafe { CompletionSlot::from_raw(raw) }; reclaimed.complete(Ok(CompletionInfo { flags: 0xCAFE, bytes: 4096, @@ -338,7 +296,7 @@ mod tests { let (slot, mut fut) = reg.allocate().unwrap(); let raw = slot.into_raw(); // SAFETY: raw was just produced by `into_raw`. - let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + let mut reclaimed = unsafe { CompletionSlot::from_raw(raw) }; reclaimed.complete(Err(FabricError::Cq { prov_errno: -3, err: -5, @@ -394,7 +352,7 @@ mod tests { #[test] fn completion_handler_can_replace_slot_at_full_capacity() { let reg = CompletionRegistry::new(1); - let (slot, _fut) = reg.allocate().unwrap(); + let (mut slot, _fut) = reg.allocate().unwrap(); let replacement = Arc::new(Mutex::new(None)); let handler_reg = Arc::clone(®); @@ -408,7 +366,7 @@ mod tests { let raw = slot.into_raw(); // SAFETY: raw was just produced by `into_raw`. - let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + let mut reclaimed = unsafe { CompletionSlot::from_raw(raw) }; reclaimed.complete(Ok(CompletionInfo { flags: 0, bytes: 0, @@ -431,7 +389,7 @@ mod tests { let (slot, mut fut) = reg.allocate().unwrap(); let raw = slot.into_raw(); // SAFETY: raw was just produced by `into_raw`. - let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + let mut reclaimed = unsafe { CompletionSlot::from_raw(raw) }; reclaimed.complete(Ok(CompletionInfo { flags: 0, bytes: 0, diff --git a/cmd/unbounded-storage/src/fabric/progress.rs b/cmd/unbounded-storage/src/fabric/progress.rs index ca4a711b7..7f936da39 100644 --- a/cmd/unbounded-storage/src/fabric/progress.rs +++ b/cmd/unbounded-storage/src/fabric/progress.rs @@ -322,7 +322,7 @@ fn deliver_success(entry: &ffi::fi_cq_data_entry) { // `CompletionSlot::into_raw` at submission; libfabric returns it // untouched on completion. Under MSG there is no tag or source // address, so those fields are zero. - let slot = unsafe { CompletionSlot::from_raw(ctx) }; + let mut slot = unsafe { CompletionSlot::from_raw(ctx) }; slot.complete(Ok(CompletionInfo { flags: entry.flags, bytes: entry.len, @@ -348,7 +348,7 @@ fn drain_errors(cq: *mut ffi::fid_cq, errors: &AtomicU64) { let ctx = err_entry.op_context; if !ctx.is_null() { // SAFETY: same provenance contract as `deliver_success`. - let slot = unsafe { CompletionSlot::from_raw(ctx) }; + let mut slot = unsafe { CompletionSlot::from_raw(ctx) }; slot.complete(Err(FabricError::Cq { prov_errno: err_entry.prov_errno, err: err_entry.err, diff --git a/cmd/unbounded-storage/src/fabric/recvpool.rs b/cmd/unbounded-storage/src/fabric/recvpool.rs index 15bea6132..9a5e4ab30 100644 --- a/cmd/unbounded-storage/src/fabric/recvpool.rs +++ b/cmd/unbounded-storage/src/fabric/recvpool.rs @@ -155,7 +155,7 @@ impl RecvPoolShared { fn post_buffer( self: &Arc, - slot: Box, + mut slot: Box, mut recv: RecvBuffer, ) -> Result { let buf_ptr = recv.ptr(); diff --git a/cmd/unbounded-storage/src/fabric/rpc.rs b/cmd/unbounded-storage/src/fabric/rpc.rs index 8f848c373..22640734e 100644 --- a/cmd/unbounded-storage/src/fabric/rpc.rs +++ b/cmd/unbounded-storage/src/fabric/rpc.rs @@ -1206,7 +1206,7 @@ fn wait_for_small_send( /// Block the calling worker thread on a libfabric completion, parking /// (not spinning) between polls. The progress-thread completion path -/// (`CompletionSlot::complete` -> `AtomicWaker::wake`) unparks us, so +/// (`CompletionSlot::complete` wakes the registered task) unparks us, so /// the wait resolves as soon as the CQE is reaped. Returns /// `FabricError::Timeout` if `timeout` elapses first, or as soon as /// `shutdown` is set so a draining server does not block joining a @@ -1266,7 +1266,7 @@ mod tests { let (slot, fut) = reg.allocate().unwrap(); let raw = slot.into_raw(); // SAFETY: raw was just produced by `into_raw`. - let reclaimed = unsafe { CompletionSlot::from_raw(raw) }; + let mut reclaimed = unsafe { CompletionSlot::from_raw(raw) }; reclaimed.complete(result); fut } diff --git a/cmd/unbounded-storage/src/fabric/sendpool.rs b/cmd/unbounded-storage/src/fabric/sendpool.rs index c0058a92e..f53c945e1 100644 --- a/cmd/unbounded-storage/src/fabric/sendpool.rs +++ b/cmd/unbounded-storage/src/fabric/sendpool.rs @@ -176,7 +176,7 @@ impl SendPool { // do not overlap. unsafe { ptr::copy_nonoverlapping(framed.as_ptr(), slot_ptr, len) }; - let (slot, fut) = match self.completions.allocate() { + let (mut slot, fut) = match self.completions.allocate() { Ok(pair) => pair, Err(e) => { slab.free.lock().unwrap().push(idx); @@ -217,7 +217,7 @@ impl SendPool { .map(|m| m.desc()) .unwrap_or(ptr::null_mut()); - let (slot, fut) = match self.completions.allocate() { + let (mut slot, fut) = match self.completions.allocate() { Ok(pair) => pair, Err(e) => { // SAFETY: `buf_addr`/`len` came from `Box::into_raw` on a From 9a6840ba0c945bbfe56c3f68aeeca6d8dcfa264c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 01:17:17 +0000 Subject: [PATCH 36/56] unbounded-storage: simplify flag wakers --- cmd/unbounded-storage/src/fanout/channel.rs | 34 +---------- cmd/unbounded-storage/src/runtime/executor.rs | 24 ++++++++ cmd/unbounded-storage/src/runtime/mod.rs | 1 + cmd/unbounded-storage/src/runtime/shard.rs | 56 +------------------ .../src/storage/page_channel.rs | 44 +-------------- 5 files changed, 31 insertions(+), 128 deletions(-) diff --git a/cmd/unbounded-storage/src/fanout/channel.rs b/cmd/unbounded-storage/src/fanout/channel.rs index 4f4099e29..de74363df 100644 --- a/cmd/unbounded-storage/src/fanout/channel.rs +++ b/cmd/unbounded-storage/src/fanout/channel.rs @@ -535,11 +535,8 @@ impl Future for AliveAwareWait { #[cfg(test)] mod tests { - use std::sync::atomic::AtomicBool; - use std::task::{RawWaker, RawWakerVTable, Waker}; - use super::*; - use crate::runtime::noop_waker; + use crate::runtime::{flag_waker, noop_waker}; fn block_on(mut fut: F) -> F::Output { let waker = noop_waker(); @@ -558,32 +555,6 @@ mod tests { } } - // A waker that flips an AtomicBool when woken, so we can assert the - // event slot actually wakes a stored waker. - fn flag_waker(flag: Arc) -> Waker { - fn clone(p: *const ()) -> RawWaker { - let arc = unsafe { Arc::from_raw(p as *const AtomicBool) }; - let cloned = arc.clone(); - std::mem::forget(arc); - RawWaker::new(Arc::into_raw(cloned) as *const (), &VTABLE) - } - fn wake(p: *const ()) { - let arc = unsafe { Arc::from_raw(p as *const AtomicBool) }; - arc.store(true, Ordering::SeqCst); - } - fn wake_by_ref(p: *const ()) { - let arc = unsafe { Arc::from_raw(p as *const AtomicBool) }; - arc.store(true, Ordering::SeqCst); - std::mem::forget(arc); - } - fn drop_fn(p: *const ()) { - unsafe { drop(Arc::from_raw(p as *const AtomicBool)) }; - } - static VTABLE: RawWakerVTable = RawWakerVTable::new(clone, wake, wake_by_ref, drop_fn); - let raw = RawWaker::new(Arc::into_raw(flag) as *const (), &VTABLE); - unsafe { Waker::from_raw(raw) } - } - #[test] fn fetch_round_trip_yields_page_events() { let (tx, rx) = FetchChannel::new(); @@ -634,8 +605,7 @@ mod tests { #[test] fn push_wakes_stored_waker() { let slot = EventSlot::new(); - let flag = Arc::new(AtomicBool::new(false)); - let waker = flag_waker(flag.clone()); + let (waker, flag) = flag_waker(); let mut cx = Context::from_waker(&waker); let mut wait = EventWait { diff --git a/cmd/unbounded-storage/src/runtime/executor.rs b/cmd/unbounded-storage/src/runtime/executor.rs index dcdbe3c77..d2e19e15e 100644 --- a/cmd/unbounded-storage/src/runtime/executor.rs +++ b/cmd/unbounded-storage/src/runtime/executor.rs @@ -37,6 +37,7 @@ use std::future::Future; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::task::{Context, Poll, RawWaker, RawWakerVTable, Wake, Waker}; use std::thread::{self, Thread}; use std::time::{Duration, Instant}; @@ -55,6 +56,29 @@ pub fn noop_waker() -> Waker { unsafe { Waker::from_raw(raw()) } } +struct FlagWaker { + flag: Arc, +} + +impl Wake for FlagWaker { + fn wake(self: Arc) { + self.flag.store(true, Ordering::Release); + } + + fn wake_by_ref(self: &Arc) { + self.flag.store(true, Ordering::Release); + } +} + +/// A [`Waker`] whose wake methods set a shared flag to `true`. +pub(crate) fn flag_waker() -> (Waker, Arc) { + let flag = Arc::new(AtomicBool::new(false)); + let waker = Waker::from(Arc::new(FlagWaker { + flag: Arc::clone(&flag), + })); + (waker, flag) +} + /// Drive `fut` to completion on the current thread with a [`noop_waker`], /// running `idle` on every `Poll::Pending`. /// diff --git a/cmd/unbounded-storage/src/runtime/mod.rs b/cmd/unbounded-storage/src/runtime/mod.rs index 94e193495..a6203953a 100644 --- a/cmd/unbounded-storage/src/runtime/mod.rs +++ b/cmd/unbounded-storage/src/runtime/mod.rs @@ -12,6 +12,7 @@ mod shard; #[cfg(target_os = "linux")] pub use pinned::{PinnedRuntime, WorkerSpec, set_preferred_node}; +pub(crate) use executor::flag_waker; pub use executor::{block_on_cooperative, noop_waker, park_block_on_until, thread_waker}; pub use shard::ShardLoop; diff --git a/cmd/unbounded-storage/src/runtime/shard.rs b/cmd/unbounded-storage/src/runtime/shard.rs index 5ec70b57b..a7593c4be 100644 --- a/cmd/unbounded-storage/src/runtime/shard.rs +++ b/cmd/unbounded-storage/src/runtime/shard.rs @@ -41,10 +41,12 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker}; +use std::task::{Context, Poll, Waker}; use std::thread; use std::time::Duration; +use super::flag_waker; + /// Default *idle* sleep between [`ShardLoop`] iterations once nothing /// made progress. Matches the shard's shutdown poll interval in /// `main.rs` (`SHUTDOWN_POLL`). When an iteration is busy the loop does @@ -211,58 +213,6 @@ impl Default for ShardLoop { } } -/// Build a `Waker` whose `wake` flips a shared `Arc` to -/// `true`, returning the waker and a clone of that flag. The shard -/// polls its futures with this waker so a cross-thread -/// [`ReplySlot::set`](crate::storage::ReplySlot) -> `Waker::wake` on -/// the storage core is observable on the shard thread (under a noop -/// waker it would be a silent no-op). `clone`/`wake`/`drop` follow the -/// standard `Arc` refcount discipline so the flag outlives every waker -/// clone the futures may stash. -fn flag_waker() -> (Waker, Arc) { - let flag = Arc::new(AtomicBool::new(false)); - let data = Arc::into_raw(flag.clone()) as *const (); - // SAFETY: `data` is a freshly leaked `Arc` pointer and - // `FLAG_VTABLE` upholds the matching clone/wake/drop refcounting. - let waker = unsafe { Waker::from_raw(RawWaker::new(data, &FLAG_VTABLE)) }; - (waker, flag) -} - -static FLAG_VTABLE: RawWakerVTable = - RawWakerVTable::new(flag_clone, flag_wake, flag_wake_by_ref, flag_drop); - -unsafe fn flag_clone(data: *const ()) -> RawWaker { - // SAFETY: `data` points at a live `Arc`; bumping the - // strong count balances the `drop` the cloned waker will perform. - unsafe { Arc::increment_strong_count(data as *const AtomicBool) }; - RawWaker::new(data, &FLAG_VTABLE) -} - -unsafe fn flag_wake(data: *const ()) { - // `wake` consumes the waker: reclaim its strong ref, flip the - // flag, then drop the ref as `arc` falls out of scope. - // SAFETY: `data` came from `Arc::into_raw` and this consumes that - // owned ref exactly once. - let arc = unsafe { Arc::from_raw(data as *const AtomicBool) }; - arc.store(true, Ordering::Release); -} - -unsafe fn flag_wake_by_ref(data: *const ()) { - // `wake_by_ref` does not consume the waker: borrow the flag - // without taking ownership of the strong ref. - // SAFETY: `data` points at a live `Arc`; we hand the - // reclaimed ref straight back via `into_raw` so the count is - // unchanged. - let arc = unsafe { Arc::from_raw(data as *const AtomicBool) }; - arc.store(true, Ordering::Release); - let _ = Arc::into_raw(arc); -} - -unsafe fn flag_drop(data: *const ()) { - // SAFETY: balances one `clone`/`into_raw` strong ref. - unsafe { Arc::decrement_strong_count(data as *const AtomicBool) }; -} - #[cfg(test)] mod tests { use super::*; diff --git a/cmd/unbounded-storage/src/storage/page_channel.rs b/cmd/unbounded-storage/src/storage/page_channel.rs index 9c0bc6356..870788437 100644 --- a/cmd/unbounded-storage/src/storage/page_channel.rs +++ b/cmd/unbounded-storage/src/storage/page_channel.rs @@ -534,10 +534,10 @@ fn spin_block_on_with_alive(fut: ReplyWait, service_alive: &AtomicBool) -> #[cfg(test)] mod tests { use super::*; + use crate::runtime::flag_waker; use crate::storage::blockdev::{MockDevice, MockDeviceConfig}; use crate::storage::engine::EngineConfig; use std::sync::mpsc::channel as std_channel; - use std::task::{RawWaker, RawWakerVTable}; use std::time::Duration; fn block_on(mut fut: F) -> F::Output { @@ -555,48 +555,6 @@ mod tests { panic!("block_on: future did not complete within spin budget"); } - /// Build a waker whose `wake` flips a shared `Arc`, - /// mirroring the shard loop's `flag_waker`. The shard polls - /// page-channel reply futures with exactly this kind of waker, so - /// the `ReplySlot` stashes it and a cross-thread `set` must flip - /// the flag. This local copy lets the page-channel tests pin that - /// contract from this side of the boundary. - fn flag_waker() -> (Waker, Arc) { - let flag = Arc::new(AtomicBool::new(false)); - let data = Arc::into_raw(flag.clone()) as *const (); - // SAFETY: `data` is a freshly leaked `Arc` and the - // vtable upholds the matching clone/wake/drop refcounting. - let waker = unsafe { Waker::from_raw(RawWaker::new(data, &FLAG_VTABLE)) }; - (waker, flag) - } - - static FLAG_VTABLE: RawWakerVTable = - RawWakerVTable::new(flag_clone, flag_wake, flag_wake_by_ref, flag_drop); - - unsafe fn flag_clone(data: *const ()) -> RawWaker { - // SAFETY: `data` points at a live `Arc`. - unsafe { Arc::increment_strong_count(data as *const AtomicBool) }; - RawWaker::new(data, &FLAG_VTABLE) - } - - unsafe fn flag_wake(data: *const ()) { - // SAFETY: consumes the one owned ref this waker held. - let arc = unsafe { Arc::from_raw(data as *const AtomicBool) }; - arc.store(true, Ordering::Release); - } - - unsafe fn flag_wake_by_ref(data: *const ()) { - // SAFETY: borrows without consuming; the ref is handed back. - let arc = unsafe { Arc::from_raw(data as *const AtomicBool) }; - arc.store(true, Ordering::Release); - let _ = Arc::into_raw(arc); - } - - unsafe fn flag_drop(data: *const ()) { - // SAFETY: balances one clone/into_raw strong ref. - unsafe { Arc::decrement_strong_count(data as *const AtomicBool) }; - } - /// Run a storage-core-like loop on the current thread: build a /// MockDevice engine, hand a [`PageChannel`] back, then drive /// [`PageService::poll_once`] and the engine mutator until the From 04d5e830797296075978e79b65a232c9efc0150e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 19:09:06 +0000 Subject: [PATCH 37/56] unbounded-storage: repost receives before dispatch --- .../src/fabric/completion.rs | 4 +++ cmd/unbounded-storage/src/fabric/recvpool.rs | 34 +++++++++++++------ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/cmd/unbounded-storage/src/fabric/completion.rs b/cmd/unbounded-storage/src/fabric/completion.rs index e28a0ca59..49a962011 100644 --- a/cmd/unbounded-storage/src/fabric/completion.rs +++ b/cmd/unbounded-storage/src/fabric/completion.rs @@ -354,14 +354,17 @@ mod tests { let reg = CompletionRegistry::new(1); let (mut slot, _fut) = reg.allocate().unwrap(); let replacement = Arc::new(Mutex::new(None)); + let dispatch_was_blocked = Arc::new(AtomicBool::new(false)); let handler_reg = Arc::clone(®); let handler_replacement = Arc::clone(&replacement); + let handler_dispatch_was_blocked = Arc::clone(&dispatch_was_blocked); slot.set_handler(move |_| { let (slot, _fut) = handler_reg .allocate() .expect("completed slot releases capacity before handler"); *handler_replacement.lock().unwrap() = Some(slot); + handler_dispatch_was_blocked.store(handler_reg.allocate().is_err(), Ordering::Release); }); let raw = slot.into_raw(); @@ -377,6 +380,7 @@ mod tests { })); assert_eq!(reg.live_count(), 1); + assert!(dispatch_was_blocked.load(Ordering::Acquire)); drop(reclaimed); assert_eq!(reg.live_count(), 1); drop(replacement.lock().unwrap().take()); diff --git a/cmd/unbounded-storage/src/fabric/recvpool.rs b/cmd/unbounded-storage/src/fabric/recvpool.rs index 9a5e4ab30..ab03dd5e3 100644 --- a/cmd/unbounded-storage/src/fabric/recvpool.rs +++ b/cmd/unbounded-storage/src/fabric/recvpool.rs @@ -168,18 +168,28 @@ impl RecvPoolShared { // this recv buffer but writes no frame into it; the // page ordinal rides in the CQ immediate data. Route it // by the packed handle instead of parsing a header. - if info.flags & ffi::FI_REMOTE_CQ_DATA != 0 { - shared.handle_remote_write(info.data); + let remote_write = if info.flags & ffi::FI_REMOTE_CQ_DATA != 0 { + Some(info.data) } else { - shared.handle_message(&recv.bytes[..], info.bytes); - } + None + }; + let message = if remote_write.is_none() { + shared.decode_message(&recv.bytes[..], info.bytes) + } else { + None + }; // Re-arm the stable buffer and local MR on a fresh - // completion slot. A failure or transient EAGAIN here - // shrinks the pool by one; record it but do not unwind - // on the progress thread. + // completion slot before dispatching. Dispatch may send an + // overload reply from this thread and must not consume the + // completion capacity reserved for the receive lane. if !matches!(shared.repost(recv), Ok(true)) { shared.errors.fetch_add(1, Ordering::Relaxed); } + if let Some(data) = remote_write { + shared.handle_remote_write(data); + } else if let Some((header, body)) = message { + shared.handle_message(header, body); + } } Err(_) => { // Cancellation on endpoint close, or a genuine recv @@ -221,19 +231,23 @@ impl RecvPoolShared { /// or truncated frames are dropped (counted as errors) rather than /// propagated, since there is no caller to surface them to on the /// progress thread. - fn handle_message(self: &Arc, buf: &[u8], n: usize) { + fn decode_message(&self, buf: &[u8], n: usize) -> Option<(MsgHeader, Vec)> { if n < MSG_HEADER_LEN { self.errors.fetch_add(1, Ordering::Relaxed); - return; + return None; } let header = match MsgHeader::read_from(&buf[..n]) { Ok(h) => h, Err(_) => { self.errors.fetch_add(1, Ordering::Relaxed); - return; + return None; } }; let body = buf[MSG_HEADER_LEN..n].to_vec(); + Some((header, body)) + } + + fn handle_message(self: &Arc, header: MsgHeader, body: Vec) { let reply = ReplyCtx::new(self.ep, self.peer); self.dispatch .route(header.kind, header.request_id, &reply, body); From 5425450e8f3ea6d916513b549d9850ea5d985384 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 06:14:56 +0000 Subject: [PATCH 38/56] unbounded-storage: share storage completion state --- .../src/storage/completion.rs | 126 ++++++++++++++++++ cmd/unbounded-storage/src/storage/mod.rs | 1 + cmd/unbounded-storage/src/storage/mutator.rs | 63 +-------- .../src/storage/page_channel.rs | 67 +--------- 4 files changed, 138 insertions(+), 119 deletions(-) create mode 100644 cmd/unbounded-storage/src/storage/completion.rs diff --git a/cmd/unbounded-storage/src/storage/completion.rs b/cmd/unbounded-storage/src/storage/completion.rs new file mode 100644 index 000000000..3981eac7c --- /dev/null +++ b/cmd/unbounded-storage/src/storage/completion.rs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Generic single-consumer completion state for storage operations. + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; + +pub struct Completion { + inner: Mutex>, +} + +struct CompletionState { + value: Option, + waker: Option, +} + +impl Completion { + pub fn new() -> Arc { + Arc::new(Self { + inner: Mutex::new(CompletionState { + value: None, + waker: None, + }), + }) + } + + pub fn set(&self, value: T) { + let waker = { + let mut state = self.inner.lock().unwrap(); + state.value = Some(value); + state.waker.take() + }; + if let Some(waker) = waker { + waker.wake(); + } + } + + pub(super) fn wait(self: Arc) -> CompletionWait { + CompletionWait { completion: self } + } +} + +pub(super) struct CompletionWait { + completion: Arc>, +} + +impl Future for CompletionWait { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut state = self.completion.inner.lock().unwrap(); + if let Some(value) = state.value.take() { + Poll::Ready(value) + } else { + if !state + .waker + .as_ref() + .is_some_and(|waker| waker.will_wake(cx.waker())) + { + state.waker = Some(cx.waker().clone()); + } + Poll::Pending + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::{Wake, Waker}; + + use super::*; + + struct FlagWake(AtomicBool); + + impl Wake for FlagWake { + fn wake(self: Arc) { + self.0.store(true, Ordering::Relaxed); + } + } + + fn poll(wait: &mut CompletionWait, waker: &Waker) -> Poll { + let mut cx = Context::from_waker(waker); + Pin::new(wait).poll(&mut cx) + } + + #[test] + fn set_before_wait_returns_value() { + let completion = Completion::new(); + completion.set(7); + let flag = Arc::new(FlagWake(AtomicBool::new(false))); + assert_eq!( + poll(&mut completion.wait(), &Waker::from(flag)), + Poll::Ready(7) + ); + } + + #[test] + fn set_wakes_waiter() { + let completion = Completion::new(); + let flag = Arc::new(FlagWake(AtomicBool::new(false))); + let waker = Waker::from(flag.clone()); + let mut wait = completion.clone().wait(); + assert!(poll(&mut wait, &waker).is_pending()); + + completion.set(9); + assert!(flag.0.load(Ordering::Relaxed)); + assert_eq!(poll(&mut wait, &waker), Poll::Ready(9)); + } + + #[test] + fn completion_can_cross_threads() { + let completion = Completion::new(); + let producer = completion.clone(); + std::thread::spawn(move || producer.set(11)).join().unwrap(); + let flag = Arc::new(FlagWake(AtomicBool::new(false))); + assert_eq!( + poll(&mut completion.wait(), &Waker::from(flag)), + Poll::Ready(11) + ); + } +} diff --git a/cmd/unbounded-storage/src/storage/mod.rs b/cmd/unbounded-storage/src/storage/mod.rs index 4d10ceae7..afdea4289 100644 --- a/cmd/unbounded-storage/src/storage/mod.rs +++ b/cmd/unbounded-storage/src/storage/mod.rs @@ -22,6 +22,7 @@ pub mod types; mod alloc; mod btree; mod checksum; +mod completion; mod engine; mod local; mod lru; diff --git a/cmd/unbounded-storage/src/storage/mutator.rs b/cmd/unbounded-storage/src/storage/mutator.rs index ed22a6ed0..114e98cca 100644 --- a/cmd/unbounded-storage/src/storage/mutator.rs +++ b/cmd/unbounded-storage/src/storage/mutator.rs @@ -25,6 +25,7 @@ use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use crate::storage::btree::LeafEntry; +use crate::storage::completion::{Completion, CompletionWait}; use crate::storage::types::PageKey; /// A single submission to the mutator. The submitter has already @@ -60,65 +61,9 @@ pub(crate) enum MutatorOutcome { Failed, } -/// Shared reply slot: filled by the mutator, awaited by the -/// submitter. Cheap to construct (`Arc`) so the engine can -/// allocate one per request. -pub(crate) struct MutatorReply { - inner: Mutex, -} - -struct ReplyInner { - result: Option, - waker: Option, -} - -impl MutatorReply { - pub(crate) fn new() -> Arc { - Arc::new(Self { - inner: Mutex::new(ReplyInner { - result: None, - waker: None, - }), - }) - } - - /// Called by the mutator once the batch this request was part - /// of has been processed. Wakes the submitter if it has - /// already polled. - pub(crate) fn set(&self, outcome: MutatorOutcome) { - let waker = { - let mut g = self.inner.lock().unwrap(); - g.result = Some(outcome); - g.waker.take() - }; - if let Some(w) = waker { - w.wake(); - } - } - - pub(crate) fn wait(self: Arc) -> ReplyWait { - ReplyWait { inner: self } - } -} - -pub(crate) struct ReplyWait { - inner: Arc, -} - -impl Future for ReplyWait { - type Output = MutatorOutcome; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut g = self.inner.inner.lock().unwrap(); - if let Some(o) = g.result.take() { - Poll::Ready(o) - } else { - if !g.waker.as_ref().is_some_and(|w| w.will_wake(cx.waker())) { - g.waker = Some(cx.waker().clone()); - } - Poll::Pending - } - } -} +/// Shared reply slot filled by the mutator and awaited by one submitter. +pub(crate) type MutatorReply = Completion; +pub(crate) type ReplyWait = CompletionWait; /// MPSC-style queue between writer/eviction tasks (producers) /// and the engine's `run_mutator` loop (single consumer). All diff --git a/cmd/unbounded-storage/src/storage/page_channel.rs b/cmd/unbounded-storage/src/storage/page_channel.rs index 870788437..440aaa2df 100644 --- a/cmd/unbounded-storage/src/storage/page_channel.rs +++ b/cmd/unbounded-storage/src/storage/page_channel.rs @@ -38,14 +38,15 @@ use std::future::Future; use std::pin::Pin; use std::ptr::NonNull; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll, Waker}; +use std::task::{Context, Poll}; use crate::bufferpool::{Error, StripeKey}; use crate::runtime::noop_waker; use crate::storage::blockdev::BlockDevice; +use crate::storage::completion::{Completion, CompletionWait}; use crate::storage::engine::StorageEngine; static NEXT_SERVICE_ID: AtomicU64 = AtomicU64::new(1); @@ -400,63 +401,9 @@ enum Inflight { }, } -/// One-shot completion slot generic over the success type. The -/// producer (service) stores a result and wakes any parked consumer; -/// the consumer polls and returns the stored result if present. -pub struct ReplySlot { - inner: Mutex>, -} - -struct ReplyInner { - result: Option>, - waker: Option, -} - -impl ReplySlot { - pub fn new() -> Arc { - Arc::new(Self { - inner: Mutex::new(ReplyInner { - result: None, - waker: None, - }), - }) - } - - pub fn set(&self, result: Result) { - let waker = { - let mut g = self.inner.lock().unwrap(); - g.result = Some(result); - g.waker.take() - }; - if let Some(w) = waker { - w.wake(); - } - } - - fn wait(self: Arc) -> ReplyWait { - ReplyWait { inner: self } - } -} - -struct ReplyWait { - inner: Arc>, -} - -impl Future for ReplyWait { - type Output = Result; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut g = self.inner.inner.lock().unwrap(); - if let Some(r) = g.result.take() { - Poll::Ready(r) - } else { - if !g.waker.as_ref().is_some_and(|w| w.will_wake(cx.waker())) { - g.waker = Some(cx.waker().clone()); - } - Poll::Pending - } - } -} +/// One-shot completion slot for page-service results. +pub type ReplySlot = Completion>; +type ReplyWait = CompletionWait>; /// `Future` adapter for the async paths (`read_page` / `write_page`). /// Resolves with `Err(Io(EIO))` if the slot is still pending once the @@ -470,7 +417,7 @@ struct AliveAwareWait { impl AliveAwareWait { fn new(reply: Arc>, service_alive: Arc) -> Self { Self { - reply: ReplyWait { inner: reply }, + reply: reply.wait(), service_alive, } } From 8ff4336dba84ad1ec167c38bd715a97fb4fd5e2c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 02:07:59 +0000 Subject: [PATCH 39/56] unbounded-storage: enforce shard memory budget --- cmd/unbounded-storage/ARCHITECTURE.md | 24 +++--- cmd/unbounded-storage/README.md | 9 ++- cmd/unbounded-storage/src/shard_layer.rs | 81 ++++++++++++++++--- .../02-configmap.yaml.tmpl | 3 +- hack/smoke-storage.py | 21 +++-- 5 files changed, 106 insertions(+), 32 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 4ea36191e..1947f88e6 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -99,7 +99,7 @@ dynamically reloadable cluster state and the startup-fixed settings - is read from the config file. The startup-fixed settings (collected into `StartupSettings`: the fabric -endpoint and thread pools, per-shard memory sizing, and CPU-topology +endpoint and thread pools, node-wide memory sizing, and CPU-topology selection) live in the config's `[startup]` section. They are read once at process start and cannot change without a restart, so they are excluded from the live-reload diff. @@ -107,8 +107,10 @@ excluded from the live-reload diff. - `[startup.memory]` - `no_hugepages` (allocate shard backings from the heap instead of 2 MiB hugepages) and `memory_total_bytes` (u64 bytes, no suffix; unset/null defaults to 128 MiB) - the total backing - pool, split evenly across the serving shards so the host footprint stays - fixed regardless of the auto-scaled shard count. + pool for shard data. The budget is floored to whole 2 MiB pages, requires + at least one page per serving shard, and distributes whole pages evenly + with any remainder assigned in worker order. Each fabric unit also allocates + a separate fixed eight-page RPC scratch backing outside this pool. - `[startup.fabric]` - one `binds` table (`tcp`, `rdma`, or `auto_rdma`), `progress_threads` (2), `progress_poll_us` (10), `rpc_worker_threads` (4), `max_inflight` (1024) - the fabric endpoint, thread pools, and in-flight @@ -182,8 +184,9 @@ per-shard `!Send` object graph: `Provider::Tcp` fallback on `lo`); build a `FabricConfig` via `fabric::defaults_for(device_name, runtime, widx)`; `Fabric::new` and resolve `self_address()`. -2. **Memory.** Allocate a NUMA-local `Backing` via `memory::allocate` **on the - pinned thread** (mempolicy keeps pages local; the hugepage variant `mbind`s). +2. **Memory.** Allocate the shard's assigned whole-page share of the node-wide + data pool as a NUMA-local `Backing` via `memory::allocate` **on the pinned + thread** (mempolicy keeps pages local; the hugepage variant `mbind`s). Register it with the fabric as a memory region (MR), and register it with the socket ring as a fixed buffer for zero-copy send/recv. 3. **Origin backend.** Resolve the origin URL and build an `HttpBackend` @@ -191,11 +194,12 @@ per-shard `!Send` object graph: 4. **Transport + blockstore + pool.** Build a `RoutedTransport` (Chord routing + fabric transport + origin backend), a `LiveShardLocalStore` over the disk channel directory as the `BlockStore`, then `Pool::new`. -5. **RPC server.** Allocate a **separate** scratch backing - (`RPC_SCRATCH_PAGES = 8`, one scratch page per in-flight serve/forward), - register it as its own fabric MR and its own `LiveShardLocalStore` (a - `PageRef` resolves through exactly one backing's geometry, so scratch needs a - distinct store). Build a `RecursiveHandler` and start the fabric RPC server. +5. **RPC server.** `FabricGroup`, constructed before the shards, allocates a + **separate** scratch backing for each fabric unit (`RPC_SCRATCH_PAGES = 8`, + one scratch page per in-flight serve/forward) and registers it as its own + fabric MR. After shard publication, it builds the unit's `RecursiveHandler` + and starts the RPC server. Scratch uses a distinct `LiveShardLocalStore` + because a `PageRef` resolves through exactly one backing's geometry. 6. **Frontends.** A shard hosts a `FrontendRegistry` of any number of frontends keyed by component name. Each spec binds its listener with `SO_REUSEPORT` and builds an `HttpDriver`/`S3Driver`; the registry can add and remove frontends diff --git a/cmd/unbounded-storage/README.md b/cmd/unbounded-storage/README.md index 029c13a97..90ef2da1e 100644 --- a/cmd/unbounded-storage/README.md +++ b/cmd/unbounded-storage/README.md @@ -159,9 +159,12 @@ source = "cache" # backend or cache component name. addr = "0.0.0.0:9000" [startup.memory] # startup-fixed; read once at process start. -no_hugepages = false # true allocates per-shard backing from the heap. -memory_total_bytes = 134217728 # u64 bytes (no K/M/G suffix). Total backing pool split - # evenly across serving shards. 0 -> 128 MiB. +no_hugepages = false # true allocates shard backings from the heap. +memory_total_bytes = 134217728 # u64 bytes (no K/M/G suffix). Node-wide data pool; + # partial 2 MiB pages are unused, then whole pages + # are split across serving shards. Unset -> 128 MiB; + # explicit 0 is invalid when shards are configured. + # RPC scratch adds 8 pages per fabric unit. [startup.fabric] progress_threads = 2 # libfabric progress threads per fabric unit. diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index ffcc73c09..d4353dc8e 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -33,6 +33,7 @@ use unbounded_storage::config::{ self, ApplyError, Config, ConfigApplyTarget, ConfigDiff, ShardControlGroup, }; use unbounded_storage::fabric::PeerId; +use unbounded_storage::memory::HUGEPAGE_2MB; use unbounded_storage::p2p::RouteTableHandle; use unbounded_storage::runtime::{JoinHandle, Threading, WorkerIdx}; use unbounded_storage::storage::StripeReq; @@ -117,15 +118,8 @@ pub fn spawn_shard_layer( let layer_stop = Arc::new(AtomicBool::new(false)); let worker_count = deps.workers.len(); let settings = &deps.settings; - // `memory_total_bytes` is the whole host backing budget; split it - // evenly across the serving shards so each gets a NUMA-local slice - // and the host footprint stays fixed regardless of the auto-scaled - // serving-shard count. - let bytes_per_shard = if worker_count == 0 { - 0 - } else { - settings.memory_total_bytes / worker_count - }; + let shard_backing_sizes = shard_backing_sizes(settings.memory_total_bytes, worker_count) + .map_err(|e| vec![format!("invalid shard memory budget: {e}")])?; let projection = config::runtime_projection(config) .map_err(|e| vec![format!("config projection failed: {e}")])?; let frontend_specs = Arc::new(config.frontends.clone()); @@ -169,6 +163,7 @@ pub fn spawn_shard_layer( for (i, (shard, _)) in deps.workers.iter().enumerate() { let widx = WorkerIdx(u16::try_from(i).expect("worker index fits in u16")); + let backing_size = shard_backing_sizes[i]; let (ctrl_tx, ctrl_rx) = mpsc::channel::(); control_senders.push((widx, ctrl_tx)); let (peer_tx, peer_rx) = mpsc::channel::>>(); @@ -200,7 +195,7 @@ pub fn spawn_shard_layer( fabric, tx, backing_kind, - bytes_per_shard, + backing_size, cache_directories, route_handle, frontend_specs, @@ -367,6 +362,28 @@ pub fn spawn_shard_layer( }) } +fn shard_backing_sizes( + memory_total_bytes: usize, + shard_count: usize, +) -> Result, String> { + if shard_count == 0 { + return Ok(Vec::new()); + } + + let total_pages = memory_total_bytes / HUGEPAGE_2MB; + if total_pages < shard_count { + return Err(format!( + "{memory_total_bytes} bytes provides {total_pages} whole 2 MiB pages for {shard_count} serving shards" + )); + } + + let pages_per_shard = total_pages / shard_count; + let remainder = total_pages % shard_count; + Ok((0..shard_count) + .map(|i| (pages_per_shard + usize::from(i < remainder)) * HUGEPAGE_2MB) + .collect()) +} + fn local_self_peer(projection: &config::RuntimeGraph) -> Result { Ok(projection.mesh.self_peer_id) } @@ -551,4 +568,48 @@ mod tests { assert_eq!(local_self_peer(&graph).unwrap(), PeerId(7)); } + + #[test] + fn shard_backing_sizes_split_evenly() { + assert_eq!( + shard_backing_sizes(4 * HUGEPAGE_2MB, 2).unwrap(), + vec![2 * HUGEPAGE_2MB, 2 * HUGEPAGE_2MB] + ); + } + + #[test] + fn shard_backing_sizes_assign_remainder_in_worker_order() { + assert_eq!( + shard_backing_sizes(5 * HUGEPAGE_2MB, 2).unwrap(), + vec![3 * HUGEPAGE_2MB, 2 * HUGEPAGE_2MB] + ); + } + + #[test] + fn shard_backing_sizes_ignore_partial_trailing_page() { + let total = 5 * HUGEPAGE_2MB + HUGEPAGE_2MB - 1; + let sizes = shard_backing_sizes(total, 2).unwrap(); + + assert_eq!(sizes, vec![3 * HUGEPAGE_2MB, 2 * HUGEPAGE_2MB]); + assert!(sizes.iter().sum::() <= total); + } + + #[test] + fn shard_backing_sizes_accept_exact_minimum() { + assert_eq!( + shard_backing_sizes(3 * HUGEPAGE_2MB, 3).unwrap(), + vec![HUGEPAGE_2MB; 3] + ); + } + + #[test] + fn shard_backing_sizes_reject_insufficient_or_zero_budget() { + assert!(shard_backing_sizes(2 * HUGEPAGE_2MB, 3).is_err()); + assert!(shard_backing_sizes(0, 1).is_err()); + } + + #[test] + fn shard_backing_sizes_allow_zero_shards() { + assert_eq!(shard_backing_sizes(0, 0).unwrap(), Vec::::new()); + } } diff --git a/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl b/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl index c02fb5d5f..188918888 100644 --- a/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl +++ b/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl @@ -58,7 +58,8 @@ data: memory: # Allocate per-shard backing from the heap instead of 2 MiB hugepages. no_hugepages: false - # Total backing pool across all serving shards in bytes (128 MiB). + # Node-wide shard data pool in bytes (128 MiB). Partial 2 MiB pages are + # unused; RPC scratch adds 8 pages per fabric unit outside this pool. memory_total_bytes: 134217728 fabric: tcp: diff --git a/hack/smoke-storage.py b/hack/smoke-storage.py index 02106baaf..4c5d9b477 100644 --- a/hack/smoke-storage.py +++ b/hack/smoke-storage.py @@ -168,19 +168,24 @@ def _openssl_bin() -> tuple[str, dict[str, str]]: # Hugepage backing. The daemon defaults to `backing_kind = "hugepage2_mb"`, # so the smoke test exercises that real path by reserving 2 MiB hugepages on -# the host up front (rather than passing `--no-hugepages` to fall back to the +# the host up front (rather than setting `no_hugepages` to fall back to the # heap). `memory_total_bytes` is pinned so the reservation below is exact. HUGEPAGE_SIZE = 2 * 1024 * 1024 # 2 MiB; matches memory::HUGEPAGE_2MB -MEMORY_TOTAL_BYTES = 128 * 1024 * 1024 # matches StorageCfg default; pinned for exactness -RPC_SCRATCH_PAGES = 8 # matches main.rs RPC_SCRATCH_PAGES +MEMORY_TOTAL_BYTES = 128 * 1024 * 1024 # node-wide data pool; matches config default +RPC_SCRATCH_PAGES_PER_FABRIC_UNIT = 8 # matches fabric_group.rs +FABRIC_UNITS_PER_NODE = 1 # the smoke config uses one shared TCP fabric unit NODES_PER_SCENARIO = 2 -# Hugepages the per-node pool backing needs: the whole memory_total_bytes -# pool (split across the node's serving shards) plus scratch, each rounded up. -_HP_PER_SHARD = (MEMORY_TOTAL_BYTES + HUGEPAGE_SIZE - 1) // HUGEPAGE_SIZE + RPC_SCRATCH_PAGES +# The daemon floors the node-wide data pool to whole pages before splitting it +# across serving shards. RPC scratch is a separate fixed backing per fabric unit. +_POOL_PAGES_PER_NODE = MEMORY_TOTAL_BYTES // HUGEPAGE_SIZE +_RPC_SCRATCH_PAGES_PER_NODE = ( + RPC_SCRATCH_PAGES_PER_FABRIC_UNIT * FABRIC_UNITS_PER_NODE +) +HUGEPAGES_PER_NODE = _POOL_PAGES_PER_NODE + _RPC_SCRATCH_PAGES_PER_NODE # Total for a scenario's two concurrent nodes, plus 50% headroom for any # allocator rounding / transient double-counting during teardown overlap. -HUGEPAGES_NEEDED = _HP_PER_SHARD * NODES_PER_SCENARIO +HUGEPAGES_NEEDED = HUGEPAGES_PER_NODE * NODES_PER_SCENARIO HUGEPAGES_RESERVE = HUGEPAGES_NEEDED + HUGEPAGES_NEEDED // 2 NR_HUGEPAGES_PATH = Path("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages") @@ -772,7 +777,7 @@ def write_config( # config table name. # # Startup-fixed knobs live in the `[startup]` section of the config: - # the fabric bind address, the per-shard hugepage backing size + # the fabric bind address, the node-wide hugepage data pool # (memory_total_bytes, leaving the daemon's hugepage default in place), and # forcing the libfabric tcp provider (disable_rdma) even on hosts that # expose an unusable RDMA HCA in sysfs. They only take effect at process From 4d944dde34ca79b4eb7c2a1c190677e5c5294c1e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 19:09:10 +0000 Subject: [PATCH 40/56] storage-supervisor: cap default serving cores --- deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl | 4 ++-- internal/storagesupervisor/render_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl b/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl index 188918888..54fbf5040 100644 --- a/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl +++ b/deploy/unbounded-storage-supervisor/02-configmap.yaml.tmpl @@ -84,8 +84,8 @@ data: allow_inactive_port: false # Disable RDMA and force the libfabric tcp provider. disable_rdma: false - # Cap on serving-shard CPUs to pin. Omit to claim every usable core. - # serving_cores: 12 + # Cap serving shards to fit the default data and RPC scratch budgets. + serving_cores: 8 # NIC-worker CPUs to pin per active HCA. nic_workers: 4 metrics: diff --git a/internal/storagesupervisor/render_test.go b/internal/storagesupervisor/render_test.go index 910a8db7b..0e29fea6b 100644 --- a/internal/storagesupervisor/render_test.go +++ b/internal/storagesupervisor/render_test.go @@ -137,7 +137,7 @@ startup: addr: "0.0.0.0:0" max_inflight: 1024 topology: - serving_cores: 0 + serving_cores: 8 nic_workers: 4 metrics: addr: "" @@ -153,7 +153,7 @@ startup: assert.NotNil(t, cfg.GetStartup().GetFabric().MaxInflight) assert.Equal(t, uint32(1024), cfg.GetStartup().GetFabric().GetMaxInflight()) assert.NotNil(t, cfg.GetStartup().GetTopology().ServingCores) - assert.Equal(t, uint64(0), cfg.GetStartup().GetTopology().GetServingCores()) + assert.Equal(t, uint64(8), cfg.GetStartup().GetTopology().GetServingCores()) assert.NotNil(t, cfg.GetStartup().GetTopology().NicWorkers) assert.Equal(t, uint64(4), cfg.GetStartup().GetTopology().GetNicWorkers()) assert.Empty(t, cfg.GetStartup().GetMetrics().GetAddr()) From 726b428a14079f7922f4e8a4ca7c23c3eda31222 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 02:24:24 +0000 Subject: [PATCH 41/56] unbounded-storage: publish disks before serving --- cmd/unbounded-storage/ARCHITECTURE.md | 23 ++-- cmd/unbounded-storage/README.md | 6 +- cmd/unbounded-storage/src/main.rs | 40 ++++-- cmd/unbounded-storage/src/shard_layer.rs | 122 ++++++++++++++---- .../tests/lifecycle/tests.rs | 5 + .../tests/lifecycle/workload.rs | 35 ++++- 6 files changed, 183 insertions(+), 48 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 1947f88e6..47944fb79 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -151,9 +151,12 @@ excluded from the live-reload diff. numeric identity as `NodeId`. 8. Each shard is spawned with `rt.spawn_pinned(widx, name, Box)`. The `!Send` shard objects are constructed **inside** `run_shard`, after pinning. -9. After every shard reports `Up`, peers are reconciled per shard, the disk - supervisor opens disks and publishes channels, and the config watcher takes - over for the lifetime of the process. +9. After every shard reports `Up`, peers are reconciled per shard and every + shard completes phase B while parked. The disk supervisor then opens the + initial disks and publishes channels. Only after publication does activation + start the recursive RPC servers and release shards into their serve loops; + the config watcher then takes over for the lifetime of the process. Disk-open + failures remain nonfatal and the successfully opened subset is published. ### Shard readiness and panic safety @@ -163,7 +166,10 @@ Each shard reports exactly one `ShardReady` message: `Up { descriptor, fabric }` panicking shard still emits one `Failed` via a dedicated panic channel; otherwise main's bounded receive would hang. Main performs a bounded `recv()` exactly `joins.len()` times (it does **not** drain to disconnect, because `Up` -shards never drop their sender). +shards never drop their sender). Preparation returns a `PreparedShardLayer` +only after all phase-B reports succeed. Activation starts RPC and releases the +parked shards; on failure it sets the layer stop flag, drops the serve gates, +joins shards in reverse order, and tears down fabric and backing keepalives. ### Shutdown @@ -172,7 +178,7 @@ SIGINT/SIGTERM handler (installed via `libc::sigaction`, relaxed atomic store). Every thread polls this flag. Teardown order is deliberate: join shard threads in reverse (releasing -`Arc` references) **first**, then drop the disk channel directory, then +`Arc` references) **first**, clear disk channel publications, then `disk_registry.drain()`. ## 5. Shard Bring-up (`run_shard`) @@ -197,9 +203,10 @@ per-shard `!Send` object graph: 5. **RPC server.** `FabricGroup`, constructed before the shards, allocates a **separate** scratch backing for each fabric unit (`RPC_SCRATCH_PAGES = 8`, one scratch page per in-flight serve/forward) and registers it as its own - fabric MR. After shard publication, it builds the unit's `RecursiveHandler` - and starts the RPC server. Scratch uses a distinct `LiveShardLocalStore` - because a `PageRef` resolves through exactly one backing's geometry. + fabric MR. After phase B and initial disk publication, activation builds the + unit's `RecursiveHandler` and starts the RPC server. Scratch uses a distinct + `LiveShardLocalStore` because a `PageRef` resolves through exactly one + backing's geometry. 6. **Frontends.** A shard hosts a `FrontendRegistry` of any number of frontends keyed by component name. Each spec binds its listener with `SO_REUSEPORT` and builds an `HttpDriver`/`S3Driver`; the registry can add and remove frontends diff --git a/cmd/unbounded-storage/README.md b/cmd/unbounded-storage/README.md index 90ef2da1e..3afd69a1d 100644 --- a/cmd/unbounded-storage/README.md +++ b/cmd/unbounded-storage/README.md @@ -59,7 +59,11 @@ restart. | `-V, --version` | - | Print version. | The daemon traps `SIGINT` and `SIGTERM` and tears shards down in a -deterministic order (disks closed first, then fabric / pool drops). +deterministic order: shards stop and join first, disk-channel publications are +cleared, and disks are drained last. At startup shards complete both readiness +phases while parked; initial disks are reconciled and published before RPC +servers and shard serving are activated. Individual disk-open failures remain +nonfatal. ## Configuration file diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 5fbc83c22..d95a88af5 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -409,10 +409,10 @@ fn main() -> ExitCode { .collect(); let mut disk_registry = DiskRegistry::new(UringDiskTarget::new(runtime.clone()), disk_slots); - // Bring up the initial shard layer. A bring-up failure is fatal: - // there is no running process to reconcile into. - let layer = match shard_layer::spawn_shard_layer(&config, &deps) { - Ok(layer) => layer, + // Prepare the initial shard layer through phase B. The shards remain + // parked and RPC serving stays off until initial disks are published. + let prepared = match shard_layer::prepare_shard_layer(&config, &deps) { + Ok(prepared) => prepared, Err(errs) => { for e in &errs { eprintln!("shard bring-up failed: {e}"); @@ -420,6 +420,24 @@ fn main() -> ExitCode { return ExitCode::FAILURE; } }; + // Reconcile and publish the startup disk set before any frontend or + // recursive RPC path can serve. Individual disk-open failures remain + // nonfatal and the successfully opened subset is published. + let projection = + config::runtime_projection(&config).expect("loaded config projects to runtime"); + reconcile_cache_disks(&mut disk_registry, &cache_directories, &projection); + + let layer = match shard_layer::activate_shard_layer(prepared) { + Ok(layer) => layer, + Err(errs) => { + for e in &errs { + eprintln!("shard activation failed: {e}"); + } + clear_cache_disk_publications(&cache_directories); + disk_registry.drain(); + return ExitCode::FAILURE; + } + }; let device_inventory = metrics::DeviceInventoryStatus::new(); device_inventory.set_rdma(device_inventory::rdma_annotation( &host, @@ -427,17 +445,12 @@ fn main() -> ExitCode { )); device_inventory.set_block(device_inventory::block_annotation(&host)); - // Reconcile the startup disk set now that the shards are up, then - // publish the channel set so shards can reach their disks. - let projection = - config::runtime_projection(&config).expect("loaded config projects to runtime"); - reconcile_cache_disks(&mut disk_registry, &cache_directories, &projection); - // The config controller is the single funnel for live changes. It // owns the running shard layer and disk supervisor (via the apply // target) and blocks each apply until the process has converged onto // the new config. - let target = shard_layer::ProcessApplyTarget::new(layer, disk_registry, cache_directories); + let target = + shard_layer::ProcessApplyTarget::new(layer, disk_registry, cache_directories.clone()); // Seeds the latest-known, latest-applied, and startup config // versions from the startup config's top-level `version`. // `controller.config_versions()` hands out a cloneable handle to all @@ -522,6 +535,7 @@ fn main() -> ExitCode { if let Some(layer) = layer { shard_layer::teardown_shard_layer(layer); } + clear_cache_disk_publications(&cache_directories); disk_registry.drain(); // The exporter polls `SHUTDOWN` on its accept loop; join it last so a @@ -1199,6 +1213,10 @@ fn reconcile_cache_disks( } } +fn clear_cache_disk_publications(cache_directories: &CacheDirectorySet) { + cache_directories.reconcile(std::iter::empty::()); +} + /// Validate and log the configured backends. /// /// [`OriginBackend`] is now the active origin tier: it is built per diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index d4353dc8e..c493c44a4 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -12,7 +12,8 @@ //! that retires just this layer's shards without touching the //! process-wide shutdown signal. //! -//! [`spawn_shard_layer`] brings a layer up from a config; +//! [`prepare_shard_layer`] brings a layer through phase B without +//! serving, [`activate_shard_layer`] starts RPC and releases the shards; //! [`teardown_shard_layer`] drains and joins it (used at process //! shutdown). [`ProcessApplyTarget`] realizes a live config apply //! entirely in place via [`ProcessApplyTarget::apply_in_place`]: @@ -65,8 +66,20 @@ pub struct ShardSpawnDeps { pub fabric_plan: FabricPlan, } +/// A prepared set of shard threads parked after phase B, before RPC +/// startup and serving activation. +pub struct PreparedShardLayer { + joins: Vec, + fabric_group: FabricGroup, + control: ShardControlGroup, + layer_stop: Arc, + backing_keepalives: Vec>, + rpc_shards: Vec, + serve_start_txs: Vec>, +} + /// A spawned set of shard threads plus the handles to drive and retire -/// them. Produced by [`spawn_shard_layer`], consumed by +/// them. Produced by [`activate_shard_layer`], consumed by /// [`teardown_shard_layer`]. pub struct ShardLayer { /// Join handles for every shard thread, in spawn order. @@ -98,8 +111,9 @@ impl ShardLayer { } } -/// Bring up a shard layer from `config` on the runtime in `deps`, -/// blocking until every shard has reported readiness. +/// Prepare a shard layer from `config` on the runtime in `deps`, +/// blocking until every shard has completed phase B. The returned +/// shards remain parked and no RPC server has started. /// /// Spawns one thread per worker in `deps.workers`, each with its own /// control channel (collected into the returned [`ShardControlGroup`]) @@ -111,10 +125,10 @@ impl ShardLayer { /// Returns `Err` with the collected per-shard error messages if any /// shard failed to come up; any shards that *did* come up are torn down /// first so no threads leak. -pub fn spawn_shard_layer( +pub fn prepare_shard_layer( config: &Config, deps: &ShardSpawnDeps, -) -> Result> { +) -> Result> { let layer_stop = Arc::new(AtomicBool::new(false)); let worker_count = deps.workers.len(); let settings = &deps.settings; @@ -134,7 +148,7 @@ pub fn spawn_shard_layer( // each shard registers its data backing against the endpoint it maps // onto. RPC servers start after shards publish their pool MRs/fetch // channels because owner responses source those bufferpool pages. - let mut fabric_group = FabricGroup::new( + let fabric_group = FabricGroup::new( &deps.runtime, &deps.fabric_plan, settings.backing_kind, @@ -253,6 +267,9 @@ pub fn spawn_shard_layer( for h in joins.into_iter().rev() { let _ = h.join(); } + drop(control_senders); + drop(fabric_group); + drop(publishes); return Err(errors); } @@ -324,44 +341,97 @@ pub fn spawn_shard_layer( } } if !phaseb_errors.is_empty() { - layer_stop.store(true, Ordering::Relaxed); - drop(serve_start_txs); - for h in joins.into_iter().rev() { - let _ = h.join(); - } + retire_failed_activation( + joins, + fabric_group, + ShardControlGroup::new(control_senders), + layer_stop, + backing_keepalives, + serve_start_txs, + ); return Err(phaseb_errors); } - if let Err(mut rpc_errors) = fabric_group.start_rpc_servers(&rpc_shards) { - layer_stop.store(true, Ordering::Relaxed); - drop(serve_start_txs); - for h in joins.into_iter().rev() { - let _ = h.join(); - } - phaseb_errors.append(&mut rpc_errors); - return Err(phaseb_errors); + Ok(PreparedShardLayer { + joins, + fabric_group, + control: ShardControlGroup::new(control_senders), + layer_stop, + backing_keepalives, + rpc_shards, + serve_start_txs, + }) +} + +/// Start recursive RPC servers and release every prepared shard into +/// its serve loop. Any failure retires and joins the entire prepared +/// layer before returning. +pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result> { + let PreparedShardLayer { + joins, + mut fabric_group, + control, + layer_stop, + backing_keepalives, + rpc_shards, + serve_start_txs, + } = prepared; + + if let Err(errors) = fabric_group.start_rpc_servers(&rpc_shards) { + retire_failed_activation( + joins, + fabric_group, + control, + layer_stop, + backing_keepalives, + serve_start_txs, + ); + return Err(errors); } - for tx in serve_start_txs { + for tx in &serve_start_txs { if tx.send(()).is_err() { - layer_stop.store(true, Ordering::Relaxed); - for h in joins.into_iter().rev() { - let _ = h.join(); - } + retire_failed_activation( + joins, + fabric_group, + control, + layer_stop, + backing_keepalives, + serve_start_txs, + ); return Err(vec![ "shard exited before recursive RPC servers were ready".to_string(), ]); } } + drop(serve_start_txs); Ok(ShardLayer { joins, fabric_group, - control: ShardControlGroup::new(control_senders), + control, layer_stop, _backing_keepalives: backing_keepalives, }) } +fn retire_failed_activation( + joins: Vec, + fabric_group: FabricGroup, + control: ShardControlGroup, + layer_stop: Arc, + backing_keepalives: Vec>, + serve_start_txs: Vec>, +) { + layer_stop.store(true, Ordering::Relaxed); + drop(serve_start_txs); + for h in joins.into_iter().rev() { + let _ = h.join(); + } + drop(control); + drop(fabric_group); + drop(backing_keepalives); +} + fn shard_backing_sizes( memory_total_bytes: usize, shard_count: usize, diff --git a/cmd/unbounded-storage/tests/lifecycle/tests.rs b/cmd/unbounded-storage/tests/lifecycle/tests.rs index 00001f0d3..9d68c867c 100644 --- a/cmd/unbounded-storage/tests/lifecycle/tests.rs +++ b/cmd/unbounded-storage/tests/lifecycle/tests.rs @@ -48,6 +48,11 @@ proptest! { report.serve_before_phase_b, 0, "serving future ran before phase-B peer publication", ); + prop_assert_eq!( + report.serve_before_initial_disk_publication, + 0, + "serving future ran before initial disk publication", + ); } /// Invariant: disk config changes publish coherent directory generations diff --git a/cmd/unbounded-storage/tests/lifecycle/workload.rs b/cmd/unbounded-storage/tests/lifecycle/workload.rs index c3799fb4c..4bcccc051 100644 --- a/cmd/unbounded-storage/tests/lifecycle/workload.rs +++ b/cmd/unbounded-storage/tests/lifecycle/workload.rs @@ -101,6 +101,7 @@ pub struct RunReport { pub phase_a_ready: usize, pub phase_b_ready: usize, pub serve_before_phase_b: u64, + pub serve_before_initial_disk_publication: u64, pub shard_apply_counts: Vec, pub broadcasts: usize, pub disk_applies: usize, @@ -225,6 +226,7 @@ struct ShardState { apply_queue: RefCell>, applied: RefCell>, serve_before_phase_b: Cell, + serve_before_initial_disk_publication: Cell, } impl ShardState { @@ -236,6 +238,7 @@ impl ShardState { apply_queue: RefCell::new(VecDeque::new()), applied: RefCell::new(Vec::new()), serve_before_phase_b: Cell::new(0), + serve_before_initial_disk_publication: Cell::new(0), } } } @@ -321,6 +324,7 @@ pub fn run_workload(seed: u64, w: Workload) -> Result { .map(|_| Rc::new(ShardState::new())) .collect(); let peer_published = Rc::new(Cell::new(false)); + let initial_disks_published = Rc::new(Cell::new(false)); let serving_ready = Rc::new(Cell::new(false)); let clients_finished = Rc::new(Cell::new(0usize)); let completed_ops = Rc::new(Cell::new(0usize)); @@ -342,13 +346,21 @@ pub fn run_workload(seed: u64, w: Workload) -> Result { pool_base, pool_len, bootstrap_done.clone(), + initial_disks_published.clone(), ); spawn_storage_cores(&mut exec, generations.clone()); - spawn_shards(&mut exec, &shard_states, peer_published.clone()); + spawn_shards( + &mut exec, + &shard_states, + peer_published.clone(), + initial_disks_published.clone(), + serving_ready.clone(), + ); spawn_phase_b_supervisor( &mut exec, &shard_states, peer_published.clone(), + initial_disks_published, serving_ready.clone(), ); spawn_apply_driver( @@ -409,6 +421,10 @@ pub fn run_workload(seed: u64, w: Workload) -> Result { .iter() .map(|s| s.serve_before_phase_b.get()) .sum(), + serve_before_initial_disk_publication: shard_states + .iter() + .map(|s| s.serve_before_initial_disk_publication.get()) + .sum(), shard_apply_counts: shard_states .iter() .map(|s| s.applied.borrow().len()) @@ -523,6 +539,7 @@ fn spawn_bootstrap( pool_base: usize, pool_len: usize, bootstrap_done: Rc>, + initial_disks_published: Rc>, ) { exec.spawn(async move { sim_cfg.max_io_delay.set(0); @@ -555,6 +572,7 @@ fn spawn_bootstrap( } sim_cfg.max_io_delay.set(w.max_io_delay); publish_generation(&directory, &generations[0]); + initial_disks_published.set(true); bootstrap_done.set(true); }); } @@ -620,15 +638,19 @@ fn spawn_shards( exec: &mut Executor, shard_states: &[Rc], peer_published: Rc>, + initial_disks_published: Rc>, + serving_ready: Rc>, ) { for state in shard_states { let state = state.clone(); let peer_published = peer_published.clone(); + let initial_disks_published = initial_disks_published.clone(); + let serving_ready = serving_ready.clone(); exec.spawn(async move { let mut loop_driver = ShardLoop::new(); let serving_state = state.clone(); loop_driver.spawn(async move { - while !serving_state.phase_b.get() && !serving_state.stop.get() { + while !serving_ready.get() && !serving_state.stop.get() { yield_once().await; } while !serving_state.stop.get() { @@ -637,6 +659,11 @@ fn spawn_shards( .serve_before_phase_b .set(serving_state.serve_before_phase_b.get() + 1); } + if !initial_disks_published.get() { + serving_state + .serve_before_initial_disk_publication + .set(serving_state.serve_before_initial_disk_publication.get() + 1); + } yield_once().await; } }); @@ -671,6 +698,7 @@ fn spawn_phase_b_supervisor( exec: &mut Executor, shard_states: &[Rc], peer_published: Rc>, + initial_disks_published: Rc>, serving_ready: Rc>, ) { let shards = shard_states.to_vec(); @@ -682,6 +710,9 @@ fn spawn_phase_b_supervisor( while !shards.iter().all(|s| s.phase_b.get()) { yield_once().await; } + while !initial_disks_published.get() { + yield_once().await; + } serving_ready.set(true); }); } From 703932557a7b3473daefc1ada74bac24b940f6eb Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 02:40:01 +0000 Subject: [PATCH 42/56] unbounded-storage: bound config apply fan-in --- cmd/unbounded-storage/ARCHITECTURE.md | 6 +- cmd/unbounded-storage/src/config/control.rs | 316 +++++++++++++++----- cmd/unbounded-storage/src/shard_layer.rs | 21 +- 3 files changed, 264 insertions(+), 79 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 47944fb79..40adbe3f5 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -578,7 +578,11 @@ registries are all updated without tearing the shard layer down. Backing memory, the topology plan, the fabric max in-flight knob, and the local `self` peer identity are fixed at startup (mostly sourced from the config `[startup]` section, see the CLI section), not reloadable config -fields. +fields. Shard apply and page-cache-drain fan-in use one 60-second deadline +for the complete worker set and report outstanding worker identities. A +timeout, acknowledgement-channel disconnect, or partial broadcast send failure +requests process shutdown: already-delivered commands may complete later, so +retrying against the old controller snapshot would be unsafe. Sections (all optional, each falling back to defaults): diff --git a/cmd/unbounded-storage/src/config/control.rs b/cmd/unbounded-storage/src/config/control.rs index 69a189cd1..ec0a6377d 100644 --- a/cmd/unbounded-storage/src/config/control.rs +++ b/cmd/unbounded-storage/src/config/control.rs @@ -30,10 +30,12 @@ //! blocking fan-out/fan-in primitive used by that implementation lives //! here as [`ShardControlGroup`]. +use std::collections::HashSet; use std::fmt; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{self, RecvError, Sender}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::time::{Duration, Instant}; use crate::config::{Config, ConfigDiff}; use crate::p2p::RouteTableSnapshot; @@ -89,7 +91,19 @@ pub enum ApplyError { ShardSend(WorkerIdx), /// The ack channel disconnected before every shard reported, so we /// cannot prove the apply completed everywhere. - AckDisconnected { expected: usize, received: usize }, + AckDisconnected { + expected: usize, + received: usize, + outstanding: Vec, + }, + /// Not every shard acknowledged before the control deadline. Commands + /// already delivered may still complete, so the live process can no + /// longer safely retry the apply. + AckTimeout { + expected: usize, + received: usize, + outstanding: Vec, + }, /// One or more shards reported a failure while applying. ShardApply(Vec<(WorkerIdx, String)>), /// The process-level apply target rejected the config before shard @@ -103,9 +117,23 @@ impl fmt::Display for ApplyError { ApplyError::ShardSend(w) => { write!(f, "shard {} control channel closed before apply", w.0) } - ApplyError::AckDisconnected { expected, received } => write!( + ApplyError::AckDisconnected { + expected, + received, + outstanding, + } => write!( f, - "ack channel disconnected after {received}/{expected} shards reported", + "ack channel disconnected after {received}/{expected} shards reported; outstanding workers: {}", + format_workers(outstanding), + ), + ApplyError::AckTimeout { + expected, + received, + outstanding, + } => write!( + f, + "ack deadline expired after {received}/{expected} shards reported; outstanding workers: {}", + format_workers(outstanding), ), ApplyError::ShardApply(failures) => { write!(f, "{} shard(s) failed to apply config:", failures.len())?; @@ -121,6 +149,18 @@ impl fmt::Display for ApplyError { impl std::error::Error for ApplyError {} +impl ApplyError { + /// Whether some shards may still apply a command after this error is + /// returned. The process must stop rather than retry against its old + /// controller snapshot. + pub fn apply_state_is_indeterminate(&self) -> bool { + matches!( + self, + Self::ShardSend(_) | Self::AckDisconnected { .. } | Self::AckTimeout { .. } + ) + } +} + /// Which path an apply took. Returned so callers (and tests) can assert /// on the work that actually happened. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -345,17 +385,22 @@ impl ConfigController { /// /// [`broadcast_apply`](Self::broadcast_apply) sends one /// [`ShardCommand::ApplyConfig`] to every shard and blocks until each -/// has acknowledged, so the caller can guarantee the change has landed -/// on every shard thread before returning. This is the concrete +/// has acknowledged or the group's absolute deadline expires, so the caller +/// can guarantee the change has landed on every shard thread before returning +/// success. This is the concrete /// "close the loop" mechanism a [`ConfigApplyTarget`] builds its Tier 1 /// path on. pub struct ShardControlGroup { senders: Vec<(WorkerIdx, Sender)>, + ack_timeout: Duration, } impl ShardControlGroup { - pub fn new(senders: Vec<(WorkerIdx, Sender)>) -> Self { - Self { senders } + pub fn new(senders: Vec<(WorkerIdx, Sender)>, ack_timeout: Duration) -> Self { + Self { + senders, + ack_timeout, + } } pub fn len(&self) -> usize { @@ -370,77 +415,40 @@ impl ShardControlGroup { /// them acknowledge. /// /// Returns `Ok(())` only when every shard reports success. If a - /// channel is closed at send time, or the ack channel disconnects - /// before all shards report, or any shard reports a failure, the - /// corresponding [`ApplyError`] is returned. The fan-in always - /// drains every ack that does arrive so a single failing shard does - /// not mask the others. + /// channel is closed at send time, the ack deadline expires, the ack + /// channel disconnects before all shards report, or any shard reports a + /// failure, the corresponding [`ApplyError`] is returned. Until a terminal + /// channel/deadline error, fan-in keeps collecting reports so one failing + /// shard does not mask failures from the others. pub fn broadcast_apply( &self, config: Arc, routes: RouteTableSnapshot, diff: ConfigDiff, ) -> Result<(), ApplyError> { - let expected = self.senders.len(); - if expected == 0 { - return Ok(()); - } - - let (ack_tx, ack_rx) = mpsc::channel::(); - - for (worker, sender) in &self.senders { - let cmd = ShardCommand::ApplyConfig(ShardApply { + self.broadcast(|ack| { + ShardCommand::ApplyConfig(ShardApply { config: config.clone(), routes: routes.clone(), diff, - ack: ack_tx.clone(), - }); - sender - .send(cmd) - .map_err(|_| ApplyError::ShardSend(*worker))?; - } - // Drop our own handle so the channel closes once every shard's - // cloned sender is dropped; that is how the fan-in loop below - // detects "no more acks coming". - drop(ack_tx); - - let mut received = 0usize; - let mut failures = Vec::new(); - loop { - match ack_rx.recv() { - Ok(ack) => { - received += 1; - if let Err(e) = ack.result { - failures.push((ack.worker, e)); - } - if received == expected { - break; - } - } - Err(RecvError) => { - return Err(ApplyError::AckDisconnected { expected, received }); - } - } - } - - if failures.is_empty() { - Ok(()) - } else { - Err(ApplyError::ShardApply(failures)) - } + ack, + }) + }) } /// Ask every shard to drain retained RAM page-cache entries and block /// until all have acknowledged. pub fn broadcast_drain_page_cache(&self) -> Result<(), ApplyError> { - self.broadcast_drain(|ack| ShardCommand::DrainPageCache(ShardDrainPageCache { ack })) + self.broadcast(|ack| ShardCommand::DrainPageCache(ShardDrainPageCache { ack })) } - fn broadcast_drain(&self, make_cmd: F) -> Result<(), ApplyError> + fn broadcast(&self, make_cmd: F) -> Result<(), ApplyError> where F: Fn(Sender) -> ShardCommand, { - let expected = self.senders.len(); + let mut outstanding: HashSet = + self.senders.iter().map(|(worker, _)| *worker).collect(); + let expected = outstanding.len(); if expected == 0 { return Ok(()); } @@ -453,21 +461,39 @@ impl ShardControlGroup { } drop(ack_tx); + let deadline = Instant::now() + self.ack_timeout; let mut received = 0usize; let mut failures = Vec::new(); - loop { - match ack_rx.recv() { + while !outstanding.is_empty() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(ApplyError::AckTimeout { + expected, + received, + outstanding: sorted_workers(&outstanding), + }); + }; + match ack_rx.recv_timeout(remaining) { Ok(ack) => { - received += 1; - if let Err(e) = ack.result { - failures.push((ack.worker, e)); - } - if received == expected { - break; + if outstanding.remove(&ack.worker) { + received += 1; + if let Err(e) = ack.result { + failures.push((ack.worker, e)); + } } } - Err(RecvError) => { - return Err(ApplyError::AckDisconnected { expected, received }); + Err(RecvTimeoutError::Timeout) => { + return Err(ApplyError::AckTimeout { + expected, + received, + outstanding: sorted_workers(&outstanding), + }); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(ApplyError::AckDisconnected { + expected, + received, + outstanding: sorted_workers(&outstanding), + }); } } } @@ -480,11 +506,26 @@ impl ShardControlGroup { } } +fn sorted_workers(workers: &HashSet) -> Vec { + let mut workers: Vec<_> = workers.iter().copied().collect(); + workers.sort_by_key(|worker| worker.0); + workers +} + +fn format_workers(workers: &[WorkerIdx]) -> String { + workers + .iter() + .map(|worker| worker.0.to_string()) + .collect::>() + .join(",") +} + #[cfg(test)] mod tests { use std::sync::Arc; use std::sync::mpsc; use std::thread; + use std::time::Duration; use super::*; use crate::p2p::RouteTableSnapshot; @@ -493,6 +534,17 @@ mod tests { RouteTableSnapshot::default() } + fn control_group(senders: Vec<(WorkerIdx, Sender)>) -> ShardControlGroup { + ShardControlGroup::new(senders, Duration::from_secs(1)) + } + + fn ack_sender(cmd: ShardCommand) -> Sender { + match cmd { + ShardCommand::ApplyConfig(apply) => apply.ack, + ShardCommand::DrainPageCache(drain) => drain.ack, + } + } + /// Spawn `n` mock shard threads that drain a control channel and ack /// with the supplied result. Returns the control senders plus the /// join handles. @@ -537,7 +589,7 @@ mod tests { #[test] fn broadcast_apply_blocks_until_every_shard_acks() { let (senders, joins) = spawn_mock_shards(vec![Ok(()), Ok(()), Ok(())]); - let group = ShardControlGroup::new(senders); + let group = control_group(senders); let out = group.broadcast_apply( Arc::new(Config::default()), @@ -557,7 +609,7 @@ mod tests { #[test] fn broadcast_apply_reports_shard_failures() { let (senders, joins) = spawn_mock_shards(vec![Ok(()), Err("boom".to_string()), Ok(())]); - let group = ShardControlGroup::new(senders); + let group = control_group(senders); let err = group .broadcast_apply( @@ -583,7 +635,7 @@ mod tests { #[test] fn broadcast_apply_on_empty_group_is_a_noop() { - let group = ShardControlGroup::new(Vec::new()); + let group = control_group(Vec::new()); assert!(group.is_empty()); assert!( group @@ -603,7 +655,7 @@ mod tests { let (dead_tx, dead_rx) = mpsc::channel::(); drop(dead_rx); senders.push((WorkerIdx(99), dead_tx)); - let group = ShardControlGroup::new(senders); + let group = control_group(senders); let err = group .broadcast_apply( @@ -613,6 +665,7 @@ mod tests { ) .expect_err("closed channel must error"); assert!(matches!(err, ApplyError::ShardSend(WorkerIdx(99)))); + assert!(err.apply_state_is_indeterminate()); drop(group); for j in joins { @@ -620,6 +673,121 @@ mod tests { } } + #[test] + fn reported_shard_failure_is_not_indeterminate() { + assert!( + !ApplyError::ShardApply(vec![(WorkerIdx(1), "failed".to_string())]) + .apply_state_is_indeterminate() + ); + assert!(!ApplyError::Target("rejected".to_string()).apply_state_is_indeterminate()); + } + + #[test] + fn broadcast_apply_timeout_lists_outstanding_workers() { + let (tx0, rx0) = mpsc::channel::(); + let (tx1, rx1) = mpsc::channel::(); + let join0 = thread::spawn(move || { + let ack = ack_sender(rx0.recv().unwrap()); + ack.send(ShardAck { + worker: WorkerIdx(0), + result: Ok(()), + }) + .unwrap(); + }); + let join1 = thread::spawn(move || { + let ack = ack_sender(rx1.recv().unwrap()); + thread::sleep(Duration::from_millis(100)); + let _ = ack.send(ShardAck { + worker: WorkerIdx(1), + result: Ok(()), + }); + }); + let group = ShardControlGroup::new( + vec![(WorkerIdx(0), tx0), (WorkerIdx(1), tx1)], + Duration::from_millis(20), + ); + + let error = group + .broadcast_apply( + Arc::new(Config::default()), + empty_routes(), + ConfigDiff::default(), + ) + .expect_err("missing acknowledgement must time out"); + assert!(matches!( + error, + ApplyError::AckTimeout { + expected: 2, + received: 1, + outstanding, + } if outstanding == vec![WorkerIdx(1)] + )); + + join0.join().unwrap(); + join1.join().unwrap(); + } + + #[test] + fn duplicate_ack_does_not_hide_outstanding_worker() { + let (tx0, rx0) = mpsc::channel::(); + let (tx1, rx1) = mpsc::channel::(); + let join0 = thread::spawn(move || { + let ack = ack_sender(rx0.recv().unwrap()); + for _ in 0..2 { + ack.send(ShardAck { + worker: WorkerIdx(0), + result: Ok(()), + }) + .unwrap(); + } + }); + let join1 = thread::spawn(move || { + let ack = ack_sender(rx1.recv().unwrap()); + thread::sleep(Duration::from_millis(100)); + drop(ack); + }); + let group = ShardControlGroup::new( + vec![(WorkerIdx(0), tx0), (WorkerIdx(1), tx1)], + Duration::from_millis(20), + ); + + let error = group + .broadcast_drain_page_cache() + .expect_err("duplicate acknowledgement must not complete fan-in"); + assert!(matches!( + error, + ApplyError::AckTimeout { + expected: 2, + received: 1, + outstanding, + } if outstanding == vec![WorkerIdx(1)] + )); + + join0.join().unwrap(); + join1.join().unwrap(); + } + + #[test] + fn disconnected_ack_channel_lists_outstanding_workers() { + let (tx, rx) = mpsc::channel::(); + let join = thread::spawn(move || drop(ack_sender(rx.recv().unwrap()))); + let group = control_group(vec![(WorkerIdx(7), tx)]); + + let error = group + .broadcast_drain_page_cache() + .expect_err("dropped acknowledgement sender must disconnect fan-in"); + assert!(matches!( + error, + ApplyError::AckDisconnected { + expected: 1, + received: 0, + outstanding, + } if outstanding == vec![WorkerIdx(7)] + )); + + join.join().unwrap(); + } + // ---- ConfigController classification ---- #[derive(Default)] diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index c493c44a4..51337452f 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -29,6 +29,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; +use std::time::Duration; use unbounded_storage::config::{ self, ApplyError, Config, ConfigApplyTarget, ConfigDiff, ShardControlGroup, @@ -44,6 +45,8 @@ use unbounded_storage::topology::ServingShard; use crate::StartupSettings; use crate::fabric_group::{FabricGroup, FabricPlan, FabricUnitAddress, RpcShardPublish}; +const SHARD_CONTROL_TIMEOUT: Duration = Duration::from_secs(60); + /// Inputs that are constant across the life of the process and used to /// spawn the shard layer. Cloned cheaply into every shard thread. pub struct ShardSpawnDeps { @@ -344,7 +347,7 @@ pub fn prepare_shard_layer( retire_failed_activation( joins, fabric_group, - ShardControlGroup::new(control_senders), + ShardControlGroup::new(control_senders, SHARD_CONTROL_TIMEOUT), layer_stop, backing_keepalives, serve_start_txs, @@ -355,7 +358,7 @@ pub fn prepare_shard_layer( Ok(PreparedShardLayer { joins, fabric_group, - control: ShardControlGroup::new(control_senders), + control: ShardControlGroup::new(control_senders, SHARD_CONTROL_TIMEOUT), layer_stop, backing_keepalives, rpc_shards, @@ -584,7 +587,12 @@ impl ConfigApplyTarget for ProcessApplyTarget { // each has acked, so the routing surface and per-shard // backend/frontend reconcile each shard performs on receipt // have provably landed. - layer.control.broadcast_apply(new.clone(), routes, *diff)?; + if let Err(error) = layer.control.broadcast_apply(new.clone(), routes, *diff) { + if error.apply_state_is_indeterminate() { + crate::SHUTDOWN.store(true, Ordering::Release); + } + return Err(error); + } } if diff.disks_changed { @@ -592,7 +600,12 @@ impl ConfigApplyTarget for ProcessApplyTarget { .layer .as_mut() .expect("shard layer present between applies"); - layer.control.broadcast_drain_page_cache()?; + if let Err(error) = layer.control.broadcast_drain_page_cache() { + if error.apply_state_is_indeterminate() { + crate::SHUTDOWN.store(true, Ordering::Release); + } + return Err(error); + } } // Disk/channel changes mutate live registries and directories, so From 526335fce5b0500a2ee171e15950407b3aede533 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 02:54:47 +0000 Subject: [PATCH 43/56] unbounded-storage: supervise shard panics --- cmd/unbounded-storage/ARCHITECTURE.md | 25 ++- cmd/unbounded-storage/src/main.rs | 236 +++++++++++++++-------- cmd/unbounded-storage/src/shard_layer.rs | 30 ++- 3 files changed, 199 insertions(+), 92 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 40adbe3f5..b9d54856d 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -160,16 +160,21 @@ excluded from the live-reload diff. ### Shard readiness and panic safety -Each shard reports exactly one `ShardReady` message: `Up { descriptor, fabric }` -(after which it parks while holding its `Sender`), or `Failed(String)`. -`report_on_panic` wraps shard bring-up in `catch_unwind`/`AssertUnwindSafe` so a -panicking shard still emits one `Failed` via a dedicated panic channel; -otherwise main's bounded receive would hang. Main performs a bounded `recv()` -exactly `joins.len()` times (it does **not** drain to disconnect, because `Up` -shards never drop their sender). Preparation returns a `PreparedShardLayer` -only after all phase-B reports succeed. Activation starts RPC and releases the -parked shards; on failure it sets the layer stop flag, drops the serve gates, -joins shards in reverse order, and tears down fabric and backing keepalives. +Each shard owns a phase-A reporter that emits exactly one `ShardReady` message: +`Up { worker_idx, publish }` or `Failed(String)`. Its Drop fallback covers an +early return or panic before the explicit report. Phase B uses the same +consuming-report/Drop pattern. Main performs a bounded `recv()` exactly +`joins.len()` times (it does **not** drain to disconnect, because live shards +retain channel senders). Preparation returns a `PreparedShardLayer` only after +all phase-B reports succeed. Activation starts RPC and releases the parked +shards; on failure it sets the layer stop flag, drops the serve gates, joins +shards in reverse order, and tears down fabric and backing keepalives. + +Shard panics are logged and resumed so the native join handle remains failed. +Once serving starts, a lifetime guard balances the live-shard metric and sends +a terminal report on abnormal exit. That report also sets process-wide +shutdown; teardown drains terminal reports and returns a failing process status +instead of continuing with a missing shard. ### Shutdown diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index d95a88af5..8d7265259 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -489,7 +489,7 @@ fn main() -> ExitCode { // Watch the config file and drive each update through the // controller until shutdown. Every apply is in place; an apply error // is logged and the process keeps serving on the last-good config. - let exit_code = ExitCode::SUCCESS; + let mut exit_code = ExitCode::SUCCESS; match config::ConfigWatcher::new(config_path.clone()) { Ok((_watcher, update_rx)) => { while !SHUTDOWN.load(Ordering::Acquire) { @@ -533,7 +533,9 @@ fn main() -> ExitCode { // stop flag. let (layer, disk_registry) = controller.into_target().into_parts(); if let Some(layer) = layer { - shard_layer::teardown_shard_layer(layer); + if shard_layer::teardown_shard_layer(layer) { + exit_code = ExitCode::FAILURE; + } } clear_cache_disk_publications(&cache_directories); disk_registry.drain(); @@ -569,8 +571,10 @@ fn run_shard( peer_rx: mpsc::Receiver>>, phaseb_tx: mpsc::Sender, serve_start_rx: mpsc::Receiver<()>, + terminal_tx: mpsc::Sender, layer_stop: Arc, ) { + let phase_a = PhaseAReporter::new(widx, tx); // The fabric endpoint is built and owned by the `FabricGroup` in the // shard layer and shared by every shard mapped onto it (one per shard // for the tcp fallback, one per HCA for verbs). This shard registers @@ -589,10 +593,7 @@ fn run_shard( }) { Ok(b) => b, Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: backing allocation failed: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: backing allocation failed: {e}", widx.0,)); return; } }; @@ -600,10 +601,7 @@ fn run_shard( let mr = match fabric.register_backing(&backing, shard.numa) { Ok(mr) => mr, Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: register_backing: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: register_backing: {e}", widx.0,)); return; } }; @@ -616,18 +614,12 @@ fn run_shard( let socket = match NetworkRing::new(256) { Ok(s) => Rc::new(s), Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: NetworkRing::new: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: NetworkRing::new: {e}", widx.0,)); return; } }; if let Err(e) = socket.register_backing(&backing) { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: socket register_backing: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: socket register_backing: {e}", widx.0,)); return; } @@ -692,10 +684,7 @@ fn run_shard( ) { Ok(r) => r, Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: build backend registry: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: build backend registry: {e}", widx.0,)); return; } }; @@ -714,10 +703,10 @@ fn run_shard( ) { Ok(t) => t, Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( + phase_a.report_failed(format!( "worker={}: RoutedTransport::with_routes: {e}", widx.0, - ))); + )); return; } }; @@ -736,10 +725,7 @@ fn run_shard( ) { Ok(p) => p, Err(e) => { - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: Pool::new: {e}", - widx.0, - ))); + phase_a.report_failed(format!("worker={}: Pool::new: {e}", widx.0,)); return; } }; @@ -787,16 +773,13 @@ fn run_shard( // publishing; exit cleanly by returning (the shard locals drop in // reverse declaration order; the shared `fabric` is released by the // `FabricGroup`, not here). - let _ = tx.send(ShardReady::Up { - worker_idx: widx, - publish: ShardPublish { - backing_base: backing_base as usize, - backing_len, - fabric_mr: mr, - numa: shard.numa, - fetch_channel, - backing_keepalive, - }, + phase_a.report_up(ShardPublish { + backing_base: backing_base as usize, + backing_len, + fabric_mr: mr, + numa: shard.numa, + fetch_channel, + backing_keepalive, }); let peers = match peer_rx.recv() { Ok(peers) => peers, @@ -807,7 +790,7 @@ fn run_shard( // The guard sends `Failed` on any early return or panic so the // layer's bounded Phase-B collection never hangs waiting on a shard // that aborted (mirroring `report_on_panic` for Phase A). - let mut phaseb_guard = PhaseBGuard::new(widx, phaseb_tx); + let phaseb_guard = PhaseBGuard::new(widx, phaseb_tx); // PHASE B: register every peer shard's backing on our socket ring // (recording the fixed-buffer index each lands at) and assemble the @@ -1029,7 +1012,7 @@ fn run_shard( Ok(()) => {} Err(_) => return, } - metrics::shards_delta(1); + let serving_guard = ServingGuard::new(widx, terminal_tx, &SHUTDOWN); // Drive the shard's cooperative future set until shutdown. The loop // busy-polls socket I/O and frontend work while active and idles @@ -1040,8 +1023,6 @@ fn run_shard( Duration::from_micros(100), ); - metrics::shards_delta(-1); - // Drop order matters: // 1. `shard_loop` first - clears tick hooks and futures, releasing // their `Rc` clones and (if registered) the `HttpDriver` @@ -1057,22 +1038,11 @@ fn run_shard( drop(shard_loop); drop(pool); drop(socket); + serving_guard.complete(); } -/// Run a shard thread body `f`, guaranteeing exactly one -/// [`ShardReady`] is observed on `tx` even when `f` panics during -/// bring-up. -/// -/// `f` (i.e. [`run_shard`]) reports its own readiness on the normal -/// success and error paths through a sender it owns. The `tx` handed -/// here is a *separate* clone reserved solely for the panic path: if -/// `f` unwinds before reporting, its own sender is dropped by the -/// unwind, but the surviving shards stay parked holding their sender -/// clones, so `main`'s bounded `recv()` would block forever. Emitting a -/// `Failed` here keeps the readiness count whole. A panic *after* `f` -/// already reported `Up` produces a harmless extra `Failed`: `main` -/// reads exactly N messages and ignores any surplus. -fn report_on_panic(tx: mpsc::Sender, widx: WorkerIdx, f: F) +/// Log a shard panic and preserve it for the owning join handle. +fn report_on_panic(widx: WorkerIdx, f: F) where F: FnOnce(), { @@ -1081,14 +1051,11 @@ where // process is headed for the coherent failure path regardless. if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) { eprintln!( - "shard worker={} panicked during bring-up: {}", + "shard worker={} panicked: {}", widx.0, - panic_payload_str(&payload), + panic_payload_str(&payload) ); - let _ = tx.send(ShardReady::Failed(format!( - "worker={}: panicked during bring-up", - widx.0, - ))); + std::panic::resume_unwind(payload); } } @@ -1484,6 +1451,48 @@ enum ShardReady { Failed(String), } +/// Owning phase-A reporter. Consuming report methods and the Drop +/// fallback guarantee exactly one readiness message per shard. +struct PhaseAReporter { + worker_idx: WorkerIdx, + tx: Option>, +} + +impl PhaseAReporter { + fn new(worker_idx: WorkerIdx, tx: mpsc::Sender) -> Self { + Self { + worker_idx, + tx: Some(tx), + } + } + + fn report_up(mut self, publish: ShardPublish) { + if let Some(tx) = self.tx.take() { + let _ = tx.send(ShardReady::Up { + worker_idx: self.worker_idx, + publish, + }); + } + } + + fn report_failed(mut self, message: String) { + if let Some(tx) = self.tx.take() { + let _ = tx.send(ShardReady::Failed(message)); + } + } +} + +impl Drop for PhaseAReporter { + fn drop(&mut self) { + if let Some(tx) = self.tx.take() { + let _ = tx.send(ShardReady::Failed(format!( + "worker={}: aborted during phase A bring-up", + self.worker_idx.0 + ))); + } + } +} + /// Phase-A publication from one shard: the backing region other shards /// must register to `SEND_ZC` from this shard's pinned pages, plus the /// channel to this shard's [`FetchService`]. The base is shipped as a @@ -1554,17 +1563,68 @@ impl PhaseBGuard { } } - fn report_ready(&mut self) { + fn report_ready(mut self) { self.reported = true; let _ = self.tx.send(PhaseBReport::Ready(self.widx)); } - fn report_failed(&mut self, msg: String) { + fn report_failed(mut self, msg: String) { self.reported = true; let _ = self.tx.send(PhaseBReport::Failed(msg)); } } +#[derive(Debug)] +struct ShardTerminalReport { + worker_idx: WorkerIdx, + message: String, +} + +struct ServingGuard { + worker_idx: WorkerIdx, + tx: mpsc::Sender, + shutdown: &'static AtomicBool, + completed: bool, +} + +impl ServingGuard { + fn new( + worker_idx: WorkerIdx, + tx: mpsc::Sender, + shutdown: &'static AtomicBool, + ) -> Self { + metrics::shards_delta(1); + Self { + worker_idx, + tx, + shutdown, + completed: false, + } + } + + fn complete(mut self) { + self.completed = true; + } +} + +impl Drop for ServingGuard { + fn drop(&mut self) { + metrics::shards_delta(-1); + if !self.completed { + let message = format!( + "worker={}: shard terminated while serving", + self.worker_idx.0 + ); + eprintln!("{message}"); + let _ = self.tx.send(ShardTerminalReport { + worker_idx: self.worker_idx, + message, + }); + self.shutdown.store(true, Ordering::Release); + } + } +} + impl Drop for PhaseBGuard { fn drop(&mut self) { if !self.reported { @@ -2079,33 +2139,51 @@ mod tests { } #[test] - fn report_on_panic_emits_failed_when_body_panics() { - // A shard body that panics before reporting must still leave - // exactly one `Failed` on the channel so `main`'s bounded recv - // count stays whole and startup cannot deadlock. + fn phase_a_reporter_drop_reports_failure_once() { let (tx, rx) = mpsc::channel::(); - report_on_panic(tx, WorkerIdx(7), || { - panic!("boom during bring-up"); - }); + drop(PhaseAReporter::new(WorkerIdx(7), tx)); match rx.recv() { Ok(ShardReady::Failed(msg)) => { assert!(msg.contains("worker=7"), "got: {msg}"); - assert!(msg.contains("panicked during bring-up"), "got: {msg}"); + assert!(msg.contains("aborted during phase A"), "got: {msg}"); } other => panic!("expected Failed, got {:?}", other.is_ok()), } - // No second message: the panic path reports exactly once. assert!(rx.try_recv().is_err()); } #[test] - fn report_on_panic_is_silent_when_body_succeeds() { - // On the normal path `run_shard` owns the reporting; the - // panic-path clone must stay quiet so no spurious `Failed` - // is appended. - let (tx, rx) = mpsc::channel::(); - report_on_panic(tx, WorkerIdx(0), || {}); + fn report_on_panic_resumes_unwind() { + let result = std::panic::catch_unwind(|| { + report_on_panic(WorkerIdx(7), || panic!("boom during serving")); + }); + assert!(result.is_err()); + } + + #[test] + fn serving_guard_reports_abnormal_termination() { + static TEST_SHUTDOWN: AtomicBool = AtomicBool::new(false); + TEST_SHUTDOWN.store(false, Ordering::Relaxed); + let (tx, rx) = mpsc::channel(); + + drop(ServingGuard::new(WorkerIdx(3), tx, &TEST_SHUTDOWN)); + + let report = rx.recv().unwrap(); + assert_eq!(report.worker_idx, WorkerIdx(3)); + assert!(report.message.contains("terminated while serving")); + assert!(TEST_SHUTDOWN.load(Ordering::Acquire)); + } + + #[test] + fn serving_guard_normal_completion_is_silent() { + static TEST_SHUTDOWN: AtomicBool = AtomicBool::new(false); + TEST_SHUTDOWN.store(false, Ordering::Relaxed); + let (tx, rx) = mpsc::channel(); + + ServingGuard::new(WorkerIdx(3), tx, &TEST_SHUTDOWN).complete(); + assert!(rx.try_recv().is_err()); + assert!(!TEST_SHUTDOWN.load(Ordering::Acquire)); } #[test] diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 51337452f..18c3978c2 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -79,6 +79,7 @@ pub struct PreparedShardLayer { backing_keepalives: Vec>, rpc_shards: Vec, serve_start_txs: Vec>, + terminal_rx: mpsc::Receiver, } /// A spawned set of shard threads plus the handles to drive and retire @@ -106,6 +107,7 @@ pub struct ShardLayer { /// strictly after all joins, so no ring ever references unmapped /// memory. _backing_keepalives: Vec>, + terminal_rx: mpsc::Receiver, } impl ShardLayer { @@ -169,6 +171,7 @@ pub fn prepare_shard_layer( // from `ready_tx` so the layer can wait for the second rendezvous // (peer registration) after broadcasting the full peer set. let (phaseb_tx, phaseb_rx) = mpsc::channel::(); + let (terminal_tx, terminal_rx) = mpsc::channel::(); let mut joins = Vec::with_capacity(worker_count); let mut control_senders = Vec::with_capacity(worker_count); // Per-shard senders for broadcasting the assembled peer set in phase @@ -199,13 +202,13 @@ pub fn prepare_shard_layer( let frontend_bindings = frontend_bindings.clone(); let backend_specs = backend_specs.clone(); let layer_stop = layer_stop.clone(); + let terminal_tx = terminal_tx.clone(); let rt = deps.runtime.clone(); - let panic_tx = tx.clone(); let handle = rt.spawn_pinned( widx, &format!("ub-storage-shard-{i}"), Box::new(move || { - crate::report_on_panic(panic_tx, widx, move || { + crate::report_on_panic(widx, move || { crate::run_shard( widx, shard, @@ -222,6 +225,7 @@ pub fn prepare_shard_layer( peer_rx, phaseb_tx, serve_start_rx, + terminal_tx, layer_stop, ); }); @@ -234,6 +238,7 @@ pub fn prepare_shard_layer( // bounded by the number of shards that came up, and each live shard // holds its sender, so this never closes the channel prematurely. drop(phaseb_tx); + drop(terminal_tx); // Bounded readiness collection: read exactly one message per spawned // thread. Shards that come up park holding their sender, so they @@ -351,6 +356,7 @@ pub fn prepare_shard_layer( layer_stop, backing_keepalives, serve_start_txs, + terminal_rx, ); return Err(phaseb_errors); } @@ -363,6 +369,7 @@ pub fn prepare_shard_layer( backing_keepalives, rpc_shards, serve_start_txs, + terminal_rx, }) } @@ -378,6 +385,7 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result Result Result Result, backing_keepalives: Vec>, serve_start_txs: Vec>, + terminal_rx: mpsc::Receiver, ) { layer_stop.store(true, Ordering::Relaxed); drop(serve_start_txs); @@ -433,6 +445,7 @@ fn retire_failed_activation( drop(control); drop(fabric_group); drop(backing_keepalives); + drop(terminal_rx); } fn shard_backing_sizes( @@ -463,20 +476,30 @@ fn local_self_peer(projection: &config::RuntimeGraph) -> Result /// Retire a shard layer: signal its shards to exit, then join every /// thread in reverse spawn order so teardown mirrors bring-up. -pub fn teardown_shard_layer(layer: ShardLayer) { +pub fn teardown_shard_layer(layer: ShardLayer) -> bool { let ShardLayer { joins, fabric_group, control, layer_stop, _backing_keepalives, + terminal_rx, } = layer; layer_stop.store(true, Ordering::Relaxed); + let mut failed = false; for h in joins.into_iter().rev() { if let Err(e) = h.join() { eprintln!("shard thread panicked during teardown: {e:?}"); + failed = true; } } + for report in terminal_rx.try_iter() { + eprintln!( + "shard worker={} terminal failure: {}", + report.worker_idx.0, report.message + ); + failed = true; + } // Drop the control senders only after every shard thread has // exited, so nothing observes a half-torn layer. drop(control); @@ -493,6 +516,7 @@ pub fn teardown_shard_layer(layer: ShardLayer) { // joined, so no mapping is unmapped out from under a live ring or an // open MR. drop(_backing_keepalives); + failed } /// The binary's [`ConfigApplyTarget`]: owns the live shard layer and From 48c3203cecbdf3476419c2c9e2f51adb354228a9 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 03:04:51 +0000 Subject: [PATCH 44/56] unbounded-storage: preserve resources on reload failure --- cmd/unbounded-storage/ARCHITECTURE.md | 6 + cmd/unbounded-storage/src/config/reconcile.rs | 141 ++++++++++-------- cmd/unbounded-storage/src/main.rs | 39 +++-- 3 files changed, 117 insertions(+), 69 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index b9d54856d..57472e4c2 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -176,6 +176,12 @@ a terminal report on abnormal exit. That report also sets process-wide shutdown; teardown drains terminal reports and returns a failing process status instead of continuing with a missing shard. +Backend and frontend reloads construct replacements before swapping them into +their shard-local registries. A failed replacement therefore leaves the prior +resource, binding, and stripe geometry live and retryable. This is a +per-resource guarantee; route, peer, shard, and disk publication across the +whole process is not yet one transaction. + ### Shutdown A process-wide `static SHUTDOWN: AtomicBool` is set by an async-signal-safe diff --git a/cmd/unbounded-storage/src/config/reconcile.rs b/cmd/unbounded-storage/src/config/reconcile.rs index 3e7c13d55..a58e015d9 100644 --- a/cmd/unbounded-storage/src/config/reconcile.rs +++ b/cmd/unbounded-storage/src/config/reconcile.rs @@ -294,16 +294,15 @@ pub type FrontendReconcileReport = SpecReconcileReport; /// Drive `target` toward the backend set described by `desired`, /// keyed by component name. /// -/// The semantics mirror [`reconcile_peers`] exactly, specialized to -/// the string-keyed `BackendSpec`: removals first, then additions; an +/// The string-keyed registries can construct a replacement before +/// inserting it, so true removals run first while updates are applied +/// by adding the new resource over the old one. A /// name present in both `current` and `desired` whose spec differs from -/// `last_applied` is a remove-then-add ("update"); failures are +/// `last_applied` is an update; failures are /// accumulated and never abort the pass; the returned `applied` map is /// the set the target now holds and is the caller's input on the next -/// pass. The failure carry-forward rules match `reconcile_peers`: a -/// failed update-remove or plain remove preserves the old spec so the -/// next pass retries, while a failed update-add (remove succeeded) -/// drops the name so the next pass re-adds via the not-current branch. +/// pass. A failed removal or update preserves the old applied spec so +/// the next pass retries without taking the live resource offline. pub fn reconcile_backends( target: &dyn BackendReconcileTarget, desired: &[BackendSpec], @@ -347,9 +346,8 @@ pub fn reconcile_frontends( /// /// Factored out because the two specs differ only in their concrete /// type, not in the diff algorithm. A name whose desired spec is not -/// equal to its `last_applied` spec is treated as an update -/// (remove-then-add). The carry-forward rules are identical to -/// [`reconcile_peers`]. +/// equal to its `last_applied` spec is replaced by `add` while the old +/// resource remains installed. Targets must build before insertion. /// /// Internally this is the remove phase followed immediately by the add /// phase with a no-op gate, so its observable behavior is exactly the @@ -363,14 +361,14 @@ fn reconcile_specs( add: impl Fn(&S) -> Result<(), String>, remove: impl Fn(&str) -> Result<(), String>, ) -> SpecReconcileReport { - let state = reconcile_remove_phase(desired_map, last_applied, list, remove); + let state = reconcile_remove_phase(desired_map, last_applied, &HashSet::new(), list, remove); reconcile_add_phase(state, add, |_id, _spec| Ok(())) } /// Intermediate state carried from the remove phase into the add phase /// of a single string-keyed reconcile pass. Holds everything the add -/// phase needs to reproduce the original single-pass behavior: -/// `current`/`to_update`/`removed_ok` plus the in-progress report. +/// phase needs to complete the pass: current resources, pending updates, +/// successful true removals, and the in-progress report. struct SpecReconcileState<'a, S> { desired_map: &'a HashMap, last_applied: Option<&'a HashMap>, @@ -380,13 +378,13 @@ struct SpecReconcileState<'a, S> { report: SpecReconcileReport, } -/// Removal half of a string-keyed reconcile pass: compute drift, -/// remove ids that are no longer desired or that need a remove-then-add -/// update, and accumulate failures. Mirrors the first half of the old -/// monolithic `reconcile_specs`. +/// Removal half of a string-keyed reconcile pass: compute drift, remove +/// ids that are no longer desired, and accumulate failures. Updated ids +/// remain live until the add phase has built their replacement. fn reconcile_remove_phase<'a, S: Clone + PartialEq>( desired_map: &'a HashMap, last_applied: Option<&'a HashMap>, + forced_updates: &HashSet, list: impl Fn() -> Vec, remove: impl Fn(&str) -> Result<(), String>, ) -> SpecReconcileState<'a, S> { @@ -394,7 +392,7 @@ fn reconcile_remove_phase<'a, S: Clone + PartialEq>( let current: HashSet = list().into_iter().collect(); - let mut to_update: HashSet = HashSet::new(); + let mut to_update = forced_updates.clone(); if let Some(prev) = last_applied { for (id, new_spec) in desired_map { if !current.contains(id) { @@ -411,22 +409,18 @@ fn reconcile_remove_phase<'a, S: Clone + PartialEq>( let mut removed_ok: HashSet = HashSet::new(); let mut to_remove: Vec = current .iter() - .filter(|id| !desired_map.contains_key(*id) || to_update.contains(*id)) + .filter(|id| !desired_map.contains_key(*id)) .cloned() .collect(); to_remove.sort(); for id in to_remove { - let is_update = to_update.contains(&id); match remove(&id) { Ok(()) => { - if !is_update { - report.removed += 1; - } + report.removed += 1; removed_ok.insert(id); } Err(e) => { - let op = if is_update { "update-remove" } else { "remove" }; - report.failures.push((id, format!("{op}: {e}"))); + report.failures.push((id, format!("remove: {e}"))); } } } @@ -444,9 +438,7 @@ fn reconcile_remove_phase<'a, S: Clone + PartialEq>( /// Addition half of a string-keyed reconcile pass. `can_add` gates each /// candidate add: `Ok(())` proceeds, `Err(reason)` defers the id /// (recorded in `report.deferred`, never added, never counted). The -/// default no-op gate (`|_, _| Ok(())`) reproduces the original -/// single-pass behavior. Carry-forward is identical to -/// [`reconcile_peers`]. +/// default no-op gate (`|_, _| Ok(())`) permits every candidate. fn reconcile_add_phase( state: SpecReconcileState<'_, S>, add: impl Fn(&S) -> Result<(), String>, @@ -463,9 +455,7 @@ fn reconcile_add_phase( let mut to_add: Vec = desired_map .keys() - .filter(|id| { - !current.contains(*id) || (to_update.contains(*id) && removed_ok.contains(*id)) - }) + .filter(|id| !current.contains(*id) || to_update.contains(*id)) .cloned() .collect(); to_add.sort(); @@ -492,15 +482,11 @@ fn reconcile_add_phase( } } - // Carry-forward identical in spirit to `reconcile_peers`: ids the - // target still holds (in `current`, not successfully removed) that - // we have not already recorded. A still-desired, non-updated id - // records the freshest desired spec; otherwise the old spec is - // preserved so the next pass can retry the removal / re-detect the - // drift. A deferred id falls out exactly like a failed update-add: - // if it was an update its remove already landed (in `removed_ok`) - // so it is dropped here, and the next pass re-adds it once the gate - // opens. + // Carry forward resources still held by the target but not recorded + // by a successful add. Stable resources use the desired spec. Failed + // or deferred replacements retain the old spec so the next pass + // retries without taking the live resource offline. Failed removals + // likewise retain the old spec until removal succeeds. if let Some(prev) = last_applied { for (id, old_spec) in prev { if current.contains(id) && !removed_ok.contains(id) && !report.applied.contains_key(id) @@ -543,9 +529,9 @@ pub struct ResourceReconcileReport { /// referenced backend is present first), and /// 4. frontend additions run last and are *gated*: a frontend whose /// referenced backend is not present after steps 2-3 (never defined, -/// removed, or whose `add` failed) is deferred - recorded in -/// `frontends.deferred` and left unregistered - rather than added -/// dangling. The next pass re-adds it once the backend is present. +/// removed, or whose `add` failed) is deferred and recorded in +/// `frontends.deferred` rather than replaced with a dangling resource. +/// An existing frontend remains live; a new frontend remains absent. /// /// Each half preserves the exact drift/counter/carry-forward semantics /// of [`reconcile_backends`] / [`reconcile_frontends`]. @@ -555,6 +541,7 @@ pub fn reconcile_backends_and_frontends( desired_backends: &[BackendSpec], desired_frontends: &[FrontendSpec], frontend_backends: &HashMap, + forced_frontend_updates: &HashSet, last_backends: Option<&HashMap>, last_frontends: Option<&HashMap>, ) -> ResourceReconcileReport { @@ -571,6 +558,7 @@ pub fn reconcile_backends_and_frontends( let frontend_state = reconcile_remove_phase( &desired_frontend_map, last_frontends, + forced_frontend_updates, || frontend_target.list(), |id| frontend_target.remove(id), ); @@ -579,6 +567,7 @@ pub fn reconcile_backends_and_frontends( let backend_state = reconcile_remove_phase( &desired_backend_map, last_backends, + &HashSet::new(), || backend_target.list(), |id| backend_target.remove(id), ); @@ -590,9 +579,8 @@ pub fn reconcile_backends_and_frontends( |_id, _spec| Ok(()), ); - // 4. Frontend additions, gated on the referenced backend being - // present after steps 2-3. - let present_backends: HashSet = backend_target.list().into_iter().collect(); + // 4. Frontend additions and replacements, gated on the desired + // backend spec having applied successfully in steps 2-3. let frontends = reconcile_add_phase( frontend_state, |spec| frontend_target.add(spec), @@ -600,7 +588,9 @@ pub fn reconcile_backends_and_frontends( let Some(backend_id) = frontend_backends.get(id) else { return Err("deferred: frontend has no resolved backend".to_string()); }; - if present_backends.contains(backend_id) { + if desired_backend_map.get(backend_id).is_some() + && backends.applied.get(backend_id) == desired_backend_map.get(backend_id) + { Ok(()) } else { Err(format!( @@ -1127,10 +1117,7 @@ mod tests { assert_eq!(r.removed, 0); assert_eq!(r.updated, 1); assert!(r.failures.is_empty()); - assert_eq!( - *t.ops.borrow(), - vec![SpecOp::Remove("a".into()), SpecOp::Add("a".into())] - ); + assert_eq!(*t.ops.borrow(), vec![SpecOp::Add("a".into())]); assert_eq!(r.applied["a"].url(), Some("new-url")); } @@ -1165,17 +1152,17 @@ mod tests { } #[test] - fn reconcile_backends_update_remove_failure_preserves_old_spec() { - let t = SpecMock::new(&["a"]).with_remove_failure("a"); + fn reconcile_backends_update_add_failure_preserves_old_spec() { + let t = SpecMock::new(&["a"]).with_add_failure("a"); let mut prev = HashMap::new(); prev.insert("a".to_string(), backend("a", "old")); let desired = vec![backend("a", "new")]; let r = reconcile_backends(&t, &desired, Some(&prev)); assert_eq!(r.updated, 0); assert_eq!(r.failures.len(), 1); - assert!(r.failures[0].1.starts_with("update-remove:")); + assert!(r.failures[0].1.starts_with("update-add:")); assert_eq!(r.applied["a"].url(), Some("old")); - assert_eq!(*t.ops.borrow(), vec![SpecOp::Remove("a".into())]); + assert_eq!(*t.ops.borrow(), vec![SpecOp::Add("a".into())]); } #[test] @@ -1194,10 +1181,7 @@ mod tests { let desired2 = vec![frontend("f1", "b2")]; let r2 = reconcile_frontends(&t2, &desired2, Some(&prev)); assert_eq!(r2.updated, 1); - assert_eq!( - *t2.ops.borrow(), - vec![SpecOp::Remove("f1".into()), SpecOp::Add("f1".into())] - ); + assert_eq!(*t2.ops.borrow(), vec![SpecOp::Add("f1".into())]); assert_eq!(r2.applied["f1"].source, "b2"); // Remove: no longer desired. @@ -1294,6 +1278,7 @@ mod tests { &backends, &frontends, &frontend_backends, + &HashSet::new(), None, None, ); @@ -1330,6 +1315,7 @@ mod tests { &[], &[], &HashMap::new(), + &HashSet::new(), Some(&last_backends), Some(&last_frontends), ); @@ -1363,6 +1349,7 @@ mod tests { &backends, &frontends, &frontend_backends, + &HashSet::new(), None, None, ); @@ -1382,6 +1369,40 @@ mod tests { ); } + #[test] + fn combined_failed_backend_update_preserves_dependent_resources() { + let combo = ComboMock::new(&["b"], &["f"]).with_backend_add_failure("b"); + let mut last_backends = HashMap::new(); + last_backends.insert("b".to_string(), backend("b", "old")); + let mut last_frontends = HashMap::new(); + last_frontends.insert("f".to_string(), frontend("f", "b")); + let backends = vec![backend("b", "new")]; + let frontends = vec![frontend("f", "b")]; + let frontend_backends = frontend_backends(&frontends); + let forced_frontend_updates = HashSet::from(["f".to_string()]); + + let r = reconcile_backends_and_frontends( + &combo, + &combo, + &backends, + &frontends, + &frontend_backends, + &forced_frontend_updates, + Some(&last_backends), + Some(&last_frontends), + ); + + assert_eq!(r.backends.failures.len(), 1); + assert_eq!(r.backends.applied["b"].url(), Some("old")); + assert_eq!(r.frontends.deferred.len(), 1); + assert_eq!(r.frontends.applied["f"].source, "b"); + assert!(combo.backends.borrow().contains("b")); + assert!(combo.frontends.borrow().contains("f")); + assert!(combo.log_pos("backend-remove:b").is_none()); + assert!(combo.log_pos("frontend-remove:f").is_none()); + assert!(combo.log_pos("frontend-add:f").is_none()); + } + #[test] fn combined_defers_frontend_when_backend_absent() { // Backend referenced by the frontend is not defined at all. @@ -1394,6 +1415,7 @@ mod tests { &[], &frontends, &frontend_backends, + &HashSet::new(), None, None, ); @@ -1419,6 +1441,7 @@ mod tests { &backends, &frontends, &frontend_backends, + &HashSet::new(), Some(&last_backends), None, ); diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 8d7265259..81c6842e5 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -937,15 +937,8 @@ fn run_shard( &last_backends, desired_backends, ); - if !frontends_to_rebuild.is_empty() { - for id in frontends_to_rebuild { - let _ = config::reconcile::FrontendReconcileTarget::remove( - &frontend_registry, - &id, - ); - last_frontends.remove(&id); - } - } + let forced_frontend_updates: HashSet = + frontends_to_rebuild.into_iter().collect(); *bindings.borrow_mut() = projection.frontends.clone(); // Drive the transport backend registry and the @@ -957,12 +950,38 @@ fn run_shard( desired_backends, desired_frontends, &desired_frontend_backends, + &forced_frontend_updates, Some(&last_backends), Some(&last_frontends), ); last_backends = combined.backends.applied; last_frontends = combined.frontends.applied; - last_bindings = projection.frontends; + + // Replacement construction happens against desired + // geometry/bindings, then failed replacements retain + // their prior live resource and prior auxiliary state. + { + let mut g = geometry.borrow_mut(); + g.clear(); + for spec in last_backends.values() { + g.insert(spec.name.clone(), spec.stripe_size_bytes()); + } + } + let mut applied_bindings = HashMap::new(); + for (id, spec) in &last_frontends { + let desired_spec = desired_frontends.iter().find(|f| f.name == *id); + if desired_spec == Some(spec) { + if let Some(binding) = projection.frontends.get(id) { + applied_bindings.insert(id.clone(), binding.clone()); + continue; + } + } + if let Some(binding) = last_bindings.get(id) { + applied_bindings.insert(id.clone(), binding.clone()); + } + } + *bindings.borrow_mut() = applied_bindings.clone(); + last_bindings = applied_bindings; let mut failures = combined.backends.failures; failures.extend(combined.frontends.failures); From 6edc7c8325c78f76e7839d6daee5b4609bfaa898 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 03:21:18 +0000 Subject: [PATCH 45/56] unbounded-storage: share finalized config snapshots --- cmd/unbounded-storage/ARCHITECTURE.md | 22 ++- cmd/unbounded-storage/src/config/control.rs | 146 ++++++++-------- cmd/unbounded-storage/src/config/graph.rs | 85 ++++++++- cmd/unbounded-storage/src/config/load.rs | 161 ++++++++++++----- cmd/unbounded-storage/src/config/mod.rs | 2 +- cmd/unbounded-storage/src/config/watch.rs | 12 +- cmd/unbounded-storage/src/main.rs | 162 ++++-------------- cmd/unbounded-storage/src/shard_layer.rs | 50 +++--- .../tests/lifecycle/workload.rs | 43 +++-- 9 files changed, 376 insertions(+), 307 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 57472e4c2..285c159f9 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -128,8 +128,9 @@ excluded from the live-reload diff. ### Startup sequence -1. Load and validate config; build `StartupSettings` from the config's - `[startup]` section. +1. Load and validate config into one immutable `LoadedConfig` containing the raw + config, owned runtime graph, and route snapshot; build `StartupSettings` from + its raw config's `[startup]` section. 2. `Host::discover()` reads hardware from sysfs. 3. `CorePlan::for_host(&host, &settings.core_plan_config)` partitions the host's usable CPUs into three disjoint, NUMA-local classes: one @@ -144,11 +145,10 @@ excluded from the live-reload diff. i-th worker thread to its assigned core and NUMA node. 6. A `DiskChannelDirectory` (Arc) is created before shards as the hot-swap publication surface for disk channels. -7. Read-only shared state (`Arc>`, `Arc>`; - startup-fixed fabric settings come from the config `[startup]` section via - `StartupSettings`) and routing (`build_routing` -> `Arc`) are - constructed once and shared across shards. `PeerId` uses the same stable - numeric identity as `NodeId`. +7. One `Arc` shares the raw config, owned runtime graph, and route + snapshot across startup consumers and shards. Startup-fixed fabric settings + still come from the raw `[startup]` section via `StartupSettings`. `PeerId` + uses the same stable numeric identity as `NodeId`. 8. Each shard is spawned with `rt.spawn_pinned(widx, name, Box)`. The `!Send` shard objects are constructed **inside** `run_shard`, after pinning. 9. After every shard reports `Up`, peers are reconciled per shard and every @@ -639,11 +639,15 @@ Sections (all optional, each falling back to defaults): - `[[frontends]]` - `name`, `source` (a backend or cache component name), and one `config` table (`http`, `s3`, or `loadgen`). -The watcher (`notify`-based) emits `ConfigUpdate`s; main's +After defaults and validation, loading constructs one immutable `LoadedConfig`: +the raw `Config`, one owned `RuntimeGraph`, and one route snapshot built from +that graph. The watcher (`notify`-based) emits these loaded snapshots; main's `wait_for_shutdown_with_updates` reconciles peers (remove + add on address/numa drift, via a `last_applied` cache), disks, and - by broadcasting the applied config to every shard - each shard's backend and frontend registries plus the -routing snapshot. It republishes the channel snapshot each update, logs +routing snapshot. The apply target and shards consume the same loaded graph and +routes; this is coherent preparation, not whole-process transactionality. It +republishes the channel snapshot each update, logs `config gen=N ...`, and sets `SHUTDOWN` if the watcher disconnects. ### 7.12 `tls/` - the shared TLS transport diff --git a/cmd/unbounded-storage/src/config/control.rs b/cmd/unbounded-storage/src/config/control.rs index ec0a6377d..b992dfba2 100644 --- a/cmd/unbounded-storage/src/config/control.rs +++ b/cmd/unbounded-storage/src/config/control.rs @@ -37,8 +37,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc::{self, RecvTimeoutError, Sender}; use std::time::{Duration, Instant}; -use crate::config::{Config, ConfigDiff}; -use crate::p2p::RouteTableSnapshot; +use crate::config::{ConfigDiff, LoadedConfig}; use crate::runtime::WorkerIdx; /// A command delivered to a single shard's control channel. The shard @@ -64,11 +63,8 @@ pub struct ShardDrainPageCache { /// Payload of [`ShardCommand::ApplyConfig`]. pub struct ShardApply { - /// The new, defaults-applied, validated configuration. - pub config: Arc, - /// The route table rebuilt from `config`. Published into the shard's - /// route-table handle so transports observe it atomically. - pub routes: RouteTableSnapshot, + /// The new finalized config, runtime graph, and route table. + pub loaded: Arc, /// Section-level diff that selected this apply path. pub diff: ConfigDiff, /// Channel the shard sends its [`ShardAck`] on once the apply has @@ -287,7 +283,11 @@ pub trait ConfigApplyTarget { /// Apply a change in place. `diff` reports exactly which sections /// changed so the implementation can skip untouched work (e.g. only /// reconcile fabric peers when `requires_routing_reload`). - fn apply_in_place(&mut self, new: &Arc, diff: &ConfigDiff) -> Result<(), ApplyError>; + fn apply_in_place( + &mut self, + new: &Arc, + diff: &ConfigDiff, + ) -> Result<(), ApplyError>; } /// The single funnel for configuration changes. Holds the currently @@ -295,7 +295,7 @@ pub trait ConfigApplyTarget { /// [`ConfigApplyTarget`]. pub struct ConfigController { target: T, - current: Arc, + current: Arc, versions: ConfigVersionStatus, } @@ -304,8 +304,8 @@ impl ConfigController { /// to the process at startup. The latest-known, latest-applied, and /// startup config versions are all seeded from `initial`'s /// [`Config::version`]. - pub fn new(target: T, initial: Arc) -> Self { - let versions = ConfigVersionStatus::new(initial.version); + pub fn new(target: T, initial: Arc) -> Self { + let versions = ConfigVersionStatus::new(initial.config().version); Self { target, current: initial, @@ -314,7 +314,7 @@ impl ConfigController { } /// The configuration currently realized by the process. - pub fn current(&self) -> &Arc { + pub fn current(&self) -> &Arc { &self.current } @@ -354,15 +354,15 @@ impl ConfigController { /// latest-applied config version is advanced to the same value only on /// full success - including a no-op apply against the already-running /// config - so observers can tell the process has converged onto it. - pub fn apply(&mut self, new: Arc) -> Result { + pub fn apply(&mut self, new: Arc) -> Result { // Record the loaded version before doing any work so the // latest-known version advances even if the apply fails below. - self.versions.record_known(new.version); + self.versions.record_known(new.config().version); - let diff = ConfigDiff::between(&self.current, &new); + let diff = ConfigDiff::between(self.current.config(), new.config()); if !diff.any() { - self.versions.record_applied(new.version); + self.versions.record_applied(new.config().version); return Ok(ApplyOutcome { tier: ApplyTier::NoChange, diff, @@ -371,7 +371,7 @@ impl ConfigController { self.target.apply_in_place(&new, &diff)?; - self.versions.record_applied(new.version); + self.versions.record_applied(new.config().version); self.current = new; Ok(ApplyOutcome { tier: ApplyTier::InPlace, @@ -411,7 +411,7 @@ impl ShardControlGroup { self.senders.is_empty() } - /// Send `config` + `routes` to every shard and block until all of + /// Send one loaded snapshot to every shard and block until all of /// them acknowledge. /// /// Returns `Ok(())` only when every shard reports success. If a @@ -422,14 +422,12 @@ impl ShardControlGroup { /// shard does not mask failures from the others. pub fn broadcast_apply( &self, - config: Arc, - routes: RouteTableSnapshot, + loaded: Arc, diff: ConfigDiff, ) -> Result<(), ApplyError> { self.broadcast(|ack| { ShardCommand::ApplyConfig(ShardApply { - config: config.clone(), - routes: routes.clone(), + loaded: loaded.clone(), diff, ack, }) @@ -528,10 +526,9 @@ mod tests { use std::time::Duration; use super::*; - use crate::p2p::RouteTableSnapshot; - - fn empty_routes() -> RouteTableSnapshot { - RouteTableSnapshot::default() + use crate::config::Config; + fn loaded(config: Config) -> Arc { + Arc::new(LoadedConfig::from_config(config).unwrap()) } fn control_group(senders: Vec<(WorkerIdx, Sender)>) -> ShardControlGroup { @@ -591,11 +588,7 @@ mod tests { let (senders, joins) = spawn_mock_shards(vec![Ok(()), Ok(()), Ok(())]); let group = control_group(senders); - let out = group.broadcast_apply( - Arc::new(Config::default()), - empty_routes(), - ConfigDiff::default(), - ); + let out = group.broadcast_apply(loaded(Config::default()), ConfigDiff::default()); assert!(out.is_ok(), "expected success, got {out:?}"); // Closing the group drops the senders so the mock shard threads @@ -606,17 +599,39 @@ mod tests { } } + #[test] + fn broadcast_apply_delivers_the_same_loaded_snapshot() { + let (tx, rx) = mpsc::channel::(); + let group = control_group(vec![(WorkerIdx(0), tx)]); + let loaded = loaded(Config::default()); + let expected = loaded.clone(); + let join = thread::spawn(move || { + let ShardCommand::ApplyConfig(apply) = rx.recv().unwrap() else { + panic!("expected config apply"); + }; + assert!(Arc::ptr_eq(&apply.loaded, &expected)); + apply + .ack + .send(ShardAck { + worker: WorkerIdx(0), + result: Ok(()), + }) + .unwrap(); + }); + + group + .broadcast_apply(loaded, ConfigDiff::default()) + .unwrap(); + join.join().unwrap(); + } + #[test] fn broadcast_apply_reports_shard_failures() { let (senders, joins) = spawn_mock_shards(vec![Ok(()), Err("boom".to_string()), Ok(())]); let group = control_group(senders); let err = group - .broadcast_apply( - Arc::new(Config::default()), - empty_routes(), - ConfigDiff::default(), - ) + .broadcast_apply(loaded(Config::default()), ConfigDiff::default()) .expect_err("a failing shard must surface as an error"); match err { ApplyError::ShardApply(failures) => { @@ -639,11 +654,7 @@ mod tests { assert!(group.is_empty()); assert!( group - .broadcast_apply( - Arc::new(Config::default()), - empty_routes(), - ConfigDiff::default(), - ) + .broadcast_apply(loaded(Config::default()), ConfigDiff::default(),) .is_ok() ); } @@ -658,11 +669,7 @@ mod tests { let group = control_group(senders); let err = group - .broadcast_apply( - Arc::new(Config::default()), - empty_routes(), - ConfigDiff::default(), - ) + .broadcast_apply(loaded(Config::default()), ConfigDiff::default()) .expect_err("closed channel must error"); assert!(matches!(err, ApplyError::ShardSend(WorkerIdx(99)))); assert!(err.apply_state_is_indeterminate()); @@ -708,11 +715,7 @@ mod tests { ); let error = group - .broadcast_apply( - Arc::new(Config::default()), - empty_routes(), - ConfigDiff::default(), - ) + .broadcast_apply(loaded(Config::default()), ConfigDiff::default()) .expect_err("missing acknowledgement must time out"); assert!(matches!( error, @@ -800,7 +803,7 @@ mod tests { impl ConfigApplyTarget for RecordingTarget { fn apply_in_place( &mut self, - _new: &Arc, + _new: &Arc, diff: &ConfigDiff, ) -> Result<(), ApplyError> { self.last_diff = Some(*diff); @@ -830,6 +833,10 @@ mod tests { c } + fn loaded_with_peer(version: u64) -> Arc { + loaded(config_with_peer(version)) + } + fn tcp_peer(name: &str, addr: &str) -> crate::config::PeerSpec { crate::config::PeerSpec { name: name.to_string(), @@ -844,7 +851,7 @@ mod tests { #[test] fn apply_no_change_runs_neither_tier() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let mut ctrl = ConfigController::new(RecordingTarget::default(), base.clone()); assert_eq!( ctrl.config_versions().snapshot(), @@ -860,7 +867,7 @@ mod tests { // advances both known and applied: the process is converged on it. let mut bumped = config_with_peer(1); bumped.version = 2; - let out = ctrl.apply(Arc::new(bumped)).unwrap(); + let out = ctrl.apply(loaded(bumped)).unwrap(); assert_eq!(out.tier, ApplyTier::NoChange); assert_eq!(ctrl.target_mut().in_place, 0); assert_eq!(ctrl.config_versions().known(), 2); @@ -872,13 +879,13 @@ mod tests { #[test] fn apply_peer_change_takes_in_place_tier() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let mut ctrl = ConfigController::new(RecordingTarget::default(), base.clone()); let mut next = config_with_peer(2); next.peers.push(tcp_peer("node-b", "127.0.0.1:9998")); - let out = ctrl.apply(Arc::new(next)).unwrap(); + let out = ctrl.apply(loaded(next)).unwrap(); assert_eq!(out.tier, ApplyTier::InPlace); assert_eq!(ctrl.target_mut().in_place, 1); assert!( @@ -892,11 +899,11 @@ mod tests { #[test] fn apply_backend_change_takes_in_place_tier() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let mut ctrl = ConfigController::new(RecordingTarget::default(), base.clone()); let mut next = config_with_peer(3); - next.backends.push(crate::config::schema::BackendSpec { + next.backends[0] = crate::config::schema::BackendSpec { name: "b".to_string(), config: Some(crate::config::backend_spec::Config::Http( crate::config::HttpBackendConfig { @@ -909,12 +916,12 @@ mod tests { client_key_path: None, }, )), - }); + }; // A backend change is now reconciled in place on the live shard // layer (each shard rebuilds its origin-backend registry from the // broadcast config); it no longer rebuilds the shard layer. - let out = ctrl.apply(Arc::new(next)).unwrap(); + let out = ctrl.apply(loaded(next)).unwrap(); assert_eq!(out.tier, ApplyTier::InPlace); assert_eq!(ctrl.target_mut().in_place, 1); assert!(ctrl.target_mut().last_diff.unwrap().backends_changed); @@ -923,7 +930,7 @@ mod tests { #[test] fn failed_apply_leaves_current_config_unchanged() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let target = RecordingTarget { fail_in_place: true, ..RecordingTarget::default() @@ -932,12 +939,15 @@ mod tests { let mut next = config_with_peer(5); next.peers.push(tcp_peer("node-b", "127.0.0.1:9998")); - let next = Arc::new(next); + let next = loaded(next); assert!(ctrl.apply(next.clone()).is_err()); // Current must still be the original so a retry re-derives the // same diff. - assert_eq!(ctrl.current().peers.len(), base.peers.len()); + assert_eq!( + ctrl.current().config().peers.len(), + base.config().peers.len() + ); // A failed apply records the version as known (we loaded it) but // must NOT advance the applied version: the process did not // converge on the submitted config. @@ -947,7 +957,7 @@ mod tests { #[test] fn config_version_handle_observes_later_applies() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let mut ctrl = ConfigController::new(RecordingTarget::default(), base.clone()); // A handle taken early must observe applies that happen later, @@ -964,7 +974,7 @@ mod tests { let mut next = config_with_peer(11); next.peers.push(tcp_peer("node-b", "127.0.0.1:9998")); - ctrl.apply(Arc::new(next)).unwrap(); + ctrl.apply(loaded(next)).unwrap(); assert_eq!(versions.known(), 11); assert_eq!(versions.applied(), 11); @@ -980,7 +990,7 @@ mod tests { #[test] fn startup_version_is_pinned_across_applies_and_failures() { - let base = Arc::new(config_with_peer(1)); + let base = loaded_with_peer(1); let target = RecordingTarget { fail_in_place: true, ..RecordingTarget::default() @@ -991,7 +1001,7 @@ mod tests { // A failed apply advances known but neither applied nor startup. let mut failing = config_with_peer(7); failing.peers.push(tcp_peer("node-b", "127.0.0.1:9998")); - assert!(ctrl.apply(Arc::new(failing)).is_err()); + assert!(ctrl.apply(loaded(failing)).is_err()); assert_eq!(ctrl.config_versions().known(), 7); assert_eq!(ctrl.config_versions().applied(), 1); assert_eq!(ctrl.config_versions().startup(), 1); @@ -1001,7 +1011,7 @@ mod tests { ctrl.target_mut().fail_in_place = false; let mut next = config_with_peer(8); next.peers.push(tcp_peer("node-c", "127.0.0.1:9997")); - ctrl.apply(Arc::new(next)).unwrap(); + ctrl.apply(loaded(next)).unwrap(); assert_eq!(ctrl.config_versions().applied(), 8); assert_eq!(ctrl.config_versions().startup(), 1); } diff --git a/cmd/unbounded-storage/src/config/graph.rs b/cmd/unbounded-storage/src/config/graph.rs index cf8bc4a11..ac27b9bb8 100644 --- a/cmd/unbounded-storage/src/config/graph.rs +++ b/cmd/unbounded-storage/src/config/graph.rs @@ -7,7 +7,10 @@ use std::collections::{HashMap, HashSet}; use super::schema::{Config, DiskSpec, PeerSpec, RoutingPlan, TopologyWeighting}; use crate::fabric::PeerId; -use crate::p2p::{NodeId, node_id_from_name}; +use crate::p2p::{ + FingerTable, FingerTableConfig, NodeId, PeerEntry, RouteTableSnapshot, TopologyPrefixWeight, + TopologySelection, TopologyTags, node_id_from_name, node_to_ring, +}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolvedFrontendBinding { @@ -124,7 +127,10 @@ fn validate_peer_names(config: &Config) -> Result<(), String> { pub fn runtime_projection(config: &Config) -> Result { validate_binding_graph(config)?; + Ok(project_runtime(config)) +} +pub(crate) fn project_runtime(config: &Config) -> RuntimeGraph { let backends = by_id(&config.backends, |b| b.name.as_str()); let caches = by_id(&config.caches, |c| c.name.as_str()); @@ -191,7 +197,7 @@ pub fn runtime_projection(config: &Config) -> Result { }) .collect(); - Ok(RuntimeGraph { + RuntimeGraph { disks: config.disks.clone(), caches, mesh: RuntimeMesh { @@ -205,7 +211,80 @@ pub fn runtime_projection(config: &Config) -> Result { peers, }, frontends: bindings, - }) + } +} + +pub(crate) fn route_snapshot(graph: &RuntimeGraph) -> RouteTableSnapshot { + if graph.caches.is_empty() { + return RouteTableSnapshot::default(); + } + + let mesh = &graph.mesh; + let local = PeerEntry { + node: mesh.self_node_id, + ring: node_to_ring(mesh.self_node_id), + tags: TopologyTags(mesh.self_tags.clone()), + }; + let fingers = if let Some(plan) = &mesh.routing_plan { + let peer_by_name: HashMap<&str, &RuntimePeer> = mesh + .peers + .iter() + .map(|peer| (peer.name.as_str(), peer)) + .collect(); + let entry_of = |name: &str| { + let peer = peer_by_name + .get(name) + .expect("validated routing_plan names reference peers"); + PeerEntry { + node: peer.node_id, + ring: node_to_ring(peer.node_id), + tags: TopologyTags(peer.spec.tags.clone()), + } + }; + FingerTable::from_explicit( + local, + plan.fingers.iter().map(|name| entry_of(name)).collect(), + plan.successor.as_deref().map(entry_of), + plan.predecessor.as_deref().map(entry_of), + ) + } else { + let peers: Vec = mesh + .peers + .iter() + .map(|peer| PeerEntry { + node: peer.node_id, + ring: node_to_ring(peer.node_id), + tags: TopologyTags(peer.spec.tags.clone()), + }) + .collect(); + FingerTable::build( + local, + &peers, + FingerTableConfig { + k: mesh.fingers_per_node.max(1), + topology: mesh + .topology_weighting + .as_ref() + .map(|weighting| crate::p2p::TopologyWeighting { + prefix_weights: weighting + .prefix_weights + .iter() + .map(|weight| TopologyPrefixWeight { + tag_index: weight.tag_index, + weight: weight.weight, + }) + .collect(), + }) + .map(TopologySelection::Weighted) + .unwrap_or_default(), + }, + ) + }; + + RouteTableSnapshot { + cache_ids: graph.caches.keys().cloned().collect(), + fingers: Some(std::sync::Arc::new(fingers)), + } } pub fn runtime_disks(graph: &RuntimeGraph) -> Vec { diff --git a/cmd/unbounded-storage/src/config/load.rs b/cmd/unbounded-storage/src/config/load.rs index 6bd1f2a46..06784f112 100644 --- a/cmd/unbounded-storage/src/config/load.rs +++ b/cmd/unbounded-storage/src/config/load.rs @@ -25,8 +25,61 @@ use std::path::Path; use prost::Message; -use super::graph::runtime_projection; +use super::graph::{RuntimeGraph, project_runtime, route_snapshot, validate_binding_graph}; use super::schema::{Config, backend_spec, disk_spec, frontend_spec, peer_spec}; +use crate::p2p::RouteTableSnapshot; + +/// One finalized configuration and every immutable runtime view derived from it. +pub struct LoadedConfig { + config: Config, + runtime: RuntimeGraph, + routes: RouteTableSnapshot, +} + +impl fmt::Debug for LoadedConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LoadedConfig") + .field("config", &self.config) + .field("runtime", &self.runtime) + .field("route_cache_ids", &self.routes.cache_ids) + .field("has_fingers", &self.routes.fingers.is_some()) + .finish() + } +} + +impl LoadedConfig { + pub fn load(path: impl AsRef) -> Result { + load(path.as_ref()) + } + + pub fn from_config(mut config: Config) -> Result { + config.apply_defaults(); + validate(&config)?; + let runtime = project_runtime(&config); + let routes = route_snapshot(&runtime); + Ok(Self { + config, + runtime, + routes, + }) + } + + pub fn config(&self) -> &Config { + &self.config + } + + pub fn runtime(&self) -> &RuntimeGraph { + &self.runtime + } + + pub fn routes(&self) -> &RouteTableSnapshot { + &self.routes + } + + pub fn into_config(self) -> Config { + self.config + } +} #[derive(Debug)] pub enum ConfigError { @@ -284,22 +337,20 @@ impl From for ConfigError { impl Config { pub fn load(path: impl AsRef) -> Result { - load(path.as_ref()) + LoadedConfig::load(path).map(LoadedConfig::into_config) } } /// Loads a config from `path`, decoding raw binary protobuf for a /// `.binpb` extension and strict TOML for anything else. Both encodings /// share the same defaulting and validation finalization. -pub fn load(path: &Path) -> Result { - let mut cfg = if has_binpb_extension(path) { +pub fn load(path: &Path) -> Result { + let cfg = if has_binpb_extension(path) { Config::decode(fs::read(path)?.as_slice())? } else { toml::from_str(&fs::read_to_string(path)?)? }; - cfg.apply_defaults(); - validate(&cfg)?; - Ok(cfg) + LoadedConfig::from_config(cfg) } fn has_binpb_extension(path: &Path) -> bool { @@ -407,7 +458,7 @@ fn validate(cfg: &Config) -> Result<(), ConfigError> { } } - runtime_projection(cfg).map_err(ConfigError::InvalidBindingGraph)?; + validate_binding_graph(cfg).map_err(ConfigError::InvalidBindingGraph)?; // The metrics exporter addr is optional; when set it must parse as a // socket address (an empty value disables the exporter). @@ -715,6 +766,7 @@ source = "b" fn loads_minimal_config() { let f = write_cfg(""); let cfg = load(f.path()).unwrap(); + let cfg = cfg.config(); assert!(cfg.peers.is_empty()); assert!(cfg.caches.is_empty()); } @@ -763,12 +815,14 @@ addr = "0.0.0.0:9000" "#; let f = write_cfg(s); let cfg = load(f.path()).unwrap(); - assert_eq!(cfg.peers.len(), 2); - assert_eq!(cfg.disks.len(), 2); - assert_eq!(cfg.self_, "node-a"); - let projection = runtime_projection(&cfg).unwrap(); - assert!(!projection.frontends["f"].bypass_cache); - assert_eq!(projection.frontends["f"].backend_id, "b"); + let raw = cfg.config(); + assert_eq!(raw.peers.len(), 2); + assert_eq!(raw.disks.len(), 2); + assert_eq!(raw.self_, "node-a"); + assert!(!cfg.runtime().frontends["f"].bypass_cache); + assert_eq!(cfg.runtime().frontends["f"].backend_id, "b"); + assert!(cfg.routes().cache_ids.contains("c")); + assert!(cfg.routes().fingers.is_some()); } #[test] @@ -935,6 +989,7 @@ addr = "hex:01020304" let f = write_cfg(&s); let cfg = load(f.path()).expect("load should succeed"); let peer = cfg + .config() .peers .iter() .find(|peer| peer.name == "node-rdma") @@ -960,6 +1015,7 @@ addr = "10.0.0.2:5000" let f = write_cfg(&s); let cfg = load(f.path()).expect("load should succeed"); let peer = cfg + .config() .peers .iter() .find(|peer| peer.name == "node-rdma") @@ -986,6 +1042,7 @@ addrs = ["hex:01020304", "10.0.0.2:5000"] let f = write_cfg(&s); let cfg = load(f.path()).expect("load should succeed"); let peer = cfg + .config() .peers .iter() .find(|peer| peer.name == "node-rdma") @@ -1078,8 +1135,8 @@ size = 16777216 ); let f = write_cfg(&s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.disks[0].kind_name(), "file"); - assert_eq!(cfg.disks[0].file_size(), Some(16 * 1024 * 1024)); + assert_eq!(cfg.config().disks[0].kind_name(), "file"); + assert_eq!(cfg.config().disks[0].file_size(), Some(16 * 1024 * 1024)); } #[test] @@ -1205,8 +1262,8 @@ addr = "10.0.0.1:9000" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.self_, "node-a"); - assert_eq!(cfg.peers.len(), 1); + assert_eq!(cfg.config().self_, "node-a"); + assert_eq!(cfg.config().peers.len(), 1); } #[test] @@ -1250,8 +1307,8 @@ addr = "0.0.0.0:9000" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.backends.len(), 1); - assert_eq!(cfg.frontends.len(), 1); + assert_eq!(cfg.config().backends.len(), 1); + assert_eq!(cfg.config().frontends.len(), 1); } #[test] @@ -1357,8 +1414,12 @@ name = "synthetic" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("fake backend without url should load"); - assert_eq!(cfg.backends[0].name, "synthetic"); - match cfg.backends[0].config.as_ref().expect("backend config set") { + assert_eq!(cfg.config().backends[0].name, "synthetic"); + match cfg.config().backends[0] + .config + .as_ref() + .expect("backend config set") + { backend_spec::Config::Fake(fake) => { assert_eq!(fake.object_size_bytes, Some(1024 * 1024)); } @@ -1499,7 +1560,10 @@ stripe_size_bytes = 8388608 "#; let f = write_cfg(s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.backends[0].stripe_size_bytes(), 8 * 1024 * 1024); + assert_eq!( + cfg.config().backends[0].stripe_size_bytes(), + 8 * 1024 * 1024 + ); } #[test] @@ -1578,14 +1642,14 @@ addr = "example.com:9000" fn accepts_valid_metrics_addr() { let f = write_cfg("[startup.metrics]\naddr = \"0.0.0.0:9100\"\n"); let cfg = load(f.path()).expect("valid metrics addr loads"); - assert_eq!(cfg.startup().metrics().addr, "0.0.0.0:9100"); + assert_eq!(cfg.config().startup().metrics().addr, "0.0.0.0:9100"); } #[test] fn empty_metrics_addr_is_allowed() { let f = write_cfg(""); let cfg = load(f.path()).expect("absent metrics section loads"); - assert_eq!(cfg.startup().metrics().addr, ""); + assert_eq!(cfg.config().startup().metrics().addr, ""); } #[test] @@ -1648,8 +1712,8 @@ read_bytes = 65536 "#; let f = write_cfg(s); let cfg = load(f.path()).expect("loadgen frontend without addr should load"); - assert_eq!(cfg.frontends[0].kind_name(), "loadgen"); - assert_eq!(cfg.frontends[0].addr(), None); + assert_eq!(cfg.config().frontends[0].kind_name(), "loadgen"); + assert_eq!(cfg.config().frontends[0].addr(), None); } #[test] @@ -1681,7 +1745,7 @@ addr = "0.0.0.0:9000" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("empty loadgen addrs should not collide"); - assert_eq!(cfg.frontends.len(), 3); + assert_eq!(cfg.config().frontends.len(), 3); } #[test] @@ -1695,8 +1759,11 @@ url = "https://s3.example.com:443" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.backends[0].kind_name(), "s3"); - assert_eq!(cfg.backends[0].url(), Some("https://s3.example.com:443")); + assert_eq!(cfg.config().backends[0].kind_name(), "s3"); + assert_eq!( + cfg.config().backends[0].url(), + Some("https://s3.example.com:443") + ); } #[test] @@ -1710,7 +1777,7 @@ url = "https://acct.blob.core.windows.net:443" "#; let f = write_cfg(s); let cfg = load(f.path()).expect("load should succeed"); - assert_eq!(cfg.backends[0].kind_name(), "azure"); + assert_eq!(cfg.config().backends[0].kind_name(), "azure"); } #[test] @@ -1795,10 +1862,10 @@ addr = "0.0.0.0:9000" let cfg: Config = toml::from_str(toml).unwrap(); let f = write_binpb(&encode_config(&cfg)); let loaded = load(f.path()).unwrap(); - assert_eq!(loaded.peers.len(), 1); - assert_eq!(loaded.peers[0].name, "node-a"); - assert_eq!(loaded.disks.len(), 1); - assert_eq!(loaded.self_, "node-a"); + assert_eq!(loaded.config().peers.len(), 1); + assert_eq!(loaded.config().peers[0].name, "node-a"); + assert_eq!(loaded.config().disks.len(), 1); + assert_eq!(loaded.config().self_, "node-a"); } #[test] @@ -1806,9 +1873,9 @@ addr = "0.0.0.0:9000" // The binpb path shares the TOML path's defaulting finalization. let f = write_binpb(&encode_config(&Config::default())); let loaded = load(f.path()).unwrap(); - assert!(loaded.peers.is_empty()); + assert!(loaded.config().peers.is_empty()); assert_eq!( - loaded.startup().fabric().default_listen_addr(), + loaded.config().startup().fabric().default_listen_addr(), Some("0.0.0.0:0") ); } @@ -1864,7 +1931,7 @@ addr = "10.0.0.2:9000" // documented defaults during load. let f = write_cfg(""); let cfg = load(f.path()).unwrap(); - let s = cfg.startup(); + let s = cfg.config().startup(); assert_eq!(s.memory().memory_total_bytes, Some(128 * 1024 * 1024)); assert_eq!(s.fabric().default_listen_addr(), Some("0.0.0.0:0")); assert_eq!(s.fabric().max_inflight, Some(1024)); @@ -1888,18 +1955,18 @@ disable_rdma = true "#; let f = write_cfg(s); let cfg = load(f.path()).unwrap(); - assert!(cfg.startup().memory().no_hugepages); + assert!(cfg.config().startup().memory().no_hugepages); assert_eq!( - cfg.startup().memory().memory_total_bytes, + cfg.config().startup().memory().memory_total_bytes, Some(64 * 1024 * 1024) ); assert_eq!( - cfg.startup().fabric().default_listen_addr(), + cfg.config().startup().fabric().default_listen_addr(), Some("10.0.0.1:7000") ); - assert!(cfg.startup().topology().disable_rdma); + assert!(cfg.config().startup().topology().disable_rdma); // Unset siblings still default. - assert_eq!(cfg.startup().fabric().progress_threads, Some(2)); + assert_eq!(cfg.config().startup().fabric().progress_threads, Some(2)); } #[test] @@ -1920,13 +1987,13 @@ disable_rdma = true let f = write_binpb(&encode_config(&cfg)); let loaded = load(f.path()).unwrap(); assert_eq!( - loaded.startup().fabric().default_listen_addr(), + loaded.config().startup().fabric().default_listen_addr(), Some("10.0.0.2:8000") ); - assert_eq!(loaded.startup().fabric().max_inflight, Some(4096)); - assert!(loaded.startup().topology().disable_rdma); + assert_eq!(loaded.config().startup().fabric().max_inflight, Some(4096)); + assert!(loaded.config().startup().topology().disable_rdma); assert_eq!( - loaded.startup().memory().memory_total_bytes, + loaded.config().startup().memory().memory_total_bytes, Some(128 * 1024 * 1024) ); } diff --git a/cmd/unbounded-storage/src/config/mod.rs b/cmd/unbounded-storage/src/config/mod.rs index 606778646..5e74be7c5 100644 --- a/cmd/unbounded-storage/src/config/mod.rs +++ b/cmd/unbounded-storage/src/config/mod.rs @@ -23,7 +23,7 @@ pub use graph::{ ResolvedFrontendBinding, RuntimeCache, RuntimeGraph, RuntimeMesh, RuntimePeer, frontend_backend_map, runtime_disks, runtime_peers, runtime_projection, validate_binding_graph, }; -pub use load::ConfigError; +pub use load::{ConfigError, LoadedConfig}; pub use reconcile::{ ApplyReport, BackendReconcileReport, BackendReconcileTarget, FrontendReconcileReport, FrontendReconcileTarget, PeerReconcileTarget, ReconcileReport, SpecReconcileReport, diff --git a/cmd/unbounded-storage/src/config/watch.rs b/cmd/unbounded-storage/src/config/watch.rs index 66a9a1f90..6c4c1ce7f 100644 --- a/cmd/unbounded-storage/src/config/watch.rs +++ b/cmd/unbounded-storage/src/config/watch.rs @@ -25,13 +25,13 @@ use std::time::Duration; use notify::{RecommendedWatcher, RecursiveMode, Watcher}; use super::load; -use super::schema::Config; +use super::load::LoadedConfig; const DEBOUNCE: Duration = Duration::from_millis(200); #[derive(Debug)] pub struct ConfigUpdate { - pub config: Arc, + pub loaded: Arc, pub generation: u64, } @@ -161,10 +161,10 @@ fn debounce_loop( } } match load::load(Path::new(&path)) { - Ok(cfg) => { + Ok(loaded) => { generation = generation.wrapping_add(1); let update = ConfigUpdate { - config: Arc::new(cfg), + loaded: Arc::new(loaded), generation, }; if update_tx.send(update).is_err() { @@ -374,7 +374,7 @@ name = "b" write(&path, VALID_B); let update = recv_within(&rx, Duration::from_secs(3)) .expect("a valid modification must yield a ConfigUpdate"); - assert_eq!(update.config.fingers_per_node, Some(5678)); + assert_eq!(update.loaded.config().fingers_per_node, Some(5678)); assert!(update.generation >= 1, "generation must advance"); // (2a) An unrelated sibling write is filtered out. @@ -390,7 +390,7 @@ name = "b" write(&path, VALID_A); let update = recv_within(&rx, Duration::from_secs(3)) .expect("a config change after a sibling write must still emit"); - assert_eq!(update.config.fingers_per_node, Some(1234)); + assert_eq!(update.loaded.config().fingers_per_node, Some(1234)); } fn event_for(paths: &[&Path]) -> notify::Event { diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 81c6842e5..458a0b2a9 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -18,7 +18,7 @@ use clap::Parser; use unbounded_storage::backend::BackendRegistry; use unbounded_storage::bufferpool::{Pool, PoolConfig}; use unbounded_storage::config::{ - self, BackendSpec, Config, FrontendSpec, ResolvedFrontendBinding, frontend_spec, + self, BackendSpec, Config, FrontendSpec, LoadedConfig, ResolvedFrontendBinding, frontend_spec, }; use unbounded_storage::fabric::{self, Fabric, MrHandle, Provider}; use unbounded_storage::fanout::{ @@ -27,11 +27,7 @@ use unbounded_storage::fanout::{ use unbounded_storage::frontend::{ HttpDriver, HttpFrontend, LoadgenDriver, LoadgenFrontend, S3Driver, S3Frontend, }; -use unbounded_storage::p2p::{ - FingerTable, FingerTableConfig, PeerEntry, RouteTableHandle, RouteTableSnapshot, - RoutedTransport, TopologyPrefixWeight, TopologySelection, TopologyTags, TopologyWeighting, - node_to_ring, -}; +use unbounded_storage::p2p::{RouteTableHandle, RoutedTransport}; use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; use unbounded_storage::storage::StripeReq; @@ -223,7 +219,7 @@ fn main() -> ExitCode { None => (PathBuf::from(DEFAULT_CONFIG_PATH), false), }; - let config = match load_config(&config_path, config_explicit) { + let loaded = match load_config(&config_path, config_explicit) { Ok(c) => c, Err(e) => { eprintln!("config error: {e}"); @@ -237,7 +233,7 @@ fn main() -> ExitCode { // allocation, and only take effect at process start. `load_config` // has already run `apply_defaults`, so every startup field is // populated with its documented default when omitted. - let startup = config.startup(); + let startup = loaded.config().startup(); let memory = startup.memory(); let fabric_cfg = startup.fabric(); // The metrics exporter addr is a startup-fixed knob; capture @@ -411,7 +407,8 @@ fn main() -> ExitCode { // Prepare the initial shard layer through phase B. The shards remain // parked and RPC serving stays off until initial disks are published. - let prepared = match shard_layer::prepare_shard_layer(&config, &deps) { + let loaded = Arc::new(loaded); + let prepared = match shard_layer::prepare_shard_layer(loaded.clone(), &deps) { Ok(prepared) => prepared, Err(errs) => { for e in &errs { @@ -423,9 +420,7 @@ fn main() -> ExitCode { // Reconcile and publish the startup disk set before any frontend or // recursive RPC path can serve. Individual disk-open failures remain // nonfatal and the successfully opened subset is published. - let projection = - config::runtime_projection(&config).expect("loaded config projects to runtime"); - reconcile_cache_disks(&mut disk_registry, &cache_directories, &projection); + reconcile_cache_disks(&mut disk_registry, &cache_directories, loaded.runtime()); let layer = match shard_layer::activate_shard_layer(prepared) { Ok(layer) => layer, @@ -459,7 +454,7 @@ fn main() -> ExitCode { // loaded and applied; the startup version stays pinned to the config // realized here at process start, since the `[startup]` knobs it // tracks only take effect on restart. - let mut controller = config::ConfigController::new(target, Arc::new(config)); + let mut controller = config::ConfigController::new(target, loaded); // Start the Prometheus exporter once the controller exists, so it can // publish the live config-version gauges. A bind failure is logged @@ -495,8 +490,8 @@ fn main() -> ExitCode { while !SHUTDOWN.load(Ordering::Acquire) { match update_rx.recv_timeout(SHUTDOWN_POLL) { Ok(update) => { - let version = update.config.version; - match controller.apply(update.config.clone()) { + let version = update.loaded.config().version; + match controller.apply(update.loaded.clone()) { Ok(outcome) => eprintln!( "config: applied gen={} version={} tier={:?}", update.generation, version, outcome.tier @@ -564,9 +559,7 @@ fn run_shard( bytes_per_shard: usize, cache_directories: Arc, routes: RouteTableHandle, - frontend_specs: Arc>, - frontend_bindings: Arc>, - backend_specs: Arc>, + loaded: Arc, ctrl_rx: mpsc::Receiver, peer_rx: mpsc::Receiver>>, phaseb_tx: mpsc::Sender, @@ -574,6 +567,9 @@ fn run_shard( terminal_tx: mpsc::Sender, layer_stop: Arc, ) { + let frontend_specs = &loaded.config().frontends; + let frontend_bindings = &loaded.runtime().frontends; + let backend_specs = &loaded.config().backends; let phase_a = PhaseAReporter::new(widx, tx); // The fabric endpoint is built and owned by the `FabricGroup` in the // shard layer and shared by every shard mapped onto it (one per shard @@ -855,7 +851,7 @@ fn run_shard( fanout: fanout.clone(), geometry: geometry.clone(), routes: routes.clone(), - bindings: Rc::new(RefCell::new((*frontend_bindings).clone())), + bindings: Rc::new(RefCell::new(frontend_bindings.clone())), page_size, worker_idx: widx.0, waker: shard_loop.waker(), @@ -895,31 +891,18 @@ fn run_shard( .iter() .map(|f| (f.name.clone(), f.clone())) .collect(); - let mut last_bindings: HashMap = - (*frontend_bindings).clone(); + let mut last_bindings: HashMap = frontend_bindings.clone(); shard_loop.add_tick_hook(move || { let mut did_work = false; while let Ok(cmd) = ctrl_rx.try_recv() { match cmd { config::ShardCommand::ApplyConfig(apply) => { - routes.store_snapshot(apply.routes.clone()); + routes.store_snapshot(apply.loaded.routes().clone()); - let projection = match config::runtime_projection(&apply.config) { - Ok(projection) => projection, - Err(e) => { - let _ = apply.ack.send(config::ShardAck { - worker: widx, - result: Err(format!("config projection failed: {e}")), - }); - did_work = true; - continue; - } - }; - - let desired_backends = apply.config.backends.as_slice(); - let desired_frontends = apply.config.frontends.as_slice(); + let desired_backends = apply.loaded.config().backends.as_slice(); + let desired_frontends = apply.loaded.config().frontends.as_slice(); let desired_frontend_backends = - config::frontend_backend_map(&projection.frontends); + config::frontend_backend_map(&apply.loaded.runtime().frontends); // Refresh stripe geometry before building any // frontend so a co-applied backend stripe change // is visible to a frontend add in the same pass. @@ -933,13 +916,13 @@ fn run_shard( let frontends_to_rebuild = frontend_rebuild_ids( &last_bindings, - &projection.frontends, + &apply.loaded.runtime().frontends, &last_backends, desired_backends, ); let forced_frontend_updates: HashSet = frontends_to_rebuild.into_iter().collect(); - *bindings.borrow_mut() = projection.frontends.clone(); + *bindings.borrow_mut() = apply.loaded.runtime().frontends.clone(); // Drive the transport backend registry and the // frontend registry together so frontend adds are @@ -971,7 +954,7 @@ fn run_shard( for (id, spec) in &last_frontends { let desired_spec = desired_frontends.iter().find(|f| f.name == *id); if desired_spec == Some(spec) { - if let Some(binding) = projection.frontends.get(id) { + if let Some(binding) = apply.loaded.runtime().frontends.get(id) { applied_bindings.insert(id.clone(), binding.clone()); continue; } @@ -1092,86 +1075,6 @@ fn panic_payload_str(payload: &(dyn std::any::Any + Send)) -> &str { } } -fn build_routes(config: &Config) -> RouteTableSnapshot { - let projection = config::runtime_projection(config).expect("loaded config projects to runtime"); - if projection.caches.is_empty() { - return RouteTableSnapshot { - cache_ids: HashSet::new(), - fingers: None, - }; - } - - let mesh = &projection.mesh; - let local = PeerEntry { - node: mesh.self_node_id, - ring: node_to_ring(mesh.self_node_id), - tags: TopologyTags(mesh.self_tags.clone()), - }; - let fingers = if let Some(plan) = &mesh.routing_plan { - let peer_by_name: HashMap<&str, &config::RuntimePeer> = mesh - .peers - .iter() - .map(|peer| (peer.name.as_str(), peer)) - .collect(); - let entry_of = |name: &str| { - let peer = peer_by_name - .get(name) - .expect("validated routing_plan names reference peers"); - PeerEntry { - node: peer.node_id, - ring: node_to_ring(peer.node_id), - tags: TopologyTags(peer.spec.tags.clone()), - } - }; - Arc::new(FingerTable::from_explicit( - local, - plan.fingers.iter().map(|name| entry_of(name)).collect(), - plan.successor.as_deref().map(entry_of), - plan.predecessor.as_deref().map(entry_of), - )) - } else { - let peers: Vec = mesh - .peers - .iter() - .map(|peer| PeerEntry { - node: peer.node_id, - ring: node_to_ring(peer.node_id), - tags: TopologyTags(peer.spec.tags.clone()), - }) - .collect(); - Arc::new(FingerTable::build( - local, - &peers, - FingerTableConfig { - k: mesh.fingers_per_node.max(1), - topology: mesh - .topology_weighting - .as_ref() - .map(p2p_topology_weighting) - .map(TopologySelection::Weighted) - .unwrap_or_default(), - }, - )) - }; - RouteTableSnapshot { - cache_ids: projection.caches.keys().cloned().collect(), - fingers: Some(fingers), - } -} - -fn p2p_topology_weighting(weighting: &config::TopologyWeighting) -> TopologyWeighting { - TopologyWeighting { - prefix_weights: weighting - .prefix_weights - .iter() - .map(|weight| TopologyPrefixWeight { - tag_index: weight.tag_index, - weight: weight.weight, - }) - .collect(), - } -} - fn reconcile_cache_disks( disk_registry: &mut DiskRegistry, cache_directories: &CacheDirectorySet, @@ -1690,11 +1593,15 @@ struct Cli { /// the file is absent, fall back to [`Config::default`] with a /// warning. Any other failure - including explicit missing paths and /// parse errors - is fatal. -fn load_config(path: &Path, explicit: bool) -> Result { - match Config::load(path) { +fn load_config(path: &Path, explicit: bool) -> Result { + match LoadedConfig::load(path) { Ok(c) => { if path.extension().and_then(|e| e.to_str()) == Some("binpb") { - eprintln!("config: loaded {} (binpb):\n{c:#?}", path.display()); + eprintln!( + "config: loaded {} (binpb):\n{:#?}", + path.display(), + c.config() + ); } Ok(c) } @@ -1710,9 +1617,8 @@ fn load_config(path: &Path, explicit: bool) -> Result { // (`startup()`, `p2p()`, ...) would panic. Promote the proto3 // zero values to documented defaults exactly as the on-disk // path does. - let mut c = Config::default(); - c.apply_defaults(); - Ok(c) + LoadedConfig::from_config(Config::default()) + .map_err(|e| format!("loading built-in defaults: {e}")) } Err(e) => Err(format!("loading {}: {e}", path.display())), } @@ -2057,11 +1963,11 @@ mod tests { let cfg = load_config(path, false).expect("missing default path falls back to defaults"); // These accessors panic if defaults were not applied. assert_eq!( - cfg.startup().fabric().default_listen_addr(), + cfg.config().startup().fabric().default_listen_addr(), Some("0.0.0.0:0") ); assert_eq!( - cfg.startup().memory().memory_total_bytes, + cfg.config().startup().memory().memory_total_bytes, Some(128 * 1024 * 1024) ); } diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 18c3978c2..4105e2e3c 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -32,13 +32,12 @@ use std::sync::mpsc; use std::time::Duration; use unbounded_storage::config::{ - self, ApplyError, Config, ConfigApplyTarget, ConfigDiff, ShardControlGroup, + self, ApplyError, ConfigApplyTarget, ConfigDiff, LoadedConfig, ShardControlGroup, }; use unbounded_storage::fabric::PeerId; use unbounded_storage::memory::HUGEPAGE_2MB; use unbounded_storage::p2p::RouteTableHandle; use unbounded_storage::runtime::{JoinHandle, Threading, WorkerIdx}; -use unbounded_storage::storage::StripeReq; use unbounded_storage::storage::disks::{CacheDirectorySet, DiskRegistry, UringDiskTarget}; use unbounded_storage::topology::ServingShard; @@ -116,7 +115,7 @@ impl ShardLayer { } } -/// Prepare a shard layer from `config` on the runtime in `deps`, +/// Prepare a shard layer from `loaded` on the runtime in `deps`, /// blocking until every shard has completed phase B. The returned /// shards remain parked and no RPC server has started. /// @@ -131,7 +130,7 @@ impl ShardLayer { /// shard failed to come up; any shards that *did* come up are torn down /// first so no threads leak. pub fn prepare_shard_layer( - config: &Config, + loaded: Arc, deps: &ShardSpawnDeps, ) -> Result> { let layer_stop = Arc::new(AtomicBool::new(false)); @@ -139,14 +138,11 @@ pub fn prepare_shard_layer( let settings = &deps.settings; let shard_backing_sizes = shard_backing_sizes(settings.memory_total_bytes, worker_count) .map_err(|e| vec![format!("invalid shard memory budget: {e}")])?; - let projection = config::runtime_projection(config) - .map_err(|e| vec![format!("config projection failed: {e}")])?; - let frontend_specs = Arc::new(config.frontends.clone()); - let frontend_bindings = Arc::new(projection.frontends.clone()); - let backend_specs = Arc::new(config.backends.clone()); - let routes = crate::build_routes(config); - let runtime_peers = config::runtime_peers(&projection); - let self_peer = local_self_peer(&projection) + let config = loaded.config(); + let runtime = loaded.runtime(); + let routes = loaded.routes(); + let runtime_peers = config::runtime_peers(runtime); + let self_peer = local_self_peer(runtime) .map_err(|e| vec![format!("unsupported fabric identity config: {e}")])?; // Bring up the shared fabric endpoints before spawning any shards: @@ -198,9 +194,7 @@ pub fn prepare_shard_layer( let backing_kind = settings.backing_kind; let cache_directories = deps.cache_directories.clone(); let route_handle = RouteTableHandle::from_snapshot(routes.clone()); - let frontend_specs = frontend_specs.clone(); - let frontend_bindings = frontend_bindings.clone(); - let backend_specs = backend_specs.clone(); + let loaded = loaded.clone(); let layer_stop = layer_stop.clone(); let terminal_tx = terminal_tx.clone(); let rt = deps.runtime.clone(); @@ -218,9 +212,7 @@ pub fn prepare_shard_layer( backing_size, cache_directories, route_handle, - frontend_specs, - frontend_bindings, - backend_specs, + loaded, ctrl_rx, peer_rx, phaseb_tx, @@ -559,16 +551,17 @@ impl ProcessApplyTarget { } impl ConfigApplyTarget for ProcessApplyTarget { - fn apply_in_place(&mut self, new: &Arc, diff: &ConfigDiff) -> Result<(), ApplyError> { + fn apply_in_place( + &mut self, + new: &Arc, + diff: &ConfigDiff, + ) -> Result<(), ApplyError> { if diff.requires_restart() { return Err(ApplyError::Target( "self peer identity changed; restart required".to_string(), )); } - let projection = config::runtime_projection(new) - .map_err(|e| ApplyError::Target(format!("config projection failed: {e}")))?; - // The shards must see a new config whenever their routing surface, // graph projection, per-disk page-cache policy, or per-shard // backend/frontend registries need to change. Cache graph changes can @@ -580,7 +573,6 @@ impl ConfigApplyTarget for ProcessApplyTarget { || diff.frontends_changed; if needs_broadcast { - let routes = crate::build_routes(new); let layer = self .layer .as_mut() @@ -592,10 +584,10 @@ impl ConfigApplyTarget for ProcessApplyTarget { // touching when the routing surface (p2p/peers) changed; a // backend/frontend-only change leaves them untouched. if diff.requires_routing_reload() { - layer.fabric_group.reload_routes(&routes); + layer.fabric_group.reload_routes(new.routes()); } if diff.requires_peer_reconcile() { - let runtime_peers = config::runtime_peers(&projection); + let runtime_peers = config::runtime_peers(new.runtime()); layer.fabric_group.reconcile_peers(&runtime_peers); } @@ -604,14 +596,16 @@ impl ConfigApplyTarget for ProcessApplyTarget { // rebuild their own transport registries against an already // up-to-date origin surface. if diff.backends_changed { - layer.fabric_group.reconcile_backends(&new.backends); + layer + .fabric_group + .reconcile_backends(&new.config().backends); } // Republish config + routing to every shard and block until // each has acked, so the routing surface and per-shard // backend/frontend reconcile each shard performs on receipt // have provably landed. - if let Err(error) = layer.control.broadcast_apply(new.clone(), routes, *diff) { + if let Err(error) = layer.control.broadcast_apply(new.clone(), *diff) { if error.apply_state_is_indeterminate() { crate::SHUTDOWN.store(true, Ordering::Release); } @@ -637,7 +631,7 @@ impl ConfigApplyTarget for ProcessApplyTarget { // If a broadcast fails, ConfigController keeps the old config as // current and the live disk topology remains on that same version. if diff.caches_changed || diff.disks_changed { - self.reconcile_disks(&projection); + self.reconcile_disks(new.runtime()); } Ok(()) diff --git a/cmd/unbounded-storage/tests/lifecycle/workload.rs b/cmd/unbounded-storage/tests/lifecycle/workload.rs index 4bcccc051..31f5e5ecb 100644 --- a/cmd/unbounded-storage/tests/lifecycle/workload.rs +++ b/cmd/unbounded-storage/tests/lifecycle/workload.rs @@ -17,9 +17,8 @@ use proptest::prelude::*; use unbounded_storage::bufferpool::{Error, StripeKey}; use unbounded_storage::config::{ ApplyError, BackendSpec, CacheSpec, Config, ConfigApplyTarget, ConfigController, ConfigDiff, - DiskSpec, FileDiskConfig, FrontendSpec, HttpBackendConfig, HttpFrontendConfig, PeerSpec, - TcpPeerConfig, backend_spec, disk_spec, frontend_spec, peer_spec, runtime_disks, - runtime_projection, + DiskSpec, FileDiskConfig, FrontendSpec, HttpBackendConfig, HttpFrontendConfig, LoadedConfig, + PeerSpec, TcpPeerConfig, backend_spec, disk_spec, frontend_spec, peer_spec, runtime_disks, }; use unbounded_storage::runtime::ShardLoop; use unbounded_storage::storage::blockdev::MockDeviceConfig; @@ -254,10 +253,11 @@ struct SimApplyTarget { } impl ConfigApplyTarget for SimApplyTarget { - fn apply_in_place(&mut self, new: &Arc, diff: &ConfigDiff) -> Result<(), ApplyError> { - let projection = runtime_projection(new) - .map_err(|e| ApplyError::Target(format!("config projection failed: {e}")))?; - + fn apply_in_place( + &mut self, + new: &Arc, + diff: &ConfigDiff, + ) -> Result<(), ApplyError> { if diff.requires_routing_reload() || diff.caches_changed || diff.backends_changed @@ -265,12 +265,15 @@ impl ConfigApplyTarget for SimApplyTarget { { self.metrics.borrow_mut().broadcasts += 1; for shard in &self.shards { - shard.apply_queue.borrow_mut().push_back(new.version); + shard + .apply_queue + .borrow_mut() + .push_back(new.config().version); } } if diff.caches_changed { - let disks = runtime_disks(&projection); + let disks = runtime_disks(new.runtime()); let report = self.registry.reconcile(&disks); let mut metrics = self.metrics.borrow_mut(); metrics.disk_applies += 1; @@ -731,13 +734,15 @@ fn spawn_apply_driver( while !bootstrap_done.get() || !shards.iter().all(|s| s.phase_b.get()) { yield_once().await; } - let initial = Arc::new(config_for_generation( - 0, - generation_disk_specs(0, w.initial_disks), - )); + let initial = Arc::new( + LoadedConfig::from_config(config_for_generation( + 0, + generation_disk_specs(0, w.initial_disks), + )) + .expect("initial sim config loads"), + ); let mut registry = DiskRegistry::new(RegistryTarget, disk_slots()); - let projection = runtime_projection(&initial).expect("initial sim config projects"); - let initial_disks = runtime_disks(&projection); + let initial_disks = runtime_disks(initial.runtime()); let initial_report = registry.reconcile(&initial_disks); metrics.borrow_mut().registry_failures += initial_report.failures.len(); let target = SimApplyTarget { @@ -752,10 +757,14 @@ fn spawn_apply_driver( let mut controller = ConfigController::new(target, initial); for (idx, apply) in w.applies.iter().enumerate() { yield_n(apply.delay).await; - let mut next = (*controller.current()).as_ref().clone(); + let mut next = controller.current().config().clone(); next.version = idx as u64 + 1; mutate_config(&mut next, apply, idx + 1); - controller.apply(Arc::new(next)).expect("sim config apply"); + controller + .apply(Arc::new( + LoadedConfig::from_config(next).expect("sim config loads"), + )) + .expect("sim config apply"); } let versions = controller.config_versions().snapshot(); let mut m = metrics.borrow_mut(); From 29f45ae0cab411047d816c159457a8d9ab993928 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 03:26:24 +0000 Subject: [PATCH 46/56] unbounded-storage: align disk lifecycle simulation --- cmd/unbounded-storage/tests/lifecycle/tests.rs | 5 ++++- cmd/unbounded-storage/tests/lifecycle/workload.rs | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/unbounded-storage/tests/lifecycle/tests.rs b/cmd/unbounded-storage/tests/lifecycle/tests.rs index 9d68c867c..55783653f 100644 --- a/cmd/unbounded-storage/tests/lifecycle/tests.rs +++ b/cmd/unbounded-storage/tests/lifecycle/tests.rs @@ -143,6 +143,9 @@ fn smoke_config_disk_and_shard_lifecycle() { }; let report: RunReport = run_workload(0x51A7E, w).expect("smoke run"); assert_eq!(report.phase_b_ready, 2); - assert_eq!(report.directory_generation, 1); + assert_eq!(report.broadcasts, 3); + assert_eq!(report.disk_applies, 1); + assert_eq!(report.shard_apply_counts, vec![3, 3]); + assert_eq!(report.directory_generation, 2); assert_eq!(report.clients_finished, 1); } diff --git a/cmd/unbounded-storage/tests/lifecycle/workload.rs b/cmd/unbounded-storage/tests/lifecycle/workload.rs index 31f5e5ecb..b5a790b99 100644 --- a/cmd/unbounded-storage/tests/lifecycle/workload.rs +++ b/cmd/unbounded-storage/tests/lifecycle/workload.rs @@ -260,6 +260,7 @@ impl ConfigApplyTarget for SimApplyTarget { ) -> Result<(), ApplyError> { if diff.requires_routing_reload() || diff.caches_changed + || diff.disks_changed || diff.backends_changed || diff.frontends_changed { @@ -272,7 +273,7 @@ impl ConfigApplyTarget for SimApplyTarget { } } - if diff.caches_changed { + if diff.caches_changed || diff.disks_changed { let disks = runtime_disks(new.runtime()); let report = self.registry.reconcile(&disks); let mut metrics = self.metrics.borrow_mut(); @@ -456,12 +457,13 @@ pub fn expected_apply_counts(w: &Workload) -> ExpectedApplyCounts { let diff = ConfigDiff::between(¤t, &next); if diff.requires_routing_reload() || diff.caches_changed + || diff.disks_changed || diff.backends_changed || diff.frontends_changed { out.broadcasts += 1; } - if diff.caches_changed { + if diff.caches_changed || diff.disks_changed { out.disk_applies += 1; } if diff.any() { From 52b85c9479a2652cdab1120dee980f7e7f949174 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 03:38:44 +0000 Subject: [PATCH 47/56] unbounded-storage: centralize route publication --- cmd/unbounded-storage/ARCHITECTURE.md | 6 ++++ cmd/unbounded-storage/src/fabric_group.rs | 19 +++-------- cmd/unbounded-storage/src/main.rs | 2 -- cmd/unbounded-storage/src/shard_layer.rs | 41 +++++++++++++++++------ 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 285c159f9..ed7f08f5a 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -594,6 +594,12 @@ for the complete worker set and report outstanding worker identities. A timeout, acknowledgement-channel disconnect, or partial broadcast send failure requests process shutdown: already-delivered commands may complete later, so retrying against the old controller snapshot would be unsafe. +All shard transports and fabric RPC handlers share one process-wide +`RouteTableHandle`. The apply target is its only writer and publishes the new +snapshot once, after shard acknowledgement, page-cache drain, and disk +publication succeed. This keeps routing on the prior snapshot when an earlier +apply step reports failure; other resource reconciliation is still not a +whole-process transaction. Sections (all optional, each falling back to defaults): diff --git a/cmd/unbounded-storage/src/fabric_group.rs b/cmd/unbounded-storage/src/fabric_group.rs index 4978111b1..d020b68bf 100644 --- a/cmd/unbounded-storage/src/fabric_group.rs +++ b/cmd/unbounded-storage/src/fabric_group.rs @@ -28,7 +28,7 @@ use unbounded_storage::fabric::{ use unbounded_storage::fanout::FetchChannel; use unbounded_storage::memory::{Backing, BackingKind, BackingRequest, HUGEPAGE_2MB, allocate}; use unbounded_storage::p2p::{ - OwnerShardSource, OwnerShardTable, RecursiveHandler, RouteTableHandle, RouteTableSnapshot, + OwnerShardSource, OwnerShardTable, RecursiveHandler, RouteTableHandle, }; use unbounded_storage::runtime::{Threading, WorkerIdx}; use unbounded_storage::storage::StripeReq; @@ -293,7 +293,7 @@ impl FabricGroup { fabric_startup: &FabricStartup, _backend_specs: &[BackendSpec], cache_directories: Arc, - routes: &RouteTableSnapshot, + routes: &RouteTableHandle, peers: &[RuntimePeer], self_peer: PeerId, ) -> Result> { @@ -350,15 +350,6 @@ impl FabricGroup { .collect() } - /// Reload every endpoint's RPC-handler routing from a new snapshot. - /// Driven in lockstep with the per-shard transport reload so the - /// classify and forward paths move together. - pub fn reload_routes(&self, snapshot: &RouteTableSnapshot) { - for unit in &self.units { - unit.routes.store_snapshot(snapshot.clone()); - } - } - /// Start every shared RPC server after shards have published the /// bufferpool MRs/fetch channels the owner path sources from. pub fn start_rpc_servers(&mut self, shards: &[RpcShardPublish]) -> Result<(), Vec> { @@ -462,7 +453,7 @@ fn build_unit( backing_kind: BackingKind, fabric_startup: &FabricStartup, cache_directories: &Arc, - routes: &RouteTableSnapshot, + routes: &RouteTableHandle, peers: &[RuntimePeer], self_peer: PeerId, ) -> Result { @@ -525,8 +516,6 @@ fn build_unit( .register_backing(&scratch, spec.numa) .map_err(|e| format!("worker={worker}: register rpc scratch: {e}"))?; - let routes = RouteTableHandle::from_snapshot(routes.clone()); - fabric.check_shared_domain_capacity(spec.expected_mr); let desired_peers = runtime_peer_connections_for_unit(peers, spec.unit_idx); @@ -539,7 +528,7 @@ fn build_unit( Ok(FabricUnit { rpc_server: None, scratch: Some(scratch), - routes, + routes: routes.clone(), fabric, scratch_mr, page_size, diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 458a0b2a9..5a9cccdfd 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -897,8 +897,6 @@ fn run_shard( while let Ok(cmd) = ctrl_rx.try_recv() { match cmd { config::ShardCommand::ApplyConfig(apply) => { - routes.store_snapshot(apply.loaded.routes().clone()); - let desired_backends = apply.loaded.config().backends.as_slice(); let desired_frontends = apply.loaded.config().frontends.as_slice(); let desired_frontend_backends = diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 4105e2e3c..1371d2724 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -79,6 +79,7 @@ pub struct PreparedShardLayer { rpc_shards: Vec, serve_start_txs: Vec>, terminal_rx: mpsc::Receiver, + routes: RouteTableHandle, } /// A spawned set of shard threads plus the handles to drive and retire @@ -107,6 +108,9 @@ pub struct ShardLayer { /// memory. _backing_keepalives: Vec>, terminal_rx: mpsc::Receiver, + /// The sole writer for the process-wide routing publication. Every + /// shard transport and fabric RPC handler holds a clone. + routes: RouteTableHandle, } impl ShardLayer { @@ -140,7 +144,7 @@ pub fn prepare_shard_layer( .map_err(|e| vec![format!("invalid shard memory budget: {e}")])?; let config = loaded.config(); let runtime = loaded.runtime(); - let routes = loaded.routes(); + let routes = RouteTableHandle::from_snapshot(loaded.routes().clone()); let runtime_peers = config::runtime_peers(runtime); let self_peer = local_self_peer(runtime) .map_err(|e| vec![format!("unsupported fabric identity config: {e}")])?; @@ -193,7 +197,7 @@ pub fn prepare_shard_layer( let tx = ready_tx.clone(); let backing_kind = settings.backing_kind; let cache_directories = deps.cache_directories.clone(); - let route_handle = RouteTableHandle::from_snapshot(routes.clone()); + let route_handle = routes.clone(); let loaded = loaded.clone(); let layer_stop = layer_stop.clone(); let terminal_tx = terminal_tx.clone(); @@ -349,6 +353,7 @@ pub fn prepare_shard_layer( backing_keepalives, serve_start_txs, terminal_rx, + routes, ); return Err(phaseb_errors); } @@ -362,6 +367,7 @@ pub fn prepare_shard_layer( rpc_shards, serve_start_txs, terminal_rx, + routes, }) } @@ -378,6 +384,7 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result Result Result Result>, serve_start_txs: Vec>, terminal_rx: mpsc::Receiver, + routes: RouteTableHandle, ) { layer_stop.store(true, Ordering::Relaxed); drop(serve_start_txs); @@ -436,6 +447,7 @@ fn retire_failed_activation( } drop(control); drop(fabric_group); + drop(routes); drop(backing_keepalives); drop(terminal_rx); } @@ -476,6 +488,7 @@ pub fn teardown_shard_layer(layer: ShardLayer) -> bool { layer_stop, _backing_keepalives, terminal_rx, + routes, } = layer; layer_stop.store(true, Ordering::Relaxed); let mut failed = false; @@ -502,6 +515,7 @@ pub fn teardown_shard_layer(layer: ShardLayer) -> bool { // assigned shard's data backing. The shard threads have already // joined, so nothing still touches those regions. drop(fabric_group); + drop(routes); // Free every shard's backing only now: the fabrics above have closed // their MRs, and all shard threads (and thus every io_uring ring that // may still reference a peer's pages as a `SEND_ZC` source) have @@ -578,14 +592,9 @@ impl ConfigApplyTarget for ProcessApplyTarget { .as_mut() .expect("shard layer present between applies"); - // Peer connections and the RPC handlers' routing live on the - // shared fabric endpoints, not on individual shards, so they - // are reconciled once per endpoint here. Both only need - // touching when the routing surface (p2p/peers) changed; a - // backend/frontend-only change leaves them untouched. - if diff.requires_routing_reload() { - layer.fabric_group.reload_routes(new.routes()); - } + // Peer connections live on the shared fabric endpoints, not + // on individual shards, so they are reconciled once per + // endpoint here. if diff.requires_peer_reconcile() { let runtime_peers = config::runtime_peers(new.runtime()); layer.fabric_group.reconcile_peers(&runtime_peers); @@ -634,6 +643,18 @@ impl ConfigApplyTarget for ProcessApplyTarget { self.reconcile_disks(new.runtime()); } + // Routing is the process-wide visibility point for a successful + // apply. Publish it only after every fallible shard broadcast and + // the disk update have completed, so failed applies retain the + // prior route snapshot everywhere. + if diff.requires_routing_reload() { + self.layer + .as_ref() + .expect("shard layer present between applies") + .routes + .store_snapshot(new.routes().clone()); + } + Ok(()) } } From 7f6e00b853ddfd0bda0bcdee284b7a8a706e2e06 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 03:57:03 +0000 Subject: [PATCH 48/56] unbounded-storage: surface disk reconcile failures --- cmd/unbounded-storage/ARCHITECTURE.md | 11 +++- cmd/unbounded-storage/README.md | 4 +- cmd/unbounded-storage/src/config/control.rs | 3 +- cmd/unbounded-storage/src/main.rs | 19 +++++-- cmd/unbounded-storage/src/shard_layer.rs | 55 +++++++++++++++++-- .../src/storage/disks/mod.rs | 43 ++++++++++++++- 6 files changed, 118 insertions(+), 17 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index ed7f08f5a..57b83ba59 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -156,7 +156,8 @@ excluded from the live-reload diff. initial disks and publishes channels. Only after publication does activation start the recursive RPC servers and release shards into their serve loops; the config watcher then takes over for the lifetime of the process. Disk-open - failures remain nonfatal and the successfully opened subset is published. + failures retire the prepared layer rather than activating a partial startup + configuration. ### Shard readiness and panic safety @@ -555,6 +556,10 @@ path-sorted for stable hashing; `drain()` clears channels before handles. `DiskChannelDirectory` is the Arc'd hot-swap publication surface, and `LiveShardLocalStore` is a per-shard view over the live directory that re-registers buffers when it observes a swap (`current_or_replay`). +Every reconcile publishes the realized open subset. A live open failure aborts +config convergence so the same desired snapshot remains retryable; a startup +open failure retires the parked shard layer. Same-path drift remains remove then +add because provisioning and recovery cannot safely run beside the old engine. **Stripe keys**: `stripe_key` derives the 32-byte content-addressed key; `METADATA_STRIPE_IDX` and `OriginRef`/`StripeReq` describe the request shape. @@ -597,7 +602,9 @@ retrying against the old controller snapshot would be unsafe. All shard transports and fabric RPC handlers share one process-wide `RouteTableHandle`. The apply target is its only writer and publishes the new snapshot once, after shard acknowledgement, page-cache drain, and disk -publication succeed. This keeps routing on the prior snapshot when an earlier +publication succeed. Disk open failures are returned after the realized subset +is published, leaving the desired config retryable and routing on the prior +snapshot. This keeps routing on the prior snapshot when an earlier apply step reports failure; other resource reconciliation is still not a whole-process transaction. diff --git a/cmd/unbounded-storage/README.md b/cmd/unbounded-storage/README.md index 3afd69a1d..e6530a663 100644 --- a/cmd/unbounded-storage/README.md +++ b/cmd/unbounded-storage/README.md @@ -62,8 +62,8 @@ The daemon traps `SIGINT` and `SIGTERM` and tears shards down in a deterministic order: shards stop and join first, disk-channel publications are cleared, and disks are drained last. At startup shards complete both readiness phases while parked; initial disks are reconciled and published before RPC -servers and shard serving are activated. Individual disk-open failures remain -nonfatal. +servers and shard serving are activated. A startup disk-open failure retires +the prepared shard layer rather than reporting a partial startup configuration. ## Configuration file diff --git a/cmd/unbounded-storage/src/config/control.rs b/cmd/unbounded-storage/src/config/control.rs index b992dfba2..f4786c071 100644 --- a/cmd/unbounded-storage/src/config/control.rs +++ b/cmd/unbounded-storage/src/config/control.rs @@ -102,8 +102,7 @@ pub enum ApplyError { }, /// One or more shards reported a failure while applying. ShardApply(Vec<(WorkerIdx, String)>), - /// The process-level apply target rejected the config before shard - /// broadcast. + /// The process-level apply target rejected the config. Target(String), } diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 5a9cccdfd..b5dbb694a 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -32,7 +32,7 @@ use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; use unbounded_storage::storage::StripeReq; use unbounded_storage::storage::disks::{ - CacheDirectorySet, ChainLocalStore, DiskRegistry, UringDiskTarget, + CacheDirectorySet, ChainLocalStore, DiskRegistry, DiskReport, UringDiskTarget, }; use unbounded_storage::topology::{CorePlan, CorePlanConfig, DiskCpuSlot, Host, ServingShard}; @@ -418,9 +418,16 @@ fn main() -> ExitCode { } }; // Reconcile and publish the startup disk set before any frontend or - // recursive RPC path can serve. Individual disk-open failures remain - // nonfatal and the successfully opened subset is published. - reconcile_cache_disks(&mut disk_registry, &cache_directories, loaded.runtime()); + // recursive RPC path can serve. There is no prior applied disk state + // to retain at startup, so an open failure aborts the prepared layer. + let disk_report = + reconcile_cache_disks(&mut disk_registry, &cache_directories, loaded.runtime()); + if !disk_report.failures.is_empty() { + shard_layer::retire_prepared_shard_layer(prepared); + clear_cache_disk_publications(&cache_directories); + disk_registry.drain(); + return ExitCode::FAILURE; + } let layer = match shard_layer::activate_shard_layer(prepared) { Ok(layer) => layer, @@ -1077,7 +1084,7 @@ fn reconcile_cache_disks( disk_registry: &mut DiskRegistry, cache_directories: &CacheDirectorySet, projection: &config::RuntimeGraph, -) { +) -> DiskReport { let mut cache_ids: Vec = projection.caches.keys().cloned().collect(); cache_ids.sort(); cache_directories.reconcile(cache_ids.iter().cloned()); @@ -1098,6 +1105,8 @@ fn reconcile_cache_disks( for cache_id in cache_ids { cache_directories.apply_channels(&cache_id, channels.clone()); } + + report } fn clear_cache_disk_publications(cache_directories: &CacheDirectorySet) { diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 1371d2724..15db530c8 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -430,6 +430,31 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result, fabric_group: FabricGroup, @@ -552,8 +577,28 @@ impl ProcessApplyTarget { /// Reconcile projected cache disks in place and republish the /// resulting channel set to the live directory (idempotent when the /// disk set is unchanged). - fn reconcile_disks(&mut self, projection: &config::RuntimeGraph) { - crate::reconcile_cache_disks(&mut self.disk_registry, &self.cache_directories, projection); + fn reconcile_disks(&mut self, projection: &config::RuntimeGraph) -> Result<(), ApplyError> { + let report = crate::reconcile_cache_disks( + &mut self.disk_registry, + &self.cache_directories, + projection, + ); + if report.failures.is_empty() { + return Ok(()); + } + + let failure_count = report.failures.len(); + let mut failures = report.failures; + failures.sort_by(|a, b| a.0.cmp(&b.0)); + let details = failures + .into_iter() + .map(|(path, error)| format!("{}: {error}", path.display())) + .collect::>() + .join("; "); + Err(ApplyError::Target(format!( + "{} disk(s) failed to open: {details}", + failure_count + ))) } /// Consume the target at shutdown, returning the live layer (if any) @@ -638,9 +683,11 @@ impl ConfigApplyTarget for ProcessApplyTarget { // Disk/channel changes mutate live registries and directories, so // all fallible shard broadcasts must complete before this point. // If a broadcast fails, ConfigController keeps the old config as - // current and the live disk topology remains on that same version. + // current and disk reconciliation does not begin. Once reconciliation + // begins, its realized subset is published even when an open fails; + // returning that failure keeps the desired snapshot retryable. if diff.caches_changed || diff.disks_changed { - self.reconcile_disks(new.runtime()); + self.reconcile_disks(new.runtime())?; } // Routing is the process-wide visibility point for a successful diff --git a/cmd/unbounded-storage/src/storage/disks/mod.rs b/cmd/unbounded-storage/src/storage/disks/mod.rs index cb5d7374b..14aa365ca 100644 --- a/cmd/unbounded-storage/src/storage/disks/mod.rs +++ b/cmd/unbounded-storage/src/storage/disks/mod.rs @@ -139,8 +139,9 @@ impl DiskRegistry { /// the disk is opened (config / queue_depth / page_size_bytes / /// skip_recovery_scan / force_format / bypass_admission / /// bypass_index_read / bypass_checksum) are treated as a remove followed by an add. - /// Partial failures during opens are reported but do not abort the - /// reconcile. + /// Partial failures during opens are reported after all desired paths + /// have been attempted. Callers publish the realized open subset before + /// deciding whether the desired configuration converged. pub fn reconcile(&mut self, desired: &[DiskSpec]) -> DiskReport { let mut report = DiskReport::default(); @@ -551,6 +552,44 @@ mod tests { assert_eq!(reg.channels_snapshot()[0].0, PathBuf::from("/good")); } + #[test] + fn failed_open_is_retried_by_the_same_desired_set() { + let (target, state) = MockDiskTarget::new(); + state.lock().unwrap().fail_on.insert(PathBuf::from("/a")); + let mut reg = DiskRegistry::new(target, vec![]); + let desired = [spec("/a", None)]; + + let failed = reg.reconcile(&desired); + assert_eq!(failed.failures.len(), 1); + assert!(reg.entries.is_empty()); + + state.lock().unwrap().fail_on.clear(); + let retried = reg.reconcile(&desired); + assert_eq!(retried.added, 1); + assert!(retried.failures.is_empty()); + assert_eq!(reg.entries[&PathBuf::from("/a")].spec, desired[0]); + } + + #[test] + fn failed_drift_replacement_remains_retryable() { + let (target, state) = MockDiskTarget::new(); + let mut reg = DiskRegistry::new(target, vec![]); + reg.reconcile(&[spec("/a", Some(8))]); + + state.lock().unwrap().fail_on.insert(PathBuf::from("/a")); + let desired = [spec("/a", Some(32))]; + let failed = reg.reconcile(&desired); + assert_eq!(failed.removed, 1); + assert_eq!(failed.failures.len(), 1); + assert!(!reg.entries.contains_key(&PathBuf::from("/a"))); + + state.lock().unwrap().fail_on.clear(); + let retried = reg.reconcile(&desired); + assert_eq!(retried.added, 1); + assert!(retried.failures.is_empty()); + assert_eq!(reg.entries[&PathBuf::from("/a")].spec.queue_depth, Some(32)); + } + /// Records the per-disk `(path, cpu_hint)` decisions so tests can /// assert on slot assignment without inspecting registry internals. struct CpuRecorder { From 513f428dd20e3cdc74b53379f10eef13d7106862 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 04:10:48 +0000 Subject: [PATCH 49/56] unbounded-storage: surface hard peer failures --- cmd/unbounded-storage/ARCHITECTURE.md | 4 ++++ cmd/unbounded-storage/src/fabric_group.rs | 29 ++++++++++++++++++++--- cmd/unbounded-storage/src/shard_layer.rs | 8 ++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 57b83ba59..5a268402f 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -599,6 +599,10 @@ for the complete worker set and report outstanding worker identities. A timeout, acknowledgement-channel disconnect, or partial broadcast send failure requests process shutdown: already-delivered commands may complete later, so retrying against the old controller snapshot would be unsafe. +Peer availability failures are accepted as desired reconnect intent inside the +fabric. Errors that reach process-level peer reconciliation are therefore hard +lifecycle/configuration failures; startup rejects them, while live apply +requests fail-stop shutdown because remove/add mutations cannot be rolled back. All shard transports and fabric RPC handlers share one process-wide `RouteTableHandle`. The apply target is its only writer and publishes the new snapshot once, after shard acknowledgement, page-cache drain, and disk diff --git a/cmd/unbounded-storage/src/fabric_group.rs b/cmd/unbounded-storage/src/fabric_group.rs index d020b68bf..b9e0f9327 100644 --- a/cmd/unbounded-storage/src/fabric_group.rs +++ b/cmd/unbounded-storage/src/fabric_group.rs @@ -416,7 +416,8 @@ impl FabricGroup { /// Re-drive every endpoint's fabric connection table toward `peers`. /// Connections live at the fabric/address-vector level, so this runs /// once per endpoint rather than once per shard. - pub fn reconcile_peers(&mut self, peers: &[RuntimePeer]) { + pub fn reconcile_peers(&mut self, peers: &[RuntimePeer]) -> Result<(), Vec> { + let mut failures = Vec::new(); for (unit_idx, unit) in self.units.iter_mut().enumerate() { let desired = runtime_peer_connections_for_unit(peers, unit_idx); let report = config::reconcile::reconcile_connections( @@ -427,6 +428,10 @@ impl FabricGroup { unit.applied_peers = report.applied; for (peer, err) in &report.failures { eprintln!("fabric peer reconcile failed: peer={} err={err}", peer.0); + failures.push(format!( + "unit={unit_idx} device={} peer={}: {err}", + unit.device_name, peer.0 + )); } // Publish the full desired set so the background reconnect // thread keeps retrying peers whose dial lost the startup @@ -435,6 +440,12 @@ impl FabricGroup { // pruned from the desired set. unit.fabric.set_desired_peers(desired); } + + if failures.is_empty() { + Ok(()) + } else { + Err(failures) + } } /// RPC owner reads are served through shard bufferpools, whose @@ -519,8 +530,20 @@ fn build_unit( fabric.check_shared_domain_capacity(spec.expected_mr); let desired_peers = runtime_peer_connections_for_unit(peers, spec.unit_idx); - let applied_peers = - config::reconcile::reconcile_connections(&fabric, &desired_peers, None).applied; + let peer_report = config::reconcile::reconcile_connections(&fabric, &desired_peers, None); + if !peer_report.failures.is_empty() { + let details = peer_report + .failures + .iter() + .map(|(peer, error)| format!("peer={}: {error}", peer.0)) + .collect::>() + .join("; "); + return Err(format!( + "worker={worker}: {} peer(s) failed to reconcile: {details}", + peer_report.failures.len() + )); + } + let applied_peers = peer_report.applied; // Seed the desired-peer set so the background reconnect thread can // retry any peer whose initial dial lost the startup race. fabric.set_desired_peers(desired_peers.clone()); diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 15db530c8..98d1e32d6 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -642,7 +642,13 @@ impl ConfigApplyTarget for ProcessApplyTarget { // endpoint here. if diff.requires_peer_reconcile() { let runtime_peers = config::runtime_peers(new.runtime()); - layer.fabric_group.reconcile_peers(&runtime_peers); + if let Err(failures) = layer.fabric_group.reconcile_peers(&runtime_peers) { + crate::SHUTDOWN.store(true, Ordering::Release); + return Err(ApplyError::Target(format!( + "hard peer reconciliation failure(s): {}", + failures.join("; ") + ))); + } } // The RPC-side backend registries also live on the shared From 15c6a31cec2edfa91337011d809f8683d877047b Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 04:31:51 +0000 Subject: [PATCH 50/56] unbounded-storage: separate frontend bind and listen --- .../src/frontend/http_serve.rs | 66 ++++------ .../src/frontend/s3_serve.rs | 61 ++++----- cmd/unbounded-storage/src/http/mod.rs | 4 +- cmd/unbounded-storage/src/http/server.rs | 122 +++++++++++++----- cmd/unbounded-storage/src/main.rs | 8 +- 5 files changed, 147 insertions(+), 114 deletions(-) diff --git a/cmd/unbounded-storage/src/frontend/http_serve.rs b/cmd/unbounded-storage/src/frontend/http_serve.rs index dae619f82..1428a9cf1 100644 --- a/cmd/unbounded-storage/src/frontend/http_serve.rs +++ b/cmd/unbounded-storage/src/frontend/http_serve.rs @@ -32,7 +32,7 @@ use std::collections::VecDeque; use std::future::{Future, poll_fn}; use std::net::SocketAddr; -use std::os::fd::RawFd; +use std::os::fd::{AsRawFd, RawFd}; use std::pin::Pin; use std::rc::Rc; use std::task::{Context, Poll, Waker}; @@ -51,9 +51,9 @@ use crate::frontend::range::{ }; use crate::frontend::serve_metrics::{ConnGuard, ReqOutcome}; use crate::http::{ - FdGuard, HttpRequest, MAX_HEADER_BYTES, Method, ParseError, RECV_CHUNK, bind_listener, - connection_header_value, request_allows_keep_alive, send_all, serialize_response_head, - split_query, + BoundListener, FdGuard, HttpRequest, ListeningSocket, MAX_HEADER_BYTES, Method, ParseError, + RECV_CHUNK, bind_socket, connection_header_value, request_allows_keep_alive, send_all, + serialize_response_head, split_query, }; use crate::ring::NetHandle; use crate::runtime::noop_waker; @@ -118,14 +118,16 @@ impl HttpFrontend { self.max_requests_per_connection } - /// Create, bind, and listen the per-shard accept socket with - /// `SO_REUSEPORT` (and `SO_REUSEADDR`) so every shard accepts on the - /// same port, returning the listening [`RawFd`]. Linux-only. - /// - /// The caller owns the returned fd and is responsible for closing - /// it; [`HttpDriver`] only reads it for `accept`. - pub fn bind_listener(&self) -> Result { - bind_listener(self.addr).map_err(|_| FrontendError::BadBind(self.addr.to_string())) + /// Bind a dormant per-shard `SO_REUSEPORT` socket. + pub fn bind_socket(&self) -> Result { + bind_socket(self.addr).map_err(|_| FrontendError::BadBind(self.addr.to_string())) + } + + /// Bind and activate the per-shard listener. + pub fn bind_listener(&self) -> Result { + self.bind_socket()? + .listen() + .map_err(|_| FrontendError::BadBind(self.addr.to_string())) } } @@ -143,7 +145,6 @@ impl HttpFrontend { pub struct HttpDriver + 'static> { pool: Rc

, handle: NetHandle, - listen_fd: RawFd, frontend_id: Rc, backend_id: String, cache_id: Option, @@ -153,12 +154,14 @@ pub struct HttpDriver + 'static> { bypass: bool, max_requests_per_connection: usize, accept_fut: Pin>>>, + // Declared after the future so the pending accept is cancelled first. + listener: ListeningSocket, conns: Vec>>>, waker: Waker, } impl + 'static> HttpDriver

{ - /// Build a serving engine over a bound `listen_fd`. + /// Build a serving engine that owns an active listener. /// /// `stripe_size` and `page_size` come from the shard's pool /// geometry. `fanout` is this shard's view of the stripe-ownership @@ -168,7 +171,7 @@ impl + 'static> HttpDriver

{ pub fn new( pool: Rc

, handle: NetHandle, - listen_fd: RawFd, + listener: ListeningSocket, frontend_id: Rc, backend_id: String, cache_id: Option, @@ -178,11 +181,10 @@ impl + 'static> HttpDriver

{ bypass: bool, max_requests_per_connection: usize, ) -> Self { - let accept_fut = Box::pin(handle.accept(listen_fd)); + let accept_fut = Box::pin(handle.accept(listener.as_raw_fd())); Self { pool, handle, - listen_fd, frontend_id, backend_id, cache_id, @@ -192,6 +194,7 @@ impl + 'static> HttpDriver

{ bypass, max_requests_per_connection, accept_fut, + listener, conns: Vec::new(), waker: noop_waker(), } @@ -231,7 +234,8 @@ impl + 'static> HttpDriver

{ unsafe { libc::close(fd); } - self.accept_fut = Box::pin(self.handle.accept(self.listen_fd)); + self.accept_fut = + Box::pin(self.handle.accept(self.listener.as_raw_fd())); continue; } let serve = serve_connection( @@ -251,7 +255,7 @@ impl + 'static> HttpDriver

{ } // Rearm regardless of accept success so a transient // accept error does not stop the listener. - self.accept_fut = Box::pin(self.handle.accept(self.listen_fd)); + self.accept_fut = Box::pin(self.handle.accept(self.listener.as_raw_fd())); } Poll::Pending => break, } @@ -271,24 +275,6 @@ impl + 'static> HttpDriver

{ } } -impl + 'static> Drop for HttpDriver

{ - /// Close the listen fd the driver owns. The driver is the sole owner - /// of this `SO_REUSEPORT` listener (the embedder hands it the fd from - /// `bind_listener` and never touches it again), so dropping the - /// driver (on shard shutdown or a live frontend removal) must close - /// it or the fd leaks. The accept future borrows only the fd number, - /// not ownership, so closing here is sound once the driver is gone. - fn drop(&mut self) { - if self.listen_fd >= 0 { - // SAFETY: `listen_fd` was returned by `bind_listener` and is - // owned exclusively by this driver; it is closed exactly once. - unsafe { - libc::close(self.listen_fd); - } - } - } -} - /// /// Owns everything it needs so the future is `'static` and can live in /// the driver's future set across ticks. All paths close `conn_fd` via @@ -1362,8 +1348,8 @@ mod tests { } }; let handle = NetHandle::new(ring); - let listen_fd = match bind_listener("127.0.0.1:0".parse().unwrap()) { - Ok(fd) => fd, + let listener = match bind_socket("127.0.0.1:0".parse().unwrap()).and_then(|b| b.listen()) { + Ok(listener) => listener, Err(e) => { eprintln!("driver_idle_progress: bind failed: {e}; skipping"); return; @@ -1374,7 +1360,7 @@ mod tests { err: Error::from("mock pool has no data"), }), handle, - listen_fd, + listener, Rc::from("primary"), "primary".to_string(), None, diff --git a/cmd/unbounded-storage/src/frontend/s3_serve.rs b/cmd/unbounded-storage/src/frontend/s3_serve.rs index 40c03a89a..640f773b3 100644 --- a/cmd/unbounded-storage/src/frontend/s3_serve.rs +++ b/cmd/unbounded-storage/src/frontend/s3_serve.rs @@ -24,7 +24,7 @@ use std::future::Future; use std::net::SocketAddr; -use std::os::fd::RawFd; +use std::os::fd::{AsRawFd, RawFd}; use std::pin::Pin; use std::rc::Rc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -40,8 +40,8 @@ use crate::frontend::range::{ByteRange, RangeError, ResolvedRange, full_object, use crate::frontend::s3_xml::{S3ErrorCode, error_xml}; use crate::frontend::serve_metrics::{ConnGuard, ReqOutcome}; use crate::http::{ - FdGuard, HttpRequest, MAX_HEADER_BYTES, Method, ParseError, RECV_CHUNK, bind_listener, - send_all, serialize_response_head, split_query, + BoundListener, FdGuard, HttpRequest, ListeningSocket, MAX_HEADER_BYTES, Method, ParseError, + RECV_CHUNK, bind_socket, send_all, serialize_response_head, split_query, }; use crate::ring::NetHandle; use crate::runtime::noop_waker; @@ -102,14 +102,16 @@ impl S3Frontend { self.addr } - /// Create, bind, and listen the per-shard accept socket with - /// `SO_REUSEPORT` (and `SO_REUSEADDR`) so every shard accepts on the - /// same port, returning the listening [`RawFd`]. Linux-only. - /// - /// The caller owns the returned fd and is responsible for closing - /// it; [`S3Driver`] only reads it for `accept`. - pub fn bind_listener(&self) -> Result { - bind_listener(self.addr).map_err(|_| FrontendError::BadBind(self.addr.to_string())) + /// Bind a dormant per-shard `SO_REUSEPORT` socket. + pub fn bind_socket(&self) -> Result { + bind_socket(self.addr).map_err(|_| FrontendError::BadBind(self.addr.to_string())) + } + + /// Bind and activate the per-shard listener. + pub fn bind_listener(&self) -> Result { + self.bind_socket()? + .listen() + .map_err(|_| FrontendError::BadBind(self.addr.to_string())) } } @@ -125,7 +127,6 @@ impl S3Frontend { pub struct S3Driver + 'static> { pool: Rc

, handle: NetHandle, - listen_fd: RawFd, frontend_id: Rc, backend_id: String, cache_id: Option, @@ -133,12 +134,14 @@ pub struct S3Driver + 'static> { page_size: usize, bypass: bool, accept_fut: Pin>>>, + // Declared after the future so the pending accept is cancelled first. + listener: ListeningSocket, conns: Vec>>>, waker: Waker, } impl + 'static> S3Driver

{ - /// Build a serving engine over a bound `listen_fd`. + /// Build a serving engine that owns an active listener. /// /// `stripe_size` and `page_size` come from the shard's pool /// geometry. When `bypass` is set, the frontend bridges straight to @@ -146,7 +149,7 @@ impl + 'static> S3Driver

{ pub fn new( pool: Rc

, handle: NetHandle, - listen_fd: RawFd, + listener: ListeningSocket, frontend_id: Rc, backend_id: String, cache_id: Option, @@ -154,11 +157,10 @@ impl + 'static> S3Driver

{ page_size: usize, bypass: bool, ) -> Self { - let accept_fut = Box::pin(handle.accept(listen_fd)); + let accept_fut = Box::pin(handle.accept(listener.as_raw_fd())); Self { pool, handle, - listen_fd, frontend_id, backend_id, cache_id, @@ -166,6 +168,7 @@ impl + 'static> S3Driver

{ page_size, bypass, accept_fut, + listener, conns: Vec::new(), waker: noop_waker(), } @@ -215,7 +218,7 @@ impl + 'static> S3Driver

{ } // Rearm regardless of accept success so a transient // accept error does not stop the listener. - self.accept_fut = Box::pin(self.handle.accept(self.listen_fd)); + self.accept_fut = Box::pin(self.handle.accept(self.listener.as_raw_fd())); } Poll::Pending => break, } @@ -235,24 +238,6 @@ impl + 'static> S3Driver

{ } } -impl + 'static> Drop for S3Driver

{ - /// Close the listen fd the driver owns. The driver is the sole owner - /// of this `SO_REUSEPORT` listener (the embedder hands it the fd from - /// `bind_listener` and never touches it again), so dropping the - /// driver (on shard shutdown or a live frontend removal) must close - /// it or the fd leaks. The accept future borrows only the fd number, - /// not ownership, so closing here is sound once the driver is gone. - fn drop(&mut self) { - if self.listen_fd >= 0 { - // SAFETY: `listen_fd` was returned by `bind_listener` and is - // owned exclusively by this driver; it is closed exactly once. - unsafe { - libc::close(self.listen_fd); - } - } - } -} - /// Serve one accepted connection end-to-end, then close its fd. /// /// Owns everything it needs so the future is `'static` and can live in @@ -864,8 +849,8 @@ mod tests { } }; let handle = NetHandle::new(ring); - let listen_fd = match bind_listener("127.0.0.1:0".parse().unwrap()) { - Ok(fd) => fd, + let listener = match bind_socket("127.0.0.1:0".parse().unwrap()).and_then(|b| b.listen()) { + Ok(listener) => listener, Err(e) => { eprintln!("driver_idle_progress: bind failed: {e}; skipping"); return; @@ -874,7 +859,7 @@ mod tests { let mut driver = S3Driver::new( Rc::new(MockPool), handle, - listen_fd, + listener, Rc::from("primary"), "primary".to_string(), None, diff --git a/cmd/unbounded-storage/src/http/mod.rs b/cmd/unbounded-storage/src/http/mod.rs index 62d2d8fde..a4daeeddc 100644 --- a/cmd/unbounded-storage/src/http/mod.rs +++ b/cmd/unbounded-storage/src/http/mod.rs @@ -37,6 +37,8 @@ pub use connection::{ pub use headers::{Header, ParseError}; pub use request::{HttpRequest, serialize_request}; pub use response::{ResponseHead, serialize_response_head}; +#[cfg(target_os = "linux")] +pub use server::{BoundListener, ListeningSocket}; /// Cross-platform, storage-policy-free server plumbing reused by the /// HTTP serving frontends. @@ -45,7 +47,7 @@ pub(crate) use server::{MAX_HEADER_BYTES, RECV_CHUNK, split_query}; /// Linux-only server plumbing that depends on `libc` and the io_uring /// [`NetHandle`](crate::ring::NetHandle). #[cfg(target_os = "linux")] -pub(crate) use server::{FdGuard, bind_listener, send_all}; +pub(crate) use server::{FdGuard, bind_socket, send_all}; /// Canonical typed `Method`/`StatusCode`, re-exported from the [`http`] /// crate so the rest of the crate has a single source for them. diff --git a/cmd/unbounded-storage/src/http/server.rs b/cmd/unbounded-storage/src/http/server.rs index bd1fde457..d6db8dcc8 100644 --- a/cmd/unbounded-storage/src/http/server.rs +++ b/cmd/unbounded-storage/src/http/server.rs @@ -6,14 +6,14 @@ //! //! This file owns the reusable, opinion-free pieces a server-side //! frontend needs regardless of which object protocol it speaks: the -//! `SO_REUSEPORT` listener helper, the RAII socket-fd closer, the +//! `SO_REUSEPORT` socket helpers, the RAII socket-fd closer, the //! request-target query splitter, the full-buffer send loop, and the //! size constants. None of it knows about ranges, stripe geometry, or //! which status a frontend returns; that policy lives with the concrete //! frontend (`crate::frontend::http_serve`) and a future S3 frontend //! can reuse everything here unchanged. //! -//! The libc/`io_uring`-bearing helpers ([`bind_listener`], [`FdGuard`], +//! The libc/`io_uring`-bearing helpers ([`bind_socket`], [`FdGuard`], //! [`send_all`]) are Linux-gated because they depend on `libc` and the //! io_uring [`NetHandle`](crate::ring::NetHandle); the pure helpers //! ([`split_query`]) and the constants are cross-platform. @@ -21,7 +21,7 @@ #[cfg(target_os = "linux")] use std::net::SocketAddr; #[cfg(target_os = "linux")] -use std::os::fd::RawFd; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; #[cfg(target_os = "linux")] use crate::ring::NetHandle; @@ -37,16 +37,49 @@ pub(crate) const RECV_CHUNK: usize = 64 * 1024; #[cfg(target_os = "linux")] const LISTEN_BACKLOG: i32 = 1024; -/// Create, bind, and listen a TCP socket on `addr` with `SO_REUSEADDR` +/// A bound TCP socket that is not yet eligible to receive traffic. +#[cfg(target_os = "linux")] +pub struct BoundListener { + fd: OwnedFd, +} + +/// An active listening socket owned by a frontend driver. +#[cfg(target_os = "linux")] +pub struct ListeningSocket { + fd: OwnedFd, +} + +#[cfg(target_os = "linux")] +impl BoundListener { + /// Enter the listening state. Consuming `self` keeps failure cleanup + /// automatic and makes the dormant-to-active transition explicit. + pub fn listen(self) -> Result { + // SAFETY: fd is a bound stream socket. + if unsafe { libc::listen(self.fd.as_raw_fd(), LISTEN_BACKLOG) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(ListeningSocket { fd: self.fd }) + } +} + +#[cfg(target_os = "linux")] +impl AsRawFd for BoundListener { + fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } +} + +#[cfg(target_os = "linux")] +impl AsRawFd for ListeningSocket { + fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } +} + +/// Create and bind a dormant TCP socket on `addr` with `SO_REUSEADDR` /// and `SO_REUSEPORT`. Supports IPv4 and IPv6 binds. -/// -/// Returns the listening [`RawFd`]; the caller owns it and is -/// responsible for closing it. Errors are surfaced as -/// [`std::io::Error`] (from the failing syscall) so this stays free of -/// any frontend-specific error type; the caller maps it onto whatever -/// error it reports. #[cfg(target_os = "linux")] -pub(crate) fn bind_listener(addr: SocketAddr) -> Result { +pub(crate) fn bind_socket(addr: SocketAddr) -> Result { let family = match addr { SocketAddr::V4(_) => libc::AF_INET, SocketAddr::V6(_) => libc::AF_INET6, @@ -56,14 +89,15 @@ pub(crate) fn bind_listener(addr: SocketAddr) -> Result { if fd < 0 { return Err(std::io::Error::last_os_error()); } - let guard = FdGuard(fd); + // SAFETY: fd was just returned by socket() and has unique ownership. + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; let one: libc::c_int = 1; for opt in [libc::SO_REUSEADDR, libc::SO_REUSEPORT] { // SAFETY: &one outlives the call; size matches a c_int. let rc = unsafe { libc::setsockopt( - fd, + fd.as_raw_fd(), libc::SOL_SOCKET, opt, &one as *const libc::c_int as *const libc::c_void, @@ -88,7 +122,7 @@ pub(crate) fn bind_listener(addr: SocketAddr) -> Result { // SAFETY: sin is a valid sockaddr_in for the call's duration. unsafe { libc::bind( - fd, + fd.as_raw_fd(), &sin as *const libc::sockaddr_in as *const libc::sockaddr, std::mem::size_of::() as libc::socklen_t, ) @@ -107,7 +141,7 @@ pub(crate) fn bind_listener(addr: SocketAddr) -> Result { // SAFETY: sin6 is a valid sockaddr_in6 for the call's duration. unsafe { libc::bind( - fd, + fd.as_raw_fd(), &sin6 as *const libc::sockaddr_in6 as *const libc::sockaddr, std::mem::size_of::() as libc::socklen_t, ) @@ -118,14 +152,7 @@ pub(crate) fn bind_listener(addr: SocketAddr) -> Result { return Err(std::io::Error::last_os_error()); } - // SAFETY: fd is a bound stream socket. - if unsafe { libc::listen(fd, LISTEN_BACKLOG) } != 0 { - return Err(std::io::Error::last_os_error()); - } - - // Hand the fd out to the caller; defuse the guard. - std::mem::forget(guard); - Ok(fd) + Ok(BoundListener { fd }) } /// RAII closer for a socket fd. The ring never creates fds; this owns @@ -184,6 +211,11 @@ pub(crate) fn split_query(target: &str) -> (&str, &str) { mod tests { use super::*; + #[cfg(target_os = "linux")] + use std::net::TcpStream; + #[cfg(target_os = "linux")] + use std::os::fd::AsRawFd; + #[test] fn split_query_strips_query() { assert_eq!(split_query("/bucket/key"), ("/bucket/key", "")); @@ -196,15 +228,43 @@ mod tests { #[cfg(target_os = "linux")] #[test] - fn bind_listener_creates_and_binds() { - match bind_listener("127.0.0.1:0".parse().unwrap()) { - Ok(fd) => { - assert!(fd >= 0); - unsafe { - libc::close(fd); - } + fn bound_socket_can_enter_listening_state() { + match bind_socket("127.0.0.1:0".parse().unwrap()) { + Ok(bound) => assert!(bound.listen().is_ok()), + Err(e) => { + eprintln!("bound_socket_can_enter_listening_state: bind failed: {e}; skipping") } - Err(e) => eprintln!("bind_listener_creates_and_binds: bind failed: {e}; skipping"), } } + + #[cfg(target_os = "linux")] + #[test] + fn bound_socket_is_dormant_until_listen() { + let bound = match bind_socket("127.0.0.1:0".parse().unwrap()) { + Ok(bound) => bound, + Err(e) => { + eprintln!("bound_socket_is_dormant_until_listen: bind failed: {e}; skipping"); + return; + } + }; + let mut addr: libc::sockaddr_in = unsafe { std::mem::zeroed() }; + let mut len = std::mem::size_of::() as libc::socklen_t; + // SAFETY: addr and len provide valid output storage for getsockname. + assert_eq!( + unsafe { + libc::getsockname( + bound.as_raw_fd(), + &mut addr as *mut libc::sockaddr_in as *mut libc::sockaddr, + &mut len, + ) + }, + 0 + ); + let local = SocketAddr::from(([127, 0, 0, 1], u16::from_be(addr.sin_port))); + + assert!(TcpStream::connect(local).is_err()); + let listening = bound.listen().expect("listen after dormant bind"); + assert!(TcpStream::connect(local).is_ok()); + drop(listening); + } } diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index b5dbb694a..c83da8be4 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -1244,13 +1244,13 @@ impl FrontendBuildCtx { Some(frontend_spec::Config::Http(_)) => { let frontend = HttpFrontend::from_spec(spec) .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; - let listen_fd = frontend + let listener = frontend .bind_listener() .map_err(|e| format!("frontend {} bind_listener: {e}", spec.name))?; Ok(ShardFrontendDriver::Http(HttpDriver::new( self.pool.clone(), self.handle.clone(), - listen_fd, + listener, Rc::from(spec.name.as_str()), binding.backend_id.clone(), binding.cache_id.clone(), @@ -1264,13 +1264,13 @@ impl FrontendBuildCtx { Some(frontend_spec::Config::S3(_)) => { let frontend = S3Frontend::from_spec(spec) .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; - let listen_fd = frontend + let listener = frontend .bind_listener() .map_err(|e| format!("frontend {} bind_listener: {e}", spec.name))?; Ok(ShardFrontendDriver::S3(S3Driver::new( self.pool.clone(), self.handle.clone(), - listen_fd, + listener, Rc::from(spec.name.as_str()), binding.backend_id.clone(), binding.cache_id.clone(), From d3adbe55e621bcf7d6c36493e72060fda9d1a665 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 04:37:27 +0000 Subject: [PATCH 51/56] unbounded-storage: prepare frontends before activation --- cmd/unbounded-storage/ARCHITECTURE.md | 15 +-- cmd/unbounded-storage/src/main.rs | 141 +++++++++++++++++++------- 2 files changed, 115 insertions(+), 41 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 5a268402f..e4ce9cad4 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -178,8 +178,10 @@ shutdown; teardown drains terminal reports and returns a failing process status instead of continuing with a missing shard. Backend and frontend reloads construct replacements before swapping them into -their shard-local registries. A failed replacement therefore leaves the prior -resource, binding, and stripe geometry live and retryable. This is a +their shard-local registries. Network frontends first validate and bind a +dormant `SO_REUSEPORT` socket, then enter the listening state only during +activation. A failed preparation therefore leaves the prior resource, binding, +and stripe geometry live and retryable. This is a per-resource guarantee; route, peer, shard, and disk publication across the whole process is not yet one transaction. @@ -220,10 +222,11 @@ per-shard `!Send` object graph: `LiveShardLocalStore` because a `PageRef` resolves through exactly one backing's geometry. 6. **Frontends.** A shard hosts a `FrontendRegistry` of any number of - frontends keyed by component name. Each spec binds its listener with `SO_REUSEPORT` and - builds an `HttpDriver`/`S3Driver`; the registry can add and remove frontends - in place on a live config apply (a removed driver's `Drop` closes its - listener fd). + frontends keyed by component name. Each network frontend is prepared as a + bound but non-listening `SO_REUSEPORT` socket, then activated into an + `HttpDriver`/`S3Driver` that owns the listening socket. The registry can add + and remove frontends in place on a live config apply; RAII closes removed + listeners after their pending accept future is dropped. 7. **Tick loop.** Register tick hooks (socket-ring `progress()`, the control-drain hook that reconciles backends/frontends/routing from applied configs, and the frontend registry's `progress()`), report `Up`, and run diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index c83da8be4..b60f8fa1d 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -27,6 +27,7 @@ use unbounded_storage::fanout::{ use unbounded_storage::frontend::{ HttpDriver, HttpFrontend, LoadgenDriver, LoadgenFrontend, S3Driver, S3Frontend, }; +use unbounded_storage::http::BoundListener; use unbounded_storage::p2p::{RouteTableHandle, RoutedTransport}; use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; @@ -1157,6 +1158,27 @@ impl ShardFrontendDriver { } } +/// A validated frontend whose fallible address binding has completed, +/// but which is not yet visible to traffic or driven by the shard loop. +enum PreparedShardFrontend { + Http { + frontend: HttpFrontend, + socket: BoundListener, + binding: ResolvedFrontendBinding, + stripe_size: u64, + }, + Loadgen { + frontend: LoadgenFrontend, + binding: ResolvedFrontendBinding, + stripe_size: u64, + }, + S3 { + socket: BoundListener, + binding: ResolvedFrontendBinding, + stripe_size: u64, + }, +} + /// Shard-local build context for frontend drivers: everything a /// [`FrontendSpec`] needs to become a running [`ShardFrontendDriver`] /// except the spec itself. Cheap to clone (all fields are handles), so @@ -1226,13 +1248,8 @@ fn frontend_rebuild_ids( } impl FrontendBuildCtx { - /// Turn one [`FrontendSpec`] into a bound, ready-to-drive - /// [`ShardFrontendDriver`]. Validates the spec, binds the shard's - /// `SO_REUSEPORT` listener, and selects the stripe size from the - /// referenced backend's geometry (falling back to the default if the - /// backend is not yet known). Returns a human-readable error string - /// so it slots into the reconcile traits' `Result<_, String>`. - fn build(&self, spec: &FrontendSpec) -> Result { + /// Validate and bind one frontend without making it traffic-visible. + fn prepare(&self, spec: &FrontendSpec) -> Result { let binding = self .bindings .borrow() @@ -1244,14 +1261,62 @@ impl FrontendBuildCtx { Some(frontend_spec::Config::Http(_)) => { let frontend = HttpFrontend::from_spec(spec) .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; - let listener = frontend - .bind_listener() - .map_err(|e| format!("frontend {} bind_listener: {e}", spec.name))?; + let socket = frontend + .bind_socket() + .map_err(|e| format!("frontend {} bind_socket: {e}", spec.name))?; + Ok(PreparedShardFrontend::Http { + frontend, + socket, + binding, + stripe_size, + }) + } + Some(frontend_spec::Config::S3(_)) => { + let frontend = S3Frontend::from_spec(spec) + .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; + let socket = frontend + .bind_socket() + .map_err(|e| format!("frontend {} bind_socket: {e}", spec.name))?; + Ok(PreparedShardFrontend::S3 { + socket, + binding, + stripe_size, + }) + } + Some(frontend_spec::Config::Loadgen(_)) => { + let frontend = LoadgenFrontend::from_spec(spec) + .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; + Ok(PreparedShardFrontend::Loadgen { + frontend, + binding, + stripe_size, + }) + } + None => Err(format!("frontend {} missing config", spec.name)), + } + } + + /// Activate a prepared frontend and build its shard-local driver. + fn activate( + &self, + id: &str, + prepared: PreparedShardFrontend, + ) -> Result { + match prepared { + PreparedShardFrontend::Http { + frontend, + socket, + binding, + stripe_size, + } => { + let listener = socket + .listen() + .map_err(|e| format!("frontend {id} listen: {e}"))?; Ok(ShardFrontendDriver::Http(HttpDriver::new( self.pool.clone(), self.handle.clone(), listener, - Rc::from(spec.name.as_str()), + Rc::from(id), binding.backend_id.clone(), binding.cache_id.clone(), stripe_size, @@ -1261,17 +1326,19 @@ impl FrontendBuildCtx { frontend.max_requests_per_connection(), ))) } - Some(frontend_spec::Config::S3(_)) => { - let frontend = S3Frontend::from_spec(spec) - .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; - let listener = frontend - .bind_listener() - .map_err(|e| format!("frontend {} bind_listener: {e}", spec.name))?; + PreparedShardFrontend::S3 { + socket, + binding, + stripe_size, + } => { + let listener = socket + .listen() + .map_err(|e| format!("frontend {id} listen: {e}"))?; Ok(ShardFrontendDriver::S3(S3Driver::new( self.pool.clone(), self.handle.clone(), listener, - Rc::from(spec.name.as_str()), + Rc::from(id), binding.backend_id.clone(), binding.cache_id.clone(), stripe_size, @@ -1279,25 +1346,29 @@ impl FrontendBuildCtx { binding.bypass_cache, ))) } - Some(frontend_spec::Config::Loadgen(_)) => { - let frontend = LoadgenFrontend::from_spec(spec) - .map_err(|e| format!("frontend {} from_spec: {e}", spec.name))?; - Ok(ShardFrontendDriver::Loadgen(LoadgenDriver::new( - frontend, - self.pool.clone(), - binding.backend_id.clone(), - binding.cache_id.clone(), - stripe_size, - self.page_size, - self.routes.clone(), - binding.bypass_cache, - self.worker_idx, - self.waker.clone(), - ))) - } - None => Err(format!("frontend {} missing config", spec.name)), + PreparedShardFrontend::Loadgen { + frontend, + binding, + stripe_size, + } => Ok(ShardFrontendDriver::Loadgen(LoadgenDriver::new( + frontend, + self.pool.clone(), + binding.backend_id.clone(), + binding.cache_id.clone(), + stripe_size, + self.page_size, + self.routes.clone(), + binding.bypass_cache, + self.worker_idx, + self.waker.clone(), + ))), } } + + fn build(&self, spec: &FrontendSpec) -> Result { + let prepared = self.prepare(spec)?; + self.activate(&spec.name, prepared) + } } /// Shard-local registry of running frontend drivers, keyed by frontend From 2e97c281ed779fb974af3289a411133ddde62392 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 04:48:44 +0000 Subject: [PATCH 52/56] unbounded-storage: transact shard-local frontend replacement --- cmd/unbounded-storage/ARCHITECTURE.md | 10 +- cmd/unbounded-storage/src/main.rs | 365 +++++++++++++----- .../src/registry_transaction.rs | 69 ++++ 3 files changed, 355 insertions(+), 89 deletions(-) create mode 100644 cmd/unbounded-storage/src/registry_transaction.rs diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index e4ce9cad4..324daa794 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -181,9 +181,13 @@ Backend and frontend reloads construct replacements before swapping them into their shard-local registries. Network frontends first validate and bind a dormant `SO_REUSEPORT` socket, then enter the listening state only during activation. A failed preparation therefore leaves the prior resource, binding, -and stripe geometry live and retryable. This is a -per-resource guarantee; route, peer, shard, and disk publication across the -whole process is not yet one transaction. +and stripe geometry live and retryable. Active frontend entries record the +binding and stripe geometry they realized, so a failed rebuild is detected and +retried on the next apply. Replacement uses an undo-log transaction: the old +driver remains owned until finalization and can be restored if a later local +activation fails. The current reconciler finalizes each frontend operation +immediately; cross-resource and cross-shard commit/rollback is not yet wired to +that transaction boundary. ### Shutdown diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index b60f8fa1d..76b3af338 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -42,8 +42,11 @@ use unbounded_storage::metrics; mod device_inventory; mod fabric_group; +mod registry_transaction; mod shard_layer; +use registry_transaction::RegistryTransaction; + const DEFAULT_CONFIG_PATH: &str = "/etc/unbounded-storage/config.toml"; const SHUTDOWN_POLL: Duration = Duration::from_millis(100); @@ -879,10 +882,10 @@ fn run_shard( // atomically by its transport; the fabric RPC handlers are reloaded // separately by the `FabricGroup`), refreshes the stripe geometry, // reconciles the transport origin-backend registry and the frontend - // registry toward the new config, applies disk-policy side effects, and then acknowledges - // so the coordinator's blocking apply can complete. Everything is - // driven from this one thread so the `ArcSwap` publishes are ordered - // and the build-from-spec (DNS resolve, listener bind) stays off the + // registry toward the new config, applies disk-policy side effects, + // and then acknowledges so the coordinator's blocking apply can + // complete. Everything is driven from this one thread, and the + // build-from-spec work (DNS resolve, listener bind) stays off the // fast path. { let routes = routes.clone(); @@ -899,7 +902,6 @@ fn run_shard( .iter() .map(|f| (f.name.clone(), f.clone())) .collect(); - let mut last_bindings: HashMap = frontend_bindings.clone(); shard_loop.add_tick_hook(move || { let mut did_work = false; while let Ok(cmd) = ctrl_rx.try_recv() { @@ -920,12 +922,14 @@ fn run_shard( } } - let frontends_to_rebuild = frontend_rebuild_ids( - &last_bindings, + let desired_activations = frontend_activations( &apply.loaded.runtime().frontends, - &last_backends, desired_backends, ); + let frontends_to_rebuild = frontend_rebuild_ids( + &frontend_registry.realized_activations(), + &desired_activations, + ); let forced_frontend_updates: HashSet = frontends_to_rebuild.into_iter().collect(); *bindings.borrow_mut() = apply.loaded.runtime().frontends.clone(); @@ -956,21 +960,11 @@ fn run_shard( g.insert(spec.name.clone(), spec.stripe_size_bytes()); } } - let mut applied_bindings = HashMap::new(); - for (id, spec) in &last_frontends { - let desired_spec = desired_frontends.iter().find(|f| f.name == *id); - if desired_spec == Some(spec) { - if let Some(binding) = apply.loaded.runtime().frontends.get(id) { - applied_bindings.insert(id.clone(), binding.clone()); - continue; - } - } - if let Some(binding) = last_bindings.get(id) { - applied_bindings.insert(id.clone(), binding.clone()); - } - } - *bindings.borrow_mut() = applied_bindings.clone(); - last_bindings = applied_bindings; + *bindings.borrow_mut() = frontend_registry + .realized_activations() + .into_iter() + .map(|(id, activation)| (id, activation.binding)) + .collect(); let mut failures = combined.backends.failures; failures.extend(combined.frontends.failures); @@ -1148,6 +1142,17 @@ enum ShardFrontendDriver { S3(S3Driver), } +#[derive(Clone, Debug, PartialEq)] +struct FrontendActivation { + binding: ResolvedFrontendBinding, + stripe_size: u64, +} + +struct ActiveShardFrontend { + driver: ShardFrontendDriver, + activation: FrontendActivation, +} + impl ShardFrontendDriver { fn progress(&mut self) -> bool { match self { @@ -1221,26 +1226,44 @@ fn frontend_stripe_size(geometry: &HashMap, backend_id: &str) -> u6 .unwrap_or(DEFAULT_STRIPE_SIZE_BYTES) } -fn frontend_rebuild_ids( - last_bindings: &HashMap, - next_bindings: &HashMap, - last_backends: &HashMap, +fn frontend_activations( + bindings: &HashMap, desired_backends: &[BackendSpec], -) -> Vec { - let changed_backend_geometry: HashSet = desired_backends +) -> HashMap { + let geometry: HashMap<&str, u64> = desired_backends .iter() - .filter_map(|backend| { - let old = last_backends.get(&backend.name)?; - (old.stripe_size_bytes() != backend.stripe_size_bytes()).then(|| backend.name.clone()) - }) + .map(|backend| (backend.name.as_str(), backend.stripe_size_bytes())) .collect(); - let mut rebuild: Vec = next_bindings + bindings + .iter() + .map(|(id, binding)| { + let stripe_size = geometry + .get(binding.backend_id.as_str()) + .copied() + .unwrap_or(DEFAULT_STRIPE_SIZE_BYTES); + ( + id.clone(), + FrontendActivation { + binding: binding.clone(), + stripe_size, + }, + ) + }) + .collect() +} + +fn frontend_rebuild_ids( + realized: &HashMap, + desired: &HashMap, +) -> Vec { + let mut rebuild: Vec = desired .iter() - .filter_map(|(id, binding)| { - (last_bindings.get(id) != Some(binding) - || changed_backend_geometry.contains(&binding.backend_id)) - .then(|| id.clone()) + .filter_map(|(id, activation)| { + realized + .get(id) + .is_some_and(|current| current != activation) + .then(|| id.clone()) }) .collect(); rebuild.sort(); @@ -1301,8 +1324,8 @@ impl FrontendBuildCtx { &self, id: &str, prepared: PreparedShardFrontend, - ) -> Result { - match prepared { + ) -> Result { + let (driver, activation) = match prepared { PreparedShardFrontend::Http { frontend, socket, @@ -1312,7 +1335,7 @@ impl FrontendBuildCtx { let listener = socket .listen() .map_err(|e| format!("frontend {id} listen: {e}"))?; - Ok(ShardFrontendDriver::Http(HttpDriver::new( + let driver = ShardFrontendDriver::Http(HttpDriver::new( self.pool.clone(), self.handle.clone(), listener, @@ -1324,7 +1347,12 @@ impl FrontendBuildCtx { self.fanout.clone(), binding.bypass_cache, frontend.max_requests_per_connection(), - ))) + )); + let activation = FrontendActivation { + binding, + stripe_size, + }; + (driver, activation) } PreparedShardFrontend::S3 { socket, @@ -1334,7 +1362,7 @@ impl FrontendBuildCtx { let listener = socket .listen() .map_err(|e| format!("frontend {id} listen: {e}"))?; - Ok(ShardFrontendDriver::S3(S3Driver::new( + let driver = ShardFrontendDriver::S3(S3Driver::new( self.pool.clone(), self.handle.clone(), listener, @@ -1344,33 +1372,72 @@ impl FrontendBuildCtx { stripe_size, self.page_size, binding.bypass_cache, - ))) + )); + let activation = FrontendActivation { + binding, + stripe_size, + }; + (driver, activation) } PreparedShardFrontend::Loadgen { frontend, binding, stripe_size, - } => Ok(ShardFrontendDriver::Loadgen(LoadgenDriver::new( - frontend, - self.pool.clone(), - binding.backend_id.clone(), - binding.cache_id.clone(), - stripe_size, - self.page_size, - self.routes.clone(), - binding.bypass_cache, - self.worker_idx, - self.waker.clone(), - ))), - } + } => { + let driver = ShardFrontendDriver::Loadgen(LoadgenDriver::new( + frontend, + self.pool.clone(), + binding.backend_id.clone(), + binding.cache_id.clone(), + stripe_size, + self.page_size, + self.routes.clone(), + binding.bypass_cache, + self.worker_idx, + self.waker.clone(), + )); + let activation = FrontendActivation { + binding, + stripe_size, + }; + (driver, activation) + } + }; + Ok(ActiveShardFrontend { driver, activation }) } - fn build(&self, spec: &FrontendSpec) -> Result { + fn build(&self, spec: &FrontendSpec) -> Result { let prepared = self.prepare(spec)?; self.activate(&spec.name, prepared) } } +struct FrontendRegistryTransaction { + ctx: FrontendBuildCtx, + entries: RegistryTransaction, +} + +impl FrontendRegistryTransaction { + fn add(&mut self, spec: &FrontendSpec) -> Result<(), String> { + let frontend = self.ctx.build(spec)?; + self.entries.replace(spec.name.clone(), frontend); + Ok(()) + } + + fn remove(&mut self, id: &str) { + self.entries.remove(id); + } + + fn finalize(self) { + self.entries.finalize(); + } + + #[cfg(test)] + fn rollback(self) { + self.entries.rollback(); + } +} + /// Shard-local registry of running frontend drivers, keyed by frontend /// name. A single permanent tick hook drives whichever drivers are live, /// and the control-drain hook adds/removes drivers on a config apply; @@ -1380,7 +1447,7 @@ impl FrontendBuildCtx { /// out of the backend registry resolving that name. #[derive(Clone)] struct FrontendRegistry { - drivers: Rc>>, + frontends: Rc>>, ctx: FrontendBuildCtx, } @@ -1391,15 +1458,15 @@ impl FrontendRegistry { /// than silently serving a subset. fn new(specs: &[FrontendSpec], ctx: FrontendBuildCtx) -> Result { let registry = Self { - drivers: Rc::new(RefCell::new(HashMap::with_capacity(specs.len()))), + frontends: Rc::new(RefCell::new(HashMap::with_capacity(specs.len()))), ctx, }; for spec in specs { - let driver = registry.ctx.build(spec)?; + let frontend = registry.ctx.build(spec)?; registry - .drivers + .frontends .borrow_mut() - .insert(spec.name.clone(), driver); + .insert(spec.name.clone(), frontend); } Ok(registry) } @@ -1410,28 +1477,44 @@ impl FrontendRegistry { /// reconcile that mutates the same map. fn progress(&self) -> bool { let mut busy = false; - for driver in self.drivers.borrow_mut().values_mut() { - busy |= driver.progress(); + for frontend in self.frontends.borrow_mut().values_mut() { + busy |= frontend.driver.progress(); } busy } + + fn transaction(&self) -> FrontendRegistryTransaction { + FrontendRegistryTransaction { + ctx: self.ctx.clone(), + entries: RegistryTransaction::new(self.frontends.clone()), + } + } + + fn realized_activations(&self) -> HashMap { + self.frontends + .borrow() + .iter() + .map(|(id, frontend)| (id.clone(), frontend.activation.clone())) + .collect() + } } impl config::reconcile::FrontendReconcileTarget for FrontendRegistry { fn list(&self) -> Vec { - self.drivers.borrow().keys().cloned().collect() + self.frontends.borrow().keys().cloned().collect() } fn add(&self, spec: &FrontendSpec) -> Result<(), String> { - let driver = self.ctx.build(spec)?; - self.drivers.borrow_mut().insert(spec.name.clone(), driver); + let mut transaction = self.transaction(); + transaction.add(spec)?; + transaction.finalize(); Ok(()) } fn remove(&self, id: &str) -> Result<(), String> { - // Dropping the driver runs its `Drop`, which closes the listen - // fd, so a removed frontend stops accepting immediately. - self.drivers.borrow_mut().remove(id); + let mut transaction = self.transaction(); + transaction.remove(id); + transaction.finalize(); Ok(()) } } @@ -2107,33 +2190,143 @@ mod tests { #[test] fn backend_stripe_size_change_rebuilds_frontend() { - let mut last_bindings = HashMap::new(); - last_bindings.insert("f".to_string(), binding("f", "b")); - - let mut next_bindings = HashMap::new(); - next_bindings.insert("f".to_string(), binding("f", "b")); - let old_backend = backend_spec("b"); - let mut new_backend = old_backend.clone(); + let mut new_backend = old_backend; let Some(config::backend_spec::Config::Http(cfg)) = new_backend.config.as_mut() else { panic!("expected http backend config"); }; *cfg.stripe_size_bytes.as_mut().expect("stripe size set") *= 2; - let mut last_backends = HashMap::new(); - last_backends.insert(old_backend.name.clone(), old_backend); + let bindings = HashMap::from([("f".to_string(), binding("f", "b"))]); + let realized = HashMap::from([( + "f".to_string(), + FrontendActivation { + binding: binding("f", "b"), + stripe_size: 4 * 1024 * 1024, + }, + )]); + let desired = frontend_activations(&bindings, &[new_backend]); + + assert_eq!( + frontend_rebuild_ids(&realized, &desired), + vec!["f".to_string()], + ); + } + + #[test] + fn binding_change_rebuilds_frontend_from_realized_state() { + let realized = HashMap::from([( + "f".to_string(), + FrontendActivation { + binding: binding("f", "old"), + stripe_size: 4 * 1024 * 1024, + }, + )]); + let desired = HashMap::from([( + "f".to_string(), + FrontendActivation { + binding: binding("f", "new"), + stripe_size: 4 * 1024 * 1024, + }, + )]); assert_eq!( - frontend_rebuild_ids( - &last_bindings, - &next_bindings, - &last_backends, - &[new_backend] - ), + frontend_rebuild_ids(&realized, &desired), vec!["f".to_string()], ); } + #[derive(Clone)] + struct DropValue { + id: &'static str, + drops: Rc>>, + } + + impl Drop for DropValue { + fn drop(&mut self) { + self.drops.borrow_mut().push(self.id); + } + } + + fn drop_value(id: &'static str, drops: &Rc>>) -> DropValue { + DropValue { + id, + drops: drops.clone(), + } + } + + #[test] + fn registry_transaction_finalize_commits_replacement() { + let drops = Rc::new(RefCell::new(Vec::new())); + let live = Rc::new(RefCell::new(HashMap::from([( + "a".to_string(), + drop_value("old", &drops), + )]))); + let mut transaction = RegistryTransaction::new(live.clone()); + transaction.replace("a".to_string(), drop_value("new", &drops)); + + assert_eq!(live.borrow()["a"].id, "new"); + assert!(drops.borrow().is_empty()); + transaction.finalize(); + + assert_eq!(&*drops.borrow(), &["old"]); + assert_eq!(live.borrow()["a"].id, "new"); + } + + #[test] + fn registry_transaction_rollback_restores_originals() { + let drops = Rc::new(RefCell::new(Vec::new())); + let live = Rc::new(RefCell::new(HashMap::from([ + ("a".to_string(), drop_value("old-a", &drops)), + ("b".to_string(), drop_value("old-b", &drops)), + ]))); + let mut transaction = RegistryTransaction::new(live.clone()); + transaction.replace("a".to_string(), drop_value("new-a", &drops)); + transaction.replace("b".to_string(), drop_value("new-b", &drops)); + transaction.rollback(); + + assert_eq!(live.borrow()["a"].id, "old-a"); + assert_eq!(live.borrow()["b"].id, "old-b"); + let mut dropped = drops.borrow().clone(); + dropped.sort_unstable(); + assert_eq!(dropped, ["new-a", "new-b"]); + } + + #[test] + fn registry_transaction_drop_rolls_back_removal() { + let drops = Rc::new(RefCell::new(Vec::new())); + let live = Rc::new(RefCell::new(HashMap::from([( + "a".to_string(), + drop_value("old", &drops), + )]))); + { + let mut transaction = RegistryTransaction::new(live.clone()); + transaction.remove("a"); + assert!(!live.borrow().contains_key("a")); + } + + assert_eq!(live.borrow()["a"].id, "old"); + assert!(drops.borrow().is_empty()); + } + + #[test] + fn registry_transaction_preserves_first_original() { + let drops = Rc::new(RefCell::new(Vec::new())); + let live = Rc::new(RefCell::new(HashMap::from([( + "a".to_string(), + drop_value("old", &drops), + )]))); + let mut transaction = RegistryTransaction::new(live.clone()); + transaction.replace("a".to_string(), drop_value("new-1", &drops)); + transaction.replace("a".to_string(), drop_value("new-2", &drops)); + transaction.rollback(); + + assert_eq!(live.borrow()["a"].id, "old"); + let mut dropped = drops.borrow().clone(); + dropped.sort_unstable(); + assert_eq!(dropped, ["new-1", "new-2"]); + } + #[test] fn log_backend_registry_counts_specs() { assert_eq!(log_backend_registry(WorkerIdx(0), &[]), 0); diff --git a/cmd/unbounded-storage/src/registry_transaction.rs b/cmd/unbounded-storage/src/registry_transaction.rs new file mode 100644 index 000000000..6535f3f0f --- /dev/null +++ b/cmd/unbounded-storage/src/registry_transaction.rs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +pub(crate) struct RegistryTransaction { + live: Rc>>, + originals: HashMap>, + completed: bool, +} + +impl RegistryTransaction { + pub(crate) fn new(live: Rc>>) -> Self { + Self { + live, + originals: HashMap::new(), + completed: false, + } + } + + pub(crate) fn replace(&mut self, id: String, value: T) { + self.record_original(&id); + self.live.borrow_mut().insert(id, value); + } + + pub(crate) fn remove(&mut self, id: &str) { + self.record_original(id); + self.live.borrow_mut().remove(id); + } + + pub(crate) fn finalize(mut self) { + self.completed = true; + self.originals.clear(); + } + + #[cfg(test)] + pub(crate) fn rollback(mut self) { + self.rollback_inner(); + self.completed = true; + } + + fn record_original(&mut self, id: &str) { + if self.originals.contains_key(id) { + return; + } + let original = self.live.borrow_mut().remove(id); + self.originals.insert(id.to_string(), original); + } + + fn rollback_inner(&mut self) { + let mut live = self.live.borrow_mut(); + for (id, original) in self.originals.drain() { + live.remove(&id); + if let Some(original) = original { + live.insert(id, original); + } + } + } +} + +impl Drop for RegistryTransaction { + fn drop(&mut self) { + if !self.completed { + self.rollback_inner(); + } + } +} From eb3e25e929bfd58a7fe475b7b95fb117c2aab800 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 19:50:06 +0000 Subject: [PATCH 53/56] unbounded-storage: guard stale evictions --- cmd/unbounded-storage/src/storage/engine.rs | 150 +++++++++++++++---- cmd/unbounded-storage/src/storage/mutator.rs | 24 +-- 2 files changed, 138 insertions(+), 36 deletions(-) diff --git a/cmd/unbounded-storage/src/storage/engine.rs b/cmd/unbounded-storage/src/storage/engine.rs index ffc0c7310..fda337724 100644 --- a/cmd/unbounded-storage/src/storage/engine.rs +++ b/cmd/unbounded-storage/src/storage/engine.rs @@ -412,7 +412,6 @@ impl StorageEngine { if victims.is_empty() { return; } - let keys = victims.iter().map(|resident| resident.key).collect(); // The btree mutation MUST commit before the allocator // slot is released: a concurrent writer that observes // the stale btree entry would otherwise call @@ -422,11 +421,11 @@ impl StorageEngine { // construction once `done.wait()` returns. let done = MutatorReply::new(); self.mutator_queue.push(MutatorReq::Delete { - keys, + victims: victims.clone(), done: done.clone(), }); - match done.wait().await { - MutatorOutcome::DeleteCommitted => {} + let removed = match done.wait().await { + MutatorOutcome::DeleteCommitted { removed } => removed, _ => { // Apply failed or queue closed: restore the // resident records so a later sweep retries. @@ -435,15 +434,15 @@ impl StorageEngine { } return; } - } - for resident in &victims { + }; + for resident in &removed { let _ = self .allocator .free_range(resident.lba, self.n_pages(resident.byte_len)); let _ = self.refcount.reset(resident.lba.0); } - self.metric(|m| m.evictions += victims.len() as u64); - for _ in &victims { + self.metric(|m| m.evictions += removed.len() as u64); + for _ in &removed { crate::metrics::storage_eviction(self.disk()); } } @@ -696,7 +695,7 @@ impl StorageEngine { let _ = self.allocator.free_range(lba, n_pages); return Err(bufferpool::Error::Io(libc::EIO)); } - MutatorOutcome::DeleteCommitted => { + MutatorOutcome::DeleteCommitted { .. } => { unreachable!("mutator returned DeleteCommitted for an Insert request") } }; @@ -765,16 +764,9 @@ impl StorageEngine { if batch.is_empty() { return; } - // Look up prior LBAs for every Insert before applying; - // the mutator is the single committer so these reads are - // consistent with what `apply_batch` is about to replace. - // Singleflight guarantees at most one Insert per key is - // in flight at a time, so an Insert key cannot also - // appear in a Delete in the same batch (a Delete only - // comes from eviction, which would not target a key that - // has an in-flight write because the LBA is admitted to - // the LRU only after publish). Debug-assert the simpler - // condition: each insert key appears at most once. + // Deletes may overlap an Insert because eviction releases the SIEVE + // record before enqueueing. Fold requests in queue order and compare + // each victim's expected LBA against that logical state. #[cfg(debug_assertions)] { use std::collections::HashSet; @@ -789,23 +781,47 @@ impl StorageEngine { } } + let mut states: std::collections::HashMap> = + std::collections::HashMap::new(); let mut priors: Vec> = Vec::with_capacity(batch.len()); + let mut removed_by_req: Vec> = Vec::with_capacity(batch.len()); let mut mutations: Vec = Vec::new(); for req in &batch { match req { MutatorReq::Insert { key, entry, .. } => { - let prior = self.btree.lookup(key).await.ok().flatten(); + let prior = if let Some(prior) = states.get(key) { + *prior + } else { + let prior = self.btree.lookup(key).await.ok().flatten(); + states.insert(*key, prior); + prior + }; priors.push(prior); + removed_by_req.push(Vec::new()); + states.insert(*key, Some(*entry)); mutations.push(Mutation::Insert { key: *key, value: *entry, }); } - MutatorReq::Delete { keys, .. } => { + MutatorReq::Delete { victims, .. } => { priors.push(None); - for k in keys { - mutations.push(Mutation::Delete { key: *k }); + let mut removed = Vec::new(); + for victim in victims { + let current = if let Some(current) = states.get(&victim.key) { + *current + } else { + let current = self.btree.lookup(&victim.key).await.ok().flatten(); + states.insert(victim.key, current); + current + }; + if current.is_some_and(|entry| entry.lba == victim.lba) { + states.insert(victim.key, None); + mutations.push(Mutation::Delete { key: victim.key }); + removed.push(*victim); + } } + removed_by_req.push(removed); } } } @@ -830,7 +846,9 @@ impl StorageEngine { MutatorReq::Insert { .. } => { MutatorOutcome::InsertCommitted { prior: priors[i] } } - MutatorReq::Delete { .. } => MutatorOutcome::DeleteCommitted, + MutatorReq::Delete { .. } => MutatorOutcome::DeleteCommitted { + removed: std::mem::take(&mut removed_by_req[i]), + }, } }; done.set(outcome); @@ -873,7 +891,7 @@ mod tests { use crate::bufferpool::BlockStore; use crate::memory::Backing; use crate::storage::blockdev::{MockDevice, MockDeviceConfig, MockFaultMode}; - use crate::storage::types::Lba; + use crate::storage::types::{Checksum, Lba}; fn test_backing(base: *mut u8, page_size: usize, page_count: usize) -> Backing { Backing { @@ -952,6 +970,88 @@ mod tests { StripeKey(s) } + fn entry(lba: u64) -> LeafEntry { + LeafEntry { + lba: Lba(lba), + data_checksum: Checksum(lba), + byte_len: 4096, + } + } + + fn resident(key: PageKey, lba: u64) -> Resident { + Resident { + key, + lba: Lba(lba), + byte_len: 4096, + } + } + + #[test] + fn stale_eviction_after_same_batch_overwrite_is_ignored() { + let (eng, _buf) = engine(256); + let key = StorageEngine::::page_key(&stripe(21), 0, 4096); + let seed = MutatorReply::new(); + block_on(eng.process_batch(vec![MutatorReq::Insert { + key, + entry: entry(200), + done: seed.clone(), + }])); + assert!(matches!( + block_on(seed.wait()), + MutatorOutcome::InsertCommitted { prior: None } + )); + + let overwrite = MutatorReply::new(); + let eviction = MutatorReply::new(); + block_on(eng.process_batch(vec![ + MutatorReq::Insert { + key, + entry: entry(201), + done: overwrite.clone(), + }, + MutatorReq::Delete { + victims: vec![resident(key, 200)], + done: eviction.clone(), + }, + ])); + + assert_eq!(eng.btree.lookup_committed_mirror(&key), Some(entry(201))); + assert!(matches!( + block_on(eviction.wait()), + MutatorOutcome::DeleteCommitted { removed } if removed.is_empty() + )); + } + + #[test] + fn stale_eviction_after_adjacent_batch_overwrite_is_ignored() { + let (eng, _buf) = engine(256); + let key = StorageEngine::::page_key(&stripe(22), 0, 4096); + let seed = MutatorReply::new(); + block_on(eng.process_batch(vec![MutatorReq::Insert { + key, + entry: entry(200), + done: seed, + }])); + + let overwrite = MutatorReply::new(); + block_on(eng.process_batch(vec![MutatorReq::Insert { + key, + entry: entry(201), + done: overwrite, + }])); + let eviction = MutatorReply::new(); + block_on(eng.process_batch(vec![MutatorReq::Delete { + victims: vec![resident(key, 200)], + done: eviction.clone(), + }])); + + assert_eq!(eng.btree.lookup_committed_mirror(&key), Some(entry(201))); + assert!(matches!( + block_on(eviction.wait()), + MutatorOutcome::DeleteCommitted { removed } if removed.is_empty() + )); + } + #[test] fn write_then_read_roundtrip() { let (eng, buf) = engine(256); diff --git a/cmd/unbounded-storage/src/storage/mutator.rs b/cmd/unbounded-storage/src/storage/mutator.rs index ed22a6ed0..317e35e32 100644 --- a/cmd/unbounded-storage/src/storage/mutator.rs +++ b/cmd/unbounded-storage/src/storage/mutator.rs @@ -25,6 +25,7 @@ use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use crate::storage::btree::LeafEntry; +use crate::storage::lru::Resident; use crate::storage::types::PageKey; /// A single submission to the mutator. The submitter has already @@ -39,7 +40,7 @@ pub(crate) enum MutatorReq { done: Arc, }, Delete { - keys: Vec, + victims: Vec, done: Arc, }, } @@ -52,8 +53,9 @@ pub(crate) enum MutatorOutcome { /// `None` if the key was unmapped. The entry's `byte_len` lets /// the submitter free the entire prior contiguous LBA range. InsertCommitted { prior: Option }, - /// `apply_batch` committed the delete set. - DeleteCommitted, + /// `apply_batch` committed. Only victims whose expected LBA still matched + /// are returned for reclamation. + DeleteCommitted { removed: Vec }, /// `apply_batch` returned an error. The submitter must clean /// up the LBA range it allocated (insert) or leave eviction /// state untouched (delete). @@ -355,9 +357,9 @@ mod tests { #[test] fn reply_resolves_after_set() { let r = MutatorReply::new(); - r.set(MutatorOutcome::DeleteCommitted); + r.set(MutatorOutcome::DeleteCommitted { removed: vec![] }); let out = block_on(r.wait()); - assert!(matches!(out, MutatorOutcome::DeleteCommitted)); + assert!(matches!(out, MutatorOutcome::DeleteCommitted { .. })); } #[test] @@ -366,11 +368,11 @@ mod tests { let r1 = MutatorReply::new(); let r2 = MutatorReply::new(); q.push(MutatorReq::Delete { - keys: vec![], + victims: vec![], done: r1.clone(), }); q.push(MutatorReq::Delete { - keys: vec![], + victims: vec![], done: r2.clone(), }); let drained = q.try_drain_up_to(10); @@ -383,7 +385,7 @@ mod tests { q.close(); let r = MutatorReply::new(); q.push(MutatorReq::Delete { - keys: vec![], + victims: vec![], done: r.clone(), }); let out = block_on(r.wait()); @@ -401,7 +403,7 @@ mod tests { fn push_n(q: &Arc, n: usize) { for _ in 0..n { q.push(MutatorReq::Delete { - keys: vec![], + victims: vec![], done: MutatorReply::new(), }); } @@ -558,7 +560,7 @@ mod tests { } } else { MutatorReq::Delete { - keys: vec![page_key(idx as u32)], + victims: vec![], done: reply.clone(), } }; @@ -595,7 +597,7 @@ mod tests { done.set(MutatorOutcome::InsertCommitted { prior: None }) } MutatorReq::Delete { done, .. } => { - done.set(MutatorOutcome::DeleteCommitted) + done.set(MutatorOutcome::DeleteCommitted { removed: vec![] }) } } } From 8fe1d02439af2dddea8bfb88d99cff6a2edb198b Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 19:50:51 +0000 Subject: [PATCH 54/56] unbounded-storage: retry failed frontend rebuilds --- cmd/unbounded-storage/src/main.rs | 131 ++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 25 deletions(-) diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index b5dbb694a..d15f17742 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -899,6 +899,7 @@ fn run_shard( .map(|f| (f.name.clone(), f.clone())) .collect(); let mut last_bindings: HashMap = frontend_bindings.clone(); + let mut pending_frontend_updates = HashSet::new(); shard_loop.add_tick_hook(move || { let mut did_work = false; while let Ok(cmd) = ctrl_rx.try_recv() { @@ -924,6 +925,7 @@ fn run_shard( &apply.loaded.runtime().frontends, &last_backends, desired_backends, + &pending_frontend_updates, ); let forced_frontend_updates: HashSet = frontends_to_rebuild.into_iter().collect(); @@ -955,19 +957,24 @@ fn run_shard( g.insert(spec.name.clone(), spec.stripe_size_bytes()); } } - let mut applied_bindings = HashMap::new(); - for (id, spec) in &last_frontends { - let desired_spec = desired_frontends.iter().find(|f| f.name == *id); - if desired_spec == Some(spec) { - if let Some(binding) = apply.loaded.runtime().frontends.get(id) { - applied_bindings.insert(id.clone(), binding.clone()); - continue; - } - } - if let Some(binding) = last_bindings.get(id) { - applied_bindings.insert(id.clone(), binding.clone()); - } - } + let unsuccessful_frontends: HashSet = combined + .frontends + .failures + .iter() + .chain(&combined.frontends.deferred) + .map(|(id, _)| id.clone()) + .collect(); + pending_frontend_updates = forced_frontend_updates + .intersection(&unsuccessful_frontends) + .cloned() + .collect(); + let applied_bindings = applied_frontend_bindings( + &last_bindings, + &apply.loaded.runtime().frontends, + desired_frontends, + &last_frontends, + &unsuccessful_frontends, + ); *bindings.borrow_mut() = applied_bindings.clone(); last_bindings = applied_bindings; @@ -1204,6 +1211,7 @@ fn frontend_rebuild_ids( next_bindings: &HashMap, last_backends: &HashMap, desired_backends: &[BackendSpec], + pending_updates: &HashSet, ) -> Vec { let changed_backend_geometry: HashSet = desired_backends .iter() @@ -1221,10 +1229,33 @@ fn frontend_rebuild_ids( .then(|| id.clone()) }) .collect(); + rebuild.extend(pending_updates.iter().cloned()); rebuild.sort(); + rebuild.dedup(); rebuild } +fn applied_frontend_bindings( + last_bindings: &HashMap, + desired_bindings: &HashMap, + desired_frontends: &[FrontendSpec], + applied_frontends: &HashMap, + unsuccessful_frontends: &HashSet, +) -> HashMap { + applied_frontends + .iter() + .filter_map(|(id, spec)| { + let desired_spec = desired_frontends.iter().find(|desired| desired.name == *id); + let binding = if desired_spec == Some(spec) && !unsuccessful_frontends.contains(id) { + desired_bindings.get(id) + } else { + last_bindings.get(id) + }?; + Some((id.clone(), binding.clone())) + }) + .collect() +} + impl FrontendBuildCtx { /// Turn one [`FrontendSpec`] into a bound, ready-to-drive /// [`ShardFrontendDriver`]. Validates the spec, binds the shard's @@ -2035,29 +2066,79 @@ mod tests { } #[test] - fn backend_stripe_size_change_rebuilds_frontend() { - let mut last_bindings = HashMap::new(); - last_bindings.insert("f".to_string(), binding("f", "b")); - - let mut next_bindings = HashMap::new(); - next_bindings.insert("f".to_string(), binding("f", "b")); - + fn failed_stripe_size_rebuild_retries() { + let bindings = HashMap::from([("f".to_string(), binding("f", "b"))]); let old_backend = backend_spec("b"); let mut new_backend = old_backend.clone(); let Some(config::backend_spec::Config::Http(cfg)) = new_backend.config.as_mut() else { panic!("expected http backend config"); }; *cfg.stripe_size_bytes.as_mut().expect("stripe size set") *= 2; + let desired_backends = [new_backend]; + let last_backends = HashMap::from([("b".to_string(), old_backend)]); - let mut last_backends = HashMap::new(); - last_backends.insert(old_backend.name.clone(), old_backend); + assert_eq!( + frontend_rebuild_ids( + &bindings, + &bindings, + &last_backends, + &desired_backends, + &HashSet::new(), + ), + vec!["f".to_string()], + ); + + // The backend update can apply before the forced frontend replacement + // fails, so geometry drift alone disappears from the next diff. + let advanced_backends = HashMap::from([("b".to_string(), desired_backends[0].clone())]); + assert_eq!( + frontend_rebuild_ids( + &bindings, + &bindings, + &advanced_backends, + &desired_backends, + &HashSet::from(["f".to_string()]), + ), + vec!["f".to_string()], + "failed forced replacement must be retried", + ); + } + #[test] + fn failed_binding_rebuild_preserves_old_binding() { + let old_bindings = HashMap::from([("f".to_string(), binding("f", "old"))]); + let new_bindings = HashMap::from([("f".to_string(), binding("f", "new"))]); + let frontend = FrontendSpec { + name: "f".to_string(), + source: "new".to_string(), + ..Default::default() + }; + let desired_frontends = [frontend.clone()]; + let applied_frontends = HashMap::from([("f".to_string(), frontend)]); + + let after_failure = applied_frontend_bindings( + &old_bindings, + &new_bindings, + &desired_frontends, + &applied_frontends, + &HashSet::from(["f".to_string()]), + ); + assert_eq!( + after_failure["f"].backend_id, "old", + "failed replacement must retain its live binding", + ); + let last_backends = HashMap::from([ + ("old".to_string(), backend_spec("old")), + ("new".to_string(), backend_spec("new")), + ]); + let desired_backends = [backend_spec("old"), backend_spec("new")]; assert_eq!( frontend_rebuild_ids( - &last_bindings, - &next_bindings, + &after_failure, + &new_bindings, &last_backends, - &[new_backend] + &desired_backends, + &HashSet::new(), ), vec!["f".to_string()], ); From 256c1392d31c40d7a42d9a1997307fa519ba2e42 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 10 Jul 2026 19:59:07 +0000 Subject: [PATCH 55/56] unbounded-storage: transact frontend activation --- cmd/unbounded-storage/ARCHITECTURE.md | 8 +- cmd/unbounded-storage/src/http/server.rs | 6 +- cmd/unbounded-storage/src/main.rs | 157 +++++++++++------- .../src/registry_transaction.rs | 71 ++++---- cmd/unbounded-storage/src/shard_layer.rs | 32 ++++ 5 files changed, 161 insertions(+), 113 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index 324daa794..36a8de534 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -183,11 +183,9 @@ dormant `SO_REUSEPORT` socket, then enter the listening state only during activation. A failed preparation therefore leaves the prior resource, binding, and stripe geometry live and retryable. Active frontend entries record the binding and stripe geometry they realized, so a failed rebuild is detected and -retried on the next apply. Replacement uses an undo-log transaction: the old -driver remains owned until finalization and can be restored if a later local -activation fails. The current reconciler finalizes each frontend operation -immediately; cross-resource and cross-shard commit/rollback is not yet wired to -that transaction boundary. +retried on the next apply. Replacement uses a staged transaction: the complete +frontend change set is prepared before activation and swaps into the live +registry only after every listener activates successfully. ### Shutdown diff --git a/cmd/unbounded-storage/src/http/server.rs b/cmd/unbounded-storage/src/http/server.rs index d6db8dcc8..a1a3d0348 100644 --- a/cmd/unbounded-storage/src/http/server.rs +++ b/cmd/unbounded-storage/src/http/server.rs @@ -239,11 +239,13 @@ mod tests { #[cfg(target_os = "linux")] #[test] - fn bound_socket_is_dormant_until_listen() { + fn prepared_startup_socket_is_dormant_until_activation() { let bound = match bind_socket("127.0.0.1:0".parse().unwrap()) { Ok(bound) => bound, Err(e) => { - eprintln!("bound_socket_is_dormant_until_listen: bind failed: {e}; skipping"); + eprintln!( + "prepared_startup_socket_is_dormant_until_activation: bind failed: {e}; skipping" + ); return; } }; diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 76b3af338..457be591d 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -575,6 +575,7 @@ fn run_shard( peer_rx: mpsc::Receiver>>, phaseb_tx: mpsc::Sender, serve_start_rx: mpsc::Receiver<()>, + activation_tx: mpsc::Sender, terminal_tx: mpsc::Sender, layer_stop: Arc, ) { @@ -888,7 +889,6 @@ fn run_shard( // build-from-spec work (DNS resolve, listener bind) stays off the // fast path. { - let routes = routes.clone(); let transport_registry = transport_registry.clone(); let frontend_registry = frontend_registry.clone(); let pool = pool.clone(); @@ -937,7 +937,9 @@ fn run_shard( // Drive the transport backend registry and the // frontend registry together so frontend adds are // gated on their referenced backend being present. - let combined = config::reconcile::reconcile_backends_and_frontends( + frontend_registry.begin_transaction(); + let previous_frontends = last_frontends.clone(); + let mut combined = config::reconcile::reconcile_backends_and_frontends( &transport_registry, &frontend_registry, desired_backends, @@ -947,8 +949,23 @@ fn run_shard( Some(&last_backends), Some(&last_frontends), ); + if combined.frontends.failures.is_empty() { + if let Err(error) = frontend_registry.commit_transaction() { + combined + .frontends + .failures + .push(("transaction".to_string(), format!("commit: {error}"))); + } + } else { + frontend_registry.rollback_transaction(); + } last_backends = combined.backends.applied; - last_frontends = combined.frontends.applied; + if combined.frontends.failures.is_empty() { + last_frontends = combined.frontends.applied; + } else { + frontend_registry.rollback_transaction(); + last_frontends = previous_frontends; + } // Replacement construction happens against desired // geometry/bindings, then failed replacements retain @@ -1014,6 +1031,14 @@ fn run_shard( Ok(()) => {} Err(_) => return, } + if let Err(error) = frontend_registry.commit_transaction() { + let _ = activation_tx.send(ShardActivationReport::Failed(format!( + "worker={}: {error}", + widx.0 + ))); + return; + } + let _ = activation_tx.send(ShardActivationReport::Ready); let serving_guard = ServingGuard::new(widx, terminal_tx, &SHUTDOWN); // Drive the shard's cooperative future set until shutdown. The loop @@ -1405,21 +1430,16 @@ impl FrontendBuildCtx { }; Ok(ActiveShardFrontend { driver, activation }) } - - fn build(&self, spec: &FrontendSpec) -> Result { - let prepared = self.prepare(spec)?; - self.activate(&spec.name, prepared) - } } struct FrontendRegistryTransaction { ctx: FrontendBuildCtx, - entries: RegistryTransaction, + entries: RegistryTransaction, } impl FrontendRegistryTransaction { fn add(&mut self, spec: &FrontendSpec) -> Result<(), String> { - let frontend = self.ctx.build(spec)?; + let frontend = self.ctx.prepare(spec)?; self.entries.replace(spec.name.clone(), frontend); Ok(()) } @@ -1428,13 +1448,9 @@ impl FrontendRegistryTransaction { self.entries.remove(id); } - fn finalize(self) { - self.entries.finalize(); - } - - #[cfg(test)] - fn rollback(self) { - self.entries.rollback(); + fn commit(self) -> Result<(), String> { + self.entries + .commit(|id, prepared| self.ctx.activate(id, prepared)) } } @@ -1449,6 +1465,7 @@ impl FrontendRegistryTransaction { struct FrontendRegistry { frontends: Rc>>, ctx: FrontendBuildCtx, + transaction: Rc>>, } impl FrontendRegistry { @@ -1460,13 +1477,14 @@ impl FrontendRegistry { let registry = Self { frontends: Rc::new(RefCell::new(HashMap::with_capacity(specs.len()))), ctx, + transaction: Rc::new(RefCell::new(None)), }; + registry.begin_transaction(); for spec in specs { - let frontend = registry.ctx.build(spec)?; - registry - .frontends - .borrow_mut() - .insert(spec.name.clone(), frontend); + if let Err(error) = registry.add_to_transaction(spec) { + registry.rollback_transaction(); + return Err(error); + } } Ok(registry) } @@ -1483,11 +1501,32 @@ impl FrontendRegistry { busy } - fn transaction(&self) -> FrontendRegistryTransaction { - FrontendRegistryTransaction { + fn begin_transaction(&self) { + let transaction = FrontendRegistryTransaction { ctx: self.ctx.clone(), entries: RegistryTransaction::new(self.frontends.clone()), - } + }; + assert!(self.transaction.borrow_mut().replace(transaction).is_none()); + } + + fn add_to_transaction(&self, spec: &FrontendSpec) -> Result<(), String> { + self.transaction + .borrow_mut() + .as_mut() + .expect("frontend transaction started") + .add(spec) + } + + fn commit_transaction(&self) -> Result<(), String> { + self.transaction + .borrow_mut() + .take() + .expect("frontend transaction started") + .commit() + } + + fn rollback_transaction(&self) { + self.transaction.borrow_mut().take(); } fn realized_activations(&self) -> HashMap { @@ -1505,16 +1544,15 @@ impl config::reconcile::FrontendReconcileTarget for FrontendRegistry { } fn add(&self, spec: &FrontendSpec) -> Result<(), String> { - let mut transaction = self.transaction(); - transaction.add(spec)?; - transaction.finalize(); - Ok(()) + self.add_to_transaction(spec) } fn remove(&self, id: &str) -> Result<(), String> { - let mut transaction = self.transaction(); - transaction.remove(id); - transaction.finalize(); + self.transaction + .borrow_mut() + .as_mut() + .expect("frontend transaction started") + .remove(id); Ok(()) } } @@ -1627,6 +1665,11 @@ enum PhaseBReport { Failed(String), } +enum ShardActivationReport { + Ready, + Failed(String), +} + /// RAII guard ensuring a shard that has entered phase B reports exactly /// once. `report_ready`/`report_failed` send the terminal report and /// disarm; if the shard returns early or panics before reporting, `Drop` @@ -2256,7 +2299,7 @@ mod tests { } #[test] - fn registry_transaction_finalize_commits_replacement() { + fn registry_transaction_keeps_replacement_dormant_until_commit() { let drops = Rc::new(RefCell::new(Vec::new())); let live = Rc::new(RefCell::new(HashMap::from([( "a".to_string(), @@ -2265,16 +2308,16 @@ mod tests { let mut transaction = RegistryTransaction::new(live.clone()); transaction.replace("a".to_string(), drop_value("new", &drops)); - assert_eq!(live.borrow()["a"].id, "new"); + assert_eq!(live.borrow()["a"].id, "old"); assert!(drops.borrow().is_empty()); - transaction.finalize(); + transaction.commit(|_, value| Ok(value)).unwrap(); assert_eq!(&*drops.borrow(), &["old"]); assert_eq!(live.borrow()["a"].id, "new"); } #[test] - fn registry_transaction_rollback_restores_originals() { + fn registry_transaction_activation_failure_has_no_partial_exposure() { let drops = Rc::new(RefCell::new(Vec::new())); let live = Rc::new(RefCell::new(HashMap::from([ ("a".to_string(), drop_value("old-a", &drops)), @@ -2283,8 +2326,15 @@ mod tests { let mut transaction = RegistryTransaction::new(live.clone()); transaction.replace("a".to_string(), drop_value("new-a", &drops)); transaction.replace("b".to_string(), drop_value("new-b", &drops)); - transaction.rollback(); + let result = transaction.commit(|id, value| { + if id == "b" { + Err("listen failed".to_string()) + } else { + Ok(value) + } + }); + assert!(result.is_err()); assert_eq!(live.borrow()["a"].id, "old-a"); assert_eq!(live.borrow()["b"].id, "old-b"); let mut dropped = drops.borrow().clone(); @@ -2293,38 +2343,19 @@ mod tests { } #[test] - fn registry_transaction_drop_rolls_back_removal() { + fn registry_transaction_stages_removal_until_commit() { let drops = Rc::new(RefCell::new(Vec::new())); let live = Rc::new(RefCell::new(HashMap::from([( "a".to_string(), drop_value("old", &drops), )]))); - { - let mut transaction = RegistryTransaction::new(live.clone()); - transaction.remove("a"); - assert!(!live.borrow().contains_key("a")); - } - + let mut transaction: RegistryTransaction = + RegistryTransaction::new(live.clone()); + transaction.remove("a"); assert_eq!(live.borrow()["a"].id, "old"); - assert!(drops.borrow().is_empty()); - } - - #[test] - fn registry_transaction_preserves_first_original() { - let drops = Rc::new(RefCell::new(Vec::new())); - let live = Rc::new(RefCell::new(HashMap::from([( - "a".to_string(), - drop_value("old", &drops), - )]))); - let mut transaction = RegistryTransaction::new(live.clone()); - transaction.replace("a".to_string(), drop_value("new-1", &drops)); - transaction.replace("a".to_string(), drop_value("new-2", &drops)); - transaction.rollback(); - - assert_eq!(live.borrow()["a"].id, "old"); - let mut dropped = drops.borrow().clone(); - dropped.sort_unstable(); - assert_eq!(dropped, ["new-1", "new-2"]); + transaction.commit(|_, value| Ok(value)).unwrap(); + assert!(!live.borrow().contains_key("a")); + assert_eq!(&*drops.borrow(), &["old"]); } #[test] diff --git a/cmd/unbounded-storage/src/registry_transaction.rs b/cmd/unbounded-storage/src/registry_transaction.rs index 6535f3f0f..0c1eb27ef 100644 --- a/cmd/unbounded-storage/src/registry_transaction.rs +++ b/cmd/unbounded-storage/src/registry_transaction.rs @@ -2,68 +2,53 @@ // Licensed under the MIT License. use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; -pub(crate) struct RegistryTransaction { +pub(crate) struct RegistryTransaction { live: Rc>>, - originals: HashMap>, - completed: bool, + replacements: HashMap, + removals: HashSet, } -impl RegistryTransaction { +impl RegistryTransaction { pub(crate) fn new(live: Rc>>) -> Self { Self { live, - originals: HashMap::new(), - completed: false, + replacements: HashMap::new(), + removals: HashSet::new(), } } - pub(crate) fn replace(&mut self, id: String, value: T) { - self.record_original(&id); - self.live.borrow_mut().insert(id, value); + pub(crate) fn replace(&mut self, id: String, value: P) { + self.removals.remove(&id); + self.replacements.insert(id, value); } pub(crate) fn remove(&mut self, id: &str) { - self.record_original(id); - self.live.borrow_mut().remove(id); - } - - pub(crate) fn finalize(mut self) { - self.completed = true; - self.originals.clear(); - } - - #[cfg(test)] - pub(crate) fn rollback(mut self) { - self.rollback_inner(); - self.completed = true; - } - - fn record_original(&mut self, id: &str) { - if self.originals.contains_key(id) { - return; + self.replacements.remove(id); + self.removals.insert(id.to_string()); + } + + pub(crate) fn commit( + self, + mut activate: impl FnMut(&str, P) -> Result, + ) -> Result<(), String> { + let mut replacements = HashMap::with_capacity(self.replacements.len()); + let mut prepared: Vec<_> = self.replacements.into_iter().collect(); + prepared.sort_by(|(a, _), (b, _)| a.cmp(b)); + for (id, value) in prepared { + let active = activate(&id, value)?; + replacements.insert(id, active); } - let original = self.live.borrow_mut().remove(id); - self.originals.insert(id.to_string(), original); - } - fn rollback_inner(&mut self) { let mut live = self.live.borrow_mut(); - for (id, original) in self.originals.drain() { + for id in self.removals { live.remove(&id); - if let Some(original) = original { - live.insert(id, original); - } } - } -} - -impl Drop for RegistryTransaction { - fn drop(&mut self) { - if !self.completed { - self.rollback_inner(); + for (id, active) in replacements { + live.insert(id, active); } + Ok(()) } } diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 98d1e32d6..7db0fba17 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -78,6 +78,7 @@ pub struct PreparedShardLayer { backing_keepalives: Vec>, rpc_shards: Vec, serve_start_txs: Vec>, + activation_rx: mpsc::Receiver, terminal_rx: mpsc::Receiver, routes: RouteTableHandle, } @@ -172,6 +173,7 @@ pub fn prepare_shard_layer( // (peer registration) after broadcasting the full peer set. let (phaseb_tx, phaseb_rx) = mpsc::channel::(); let (terminal_tx, terminal_rx) = mpsc::channel::(); + let (activation_tx, activation_rx) = mpsc::channel::(); let mut joins = Vec::with_capacity(worker_count); let mut control_senders = Vec::with_capacity(worker_count); // Per-shard senders for broadcasting the assembled peer set in phase @@ -201,6 +203,7 @@ pub fn prepare_shard_layer( let loaded = loaded.clone(); let layer_stop = layer_stop.clone(); let terminal_tx = terminal_tx.clone(); + let activation_tx = activation_tx.clone(); let rt = deps.runtime.clone(); let handle = rt.spawn_pinned( widx, @@ -221,6 +224,7 @@ pub fn prepare_shard_layer( peer_rx, phaseb_tx, serve_start_rx, + activation_tx, terminal_tx, layer_stop, ); @@ -235,6 +239,7 @@ pub fn prepare_shard_layer( // holds its sender, so this never closes the channel prematurely. drop(phaseb_tx); drop(terminal_tx); + drop(activation_tx); // Bounded readiness collection: read exactly one message per spawned // thread. Shards that come up park holding their sender, so they @@ -367,6 +372,7 @@ pub fn prepare_shard_layer( rpc_shards, serve_start_txs, terminal_rx, + activation_rx, routes, }) } @@ -384,6 +390,7 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result Result {} + Ok(crate::ShardActivationReport::Failed(error)) => activation_errors.push(error), + Err(_) => { + activation_errors.push("shard exited without reporting activation".to_string()); + break; + } + } + } + if !activation_errors.is_empty() { + retire_failed_activation( + joins, + fabric_group, + control, + layer_stop, + backing_keepalives, + serve_start_txs, + terminal_rx, + routes, + ); + return Err(activation_errors); + } drop(serve_start_txs); Ok(ShardLayer { @@ -440,6 +471,7 @@ pub fn retire_prepared_shard_layer(prepared: PreparedShardLayer) { backing_keepalives, rpc_shards: _, serve_start_txs, + activation_rx: _, terminal_rx, routes, } = prepared; From e33a6265277f8f8442264e93f9b6cde6d0abc336 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:00:58 +0000 Subject: [PATCH 56/56] Resolve merge conflicts with main in unbounded-storage frontend lifecycle --- cmd/unbounded-storage/ARCHITECTURE.md | 11 +- cmd/unbounded-storage/src/main.rs | 155 ----------------------- cmd/unbounded-storage/src/shard_layer.rs | 27 ---- 3 files changed, 3 insertions(+), 190 deletions(-) diff --git a/cmd/unbounded-storage/ARCHITECTURE.md b/cmd/unbounded-storage/ARCHITECTURE.md index e334f2bf0..2aa766e4a 100644 --- a/cmd/unbounded-storage/ARCHITECTURE.md +++ b/cmd/unbounded-storage/ARCHITECTURE.md @@ -178,7 +178,6 @@ shutdown; teardown drains terminal reports and returns a failing process status instead of continuing with a missing shard. Backend and frontend reloads construct replacements before swapping them into -<<<<<<< HEAD their shard-local registries. Network frontends first validate and bind a dormant `SO_REUSEPORT` socket, then enter the listening state only during activation. A failed preparation therefore leaves the prior resource, binding, @@ -186,13 +185,9 @@ and stripe geometry live and retryable. Active frontend entries record the binding and stripe geometry they realized, so a failed rebuild is detected and retried on the next apply. Replacement uses a staged transaction: the complete frontend change set is prepared before activation and swaps into the live -registry only after every listener activates successfully. -======= -their shard-local registries. A failed replacement therefore leaves the prior -resource, binding, and stripe geometry live and retryable. This is a -per-resource guarantee; route, peer, shard, and disk publication across the -whole process is not yet one transaction. ->>>>>>> origin/main +registry only after every listener activates successfully. This is not yet +a whole-process guarantee; route, peer, shard, and disk publication across +the whole process is not yet one transaction. ### Shutdown diff --git a/cmd/unbounded-storage/src/main.rs b/cmd/unbounded-storage/src/main.rs index 2c7e771cf..87d705a79 100644 --- a/cmd/unbounded-storage/src/main.rs +++ b/cmd/unbounded-storage/src/main.rs @@ -27,10 +27,7 @@ use unbounded_storage::fanout::{ use unbounded_storage::frontend::{ HttpDriver, HttpFrontend, LoadgenDriver, LoadgenFrontend, S3Driver, S3Frontend, }; -<<<<<<< HEAD use unbounded_storage::http::BoundListener; -======= ->>>>>>> origin/main use unbounded_storage::p2p::{RouteTableHandle, RoutedTransport}; use unbounded_storage::ring::{NetHandle, NetworkRing}; use unbounded_storage::runtime::{PinnedRuntime, ShardLoop, WorkerIdx, WorkerSpec}; @@ -578,10 +575,7 @@ fn run_shard( peer_rx: mpsc::Receiver>>, phaseb_tx: mpsc::Sender, serve_start_rx: mpsc::Receiver<()>, -<<<<<<< HEAD activation_tx: mpsc::Sender, -======= ->>>>>>> origin/main terminal_tx: mpsc::Sender, layer_stop: Arc, ) { @@ -908,11 +902,6 @@ fn run_shard( .iter() .map(|f| (f.name.clone(), f.clone())) .collect(); -<<<<<<< HEAD -======= - let mut last_bindings: HashMap = frontend_bindings.clone(); - let mut pending_frontend_updates = HashSet::new(); ->>>>>>> origin/main shard_loop.add_tick_hook(move || { let mut did_work = false; while let Ok(cmd) = ctrl_rx.try_recv() { @@ -933,25 +922,14 @@ fn run_shard( } } -<<<<<<< HEAD let desired_activations = frontend_activations( &apply.loaded.runtime().frontends, -======= - let frontends_to_rebuild = frontend_rebuild_ids( - &last_bindings, - &apply.loaded.runtime().frontends, - &last_backends, ->>>>>>> origin/main desired_backends, - &pending_frontend_updates, ); -<<<<<<< HEAD let frontends_to_rebuild = frontend_rebuild_ids( &frontend_registry.realized_activations(), &desired_activations, ); -======= ->>>>>>> origin/main let forced_frontend_updates: HashSet = frontends_to_rebuild.into_iter().collect(); *bindings.borrow_mut() = apply.loaded.runtime().frontends.clone(); @@ -982,16 +960,12 @@ fn run_shard( frontend_registry.rollback_transaction(); } last_backends = combined.backends.applied; -<<<<<<< HEAD if combined.frontends.failures.is_empty() { last_frontends = combined.frontends.applied; } else { frontend_registry.rollback_transaction(); last_frontends = previous_frontends; } -======= - last_frontends = combined.frontends.applied; ->>>>>>> origin/main // Replacement construction happens against desired // geometry/bindings, then failed replacements retain @@ -1003,34 +977,11 @@ fn run_shard( g.insert(spec.name.clone(), spec.stripe_size_bytes()); } } -<<<<<<< HEAD *bindings.borrow_mut() = frontend_registry .realized_activations() .into_iter() .map(|(id, activation)| (id, activation.binding)) .collect(); -======= - let unsuccessful_frontends: HashSet = combined - .frontends - .failures - .iter() - .chain(&combined.frontends.deferred) - .map(|(id, _)| id.clone()) - .collect(); - pending_frontend_updates = forced_frontend_updates - .intersection(&unsuccessful_frontends) - .cloned() - .collect(); - let applied_bindings = applied_frontend_bindings( - &last_bindings, - &apply.loaded.runtime().frontends, - desired_frontends, - &last_frontends, - &unsuccessful_frontends, - ); - *bindings.borrow_mut() = applied_bindings.clone(); - last_bindings = applied_bindings; ->>>>>>> origin/main let mut failures = combined.backends.failures; failures.extend(combined.frontends.failures); @@ -1080,7 +1031,6 @@ fn run_shard( Ok(()) => {} Err(_) => return, } -<<<<<<< HEAD if let Err(error) = frontend_registry.commit_transaction() { let _ = activation_tx.send(ShardActivationReport::Failed(format!( "worker={}: {error}", @@ -1089,8 +1039,6 @@ fn run_shard( return; } let _ = activation_tx.send(ShardActivationReport::Ready); -======= ->>>>>>> origin/main let serving_guard = ServingGuard::new(widx, terminal_tx, &SHUTDOWN); // Drive the shard's cooperative future set until shutdown. The loop @@ -1307,14 +1255,8 @@ fn frontend_stripe_size(geometry: &HashMap, backend_id: &str) -> u6 fn frontend_activations( bindings: &HashMap, desired_backends: &[BackendSpec], -<<<<<<< HEAD ) -> HashMap { let geometry: HashMap<&str, u64> = desired_backends -======= - pending_updates: &HashSet, -) -> Vec { - let changed_backend_geometry: HashSet = desired_backends ->>>>>>> origin/main .iter() .map(|backend| (backend.name.as_str(), backend.stripe_size_bytes())) .collect(); @@ -1350,33 +1292,10 @@ fn frontend_rebuild_ids( .then(|| id.clone()) }) .collect(); - rebuild.extend(pending_updates.iter().cloned()); rebuild.sort(); - rebuild.dedup(); rebuild } -fn applied_frontend_bindings( - last_bindings: &HashMap, - desired_bindings: &HashMap, - desired_frontends: &[FrontendSpec], - applied_frontends: &HashMap, - unsuccessful_frontends: &HashSet, -) -> HashMap { - applied_frontends - .iter() - .filter_map(|(id, spec)| { - let desired_spec = desired_frontends.iter().find(|desired| desired.name == *id); - let binding = if desired_spec == Some(spec) && !unsuccessful_frontends.contains(id) { - desired_bindings.get(id) - } else { - last_bindings.get(id) - }?; - Some((id.clone(), binding.clone())) - }) - .collect() -} - impl FrontendBuildCtx { /// Validate and bind one frontend without making it traffic-visible. fn prepare(&self, spec: &FrontendSpec) -> Result { @@ -2314,19 +2233,13 @@ mod tests { } #[test] -<<<<<<< HEAD fn backend_stripe_size_change_rebuilds_frontend() { -======= - fn failed_stripe_size_rebuild_retries() { - let bindings = HashMap::from([("f".to_string(), binding("f", "b"))]); ->>>>>>> origin/main let old_backend = backend_spec("b"); let mut new_backend = old_backend; let Some(config::backend_spec::Config::Http(cfg)) = new_backend.config.as_mut() else { panic!("expected http backend config"); }; *cfg.stripe_size_bytes.as_mut().expect("stripe size set") *= 2; -<<<<<<< HEAD let bindings = HashMap::from([("f".to_string(), binding("f", "b"))]); let realized = HashMap::from([( @@ -2340,74 +2253,6 @@ mod tests { assert_eq!( frontend_rebuild_ids(&realized, &desired), -======= - let desired_backends = [new_backend]; - let last_backends = HashMap::from([("b".to_string(), old_backend)]); - - assert_eq!( - frontend_rebuild_ids( - &bindings, - &bindings, - &last_backends, - &desired_backends, - &HashSet::new(), - ), - vec!["f".to_string()], - ); - - // The backend update can apply before the forced frontend replacement - // fails, so geometry drift alone disappears from the next diff. - let advanced_backends = HashMap::from([("b".to_string(), desired_backends[0].clone())]); - assert_eq!( - frontend_rebuild_ids( - &bindings, - &bindings, - &advanced_backends, - &desired_backends, - &HashSet::from(["f".to_string()]), - ), - vec!["f".to_string()], - "failed forced replacement must be retried", - ); - } - - #[test] - fn failed_binding_rebuild_preserves_old_binding() { - let old_bindings = HashMap::from([("f".to_string(), binding("f", "old"))]); - let new_bindings = HashMap::from([("f".to_string(), binding("f", "new"))]); - let frontend = FrontendSpec { - name: "f".to_string(), - source: "new".to_string(), - ..Default::default() - }; - let desired_frontends = [frontend.clone()]; - let applied_frontends = HashMap::from([("f".to_string(), frontend)]); - - let after_failure = applied_frontend_bindings( - &old_bindings, - &new_bindings, - &desired_frontends, - &applied_frontends, - &HashSet::from(["f".to_string()]), - ); - assert_eq!( - after_failure["f"].backend_id, "old", - "failed replacement must retain its live binding", - ); - let last_backends = HashMap::from([ - ("old".to_string(), backend_spec("old")), - ("new".to_string(), backend_spec("new")), - ]); - let desired_backends = [backend_spec("old"), backend_spec("new")]; - assert_eq!( - frontend_rebuild_ids( - &after_failure, - &new_bindings, - &last_backends, - &desired_backends, - &HashSet::new(), - ), ->>>>>>> origin/main vec!["f".to_string()], ); } diff --git a/cmd/unbounded-storage/src/shard_layer.rs b/cmd/unbounded-storage/src/shard_layer.rs index 3e3545b7a..7db0fba17 100644 --- a/cmd/unbounded-storage/src/shard_layer.rs +++ b/cmd/unbounded-storage/src/shard_layer.rs @@ -78,10 +78,7 @@ pub struct PreparedShardLayer { backing_keepalives: Vec>, rpc_shards: Vec, serve_start_txs: Vec>, -<<<<<<< HEAD activation_rx: mpsc::Receiver, -======= ->>>>>>> origin/main terminal_rx: mpsc::Receiver, routes: RouteTableHandle, } @@ -176,10 +173,7 @@ pub fn prepare_shard_layer( // (peer registration) after broadcasting the full peer set. let (phaseb_tx, phaseb_rx) = mpsc::channel::(); let (terminal_tx, terminal_rx) = mpsc::channel::(); -<<<<<<< HEAD let (activation_tx, activation_rx) = mpsc::channel::(); -======= ->>>>>>> origin/main let mut joins = Vec::with_capacity(worker_count); let mut control_senders = Vec::with_capacity(worker_count); // Per-shard senders for broadcasting the assembled peer set in phase @@ -209,10 +203,7 @@ pub fn prepare_shard_layer( let loaded = loaded.clone(); let layer_stop = layer_stop.clone(); let terminal_tx = terminal_tx.clone(); -<<<<<<< HEAD let activation_tx = activation_tx.clone(); -======= ->>>>>>> origin/main let rt = deps.runtime.clone(); let handle = rt.spawn_pinned( widx, @@ -233,10 +224,7 @@ pub fn prepare_shard_layer( peer_rx, phaseb_tx, serve_start_rx, -<<<<<<< HEAD activation_tx, -======= ->>>>>>> origin/main terminal_tx, layer_stop, ); @@ -251,10 +239,7 @@ pub fn prepare_shard_layer( // holds its sender, so this never closes the channel prematurely. drop(phaseb_tx); drop(terminal_tx); -<<<<<<< HEAD drop(activation_tx); -======= ->>>>>>> origin/main // Bounded readiness collection: read exactly one message per spawned // thread. Shards that come up park holding their sender, so they @@ -387,10 +372,7 @@ pub fn prepare_shard_layer( rpc_shards, serve_start_txs, terminal_rx, -<<<<<<< HEAD activation_rx, -======= ->>>>>>> origin/main routes, }) } @@ -408,10 +390,7 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result>>>>>> origin/main routes, } = prepared; @@ -445,7 +424,6 @@ pub fn activate_shard_layer(prepared: PreparedShardLayer) -> Result Result>>>>>> origin/main drop(serve_start_txs); Ok(ShardLayer { @@ -495,10 +471,7 @@ pub fn retire_prepared_shard_layer(prepared: PreparedShardLayer) { backing_keepalives, rpc_shards: _, serve_start_txs, -<<<<<<< HEAD activation_rx: _, -======= ->>>>>>> origin/main terminal_rx, routes, } = prepared;