perf(core): ref-based FEC encode — packetize shards reference the frame in place
Stage A of the zero-copy host packetize path (networking-audit deferred plan §1): ErasureCoder::encode now takes &[&[u8]], so Packetizer::packetize builds each block's data shards as slices straight into the frame buffer instead of allocating + copying a Vec per data shard. Only the frame's final (possibly partial) shard is staged in a reusable zero-padded scratch; blocks are consecutive shard ranges, so every other shard is a full payload-sized slice. - gf8: encode_sep() over the same Cauchy codec — parity byte-identical to nanors/Moonlight (nanors_exact_parity_vectors unchanged and green) - gf16: reed_solomon_simd::encode is already generic over AsRef<[u8]> - loss-harness sweep: recovery rates identical before/after - bench pipeline (end-to-end, host+client): gf8/64K -3.0%, gf16/64K -2.2%, gf16/1M -3.4%, gf8/1M -0.7% Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,7 +12,7 @@ impl ErasureCoder for Gf16Coder {
|
||||
FecScheme::Gf16
|
||||
}
|
||||
|
||||
fn encode(&self, data: &[Vec<u8>], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError> {
|
||||
fn encode(&self, data: &[&[u8]], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError> {
|
||||
if recovery_count == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ impl ErasureCoder for Gf8Coder {
|
||||
FecScheme::Gf8
|
||||
}
|
||||
|
||||
fn encode(&self, data: &[Vec<u8>], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError> {
|
||||
fn encode(&self, data: &[&[u8]], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError> {
|
||||
if recovery_count == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -24,13 +24,12 @@ impl ErasureCoder for Gf8Coder {
|
||||
let shard_len = data[0].len();
|
||||
let rs = ReedSolomon::new(k, recovery_count)
|
||||
.map_err(|_| FecError::Config("invalid GF(2^8) shard counts"))?;
|
||||
// fec-rs fills parity in place: shards = data || zeroed parity.
|
||||
let mut shards: Vec<Vec<u8>> = Vec::with_capacity(k + recovery_count);
|
||||
shards.extend_from_slice(data);
|
||||
shards.resize_with(k + recovery_count, || vec![0u8; shard_len]);
|
||||
rs.encode(&mut shards)
|
||||
// `encode_sep` reads the data shards by reference and fills the parity in place —
|
||||
// same Cauchy codec as `encode`, without copying the data into a shards scratch.
|
||||
let mut parity: Vec<Vec<u8>> = (0..recovery_count).map(|_| vec![0u8; shard_len]).collect();
|
||||
rs.encode_sep(data, &mut parity)
|
||||
.map_err(|_| FecError::Backend("gf8 encode"))?;
|
||||
Ok(shards.split_off(k))
|
||||
Ok(parity)
|
||||
}
|
||||
|
||||
fn reconstruct(
|
||||
@@ -84,7 +83,7 @@ mod tests {
|
||||
fn nanors_exact_parity_vectors() {
|
||||
let coder = Gf8Coder;
|
||||
// The definitive nanors vector (k=4, m=2): single-byte shards [10,20,30,40] → [136, 0].
|
||||
let data = vec![vec![10u8], vec![20], vec![30], vec![40]];
|
||||
let data: [&[u8]; 4] = [&[10u8], &[20], &[30], &[40]];
|
||||
let parity = coder.encode(&data, 2).unwrap();
|
||||
assert_eq!(parity, vec![vec![136u8], vec![0u8]]);
|
||||
|
||||
@@ -106,7 +105,8 @@ mod tests {
|
||||
fn recovers_erased_data_shards() {
|
||||
let coder = Gf8Coder;
|
||||
let data: Vec<Vec<u8>> = (0..6).map(|i| vec![i as u8; 8]).collect();
|
||||
let parity = coder.encode(&data, 3).unwrap();
|
||||
let refs: Vec<&[u8]> = data.iter().map(|s| s.as_slice()).collect();
|
||||
let parity = coder.encode(&refs, 3).unwrap();
|
||||
let mut received: Vec<Option<Vec<u8>>> = data
|
||||
.iter()
|
||||
.cloned()
|
||||
|
||||
@@ -30,7 +30,9 @@ pub trait ErasureCoder: Send + Sync {
|
||||
|
||||
/// Encode `data` (K original shards) into `recovery_count` (M) parity shards.
|
||||
/// Returns the M recovery shards. `recovery_count == 0` returns an empty `Vec`.
|
||||
fn encode(&self, data: &[Vec<u8>], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError>;
|
||||
/// Takes shard *references* so the packetizer can point straight into the frame
|
||||
/// buffer instead of copying every data byte into per-shard `Vec`s first.
|
||||
fn encode(&self, data: &[&[u8]], recovery_count: usize) -> Result<Vec<Vec<u8>>, FecError>;
|
||||
|
||||
/// Reconstruct the K original shards. `received` has length K+M: indices `0..K` are
|
||||
/// originals, `K..K+M` are recovery shards; `Some` = present, `None` = lost.
|
||||
@@ -79,7 +81,7 @@ pub(crate) fn validate_block_shape(
|
||||
}
|
||||
|
||||
/// Validate `encode` inputs: at least one data shard, all of equal length.
|
||||
pub(crate) fn validate_encode_shape(data: &[Vec<u8>]) -> Result<(), FecError> {
|
||||
pub(crate) fn validate_encode_shape(data: &[&[u8]]) -> Result<(), FecError> {
|
||||
let first = data
|
||||
.first()
|
||||
.ok_or(FecError::Config("no data shards"))?
|
||||
@@ -100,7 +102,8 @@ mod tests {
|
||||
let data: Vec<Vec<u8>> = (0..k)
|
||||
.map(|i| (0..shard_len).map(|b| (i * 31 + b * 7) as u8).collect())
|
||||
.collect();
|
||||
let recovery = coder.encode(&data, m).unwrap();
|
||||
let refs: Vec<&[u8]> = data.iter().map(|s| s.as_slice()).collect();
|
||||
let recovery = coder.encode(&refs, m).unwrap();
|
||||
assert_eq!(recovery.len(), m);
|
||||
|
||||
let mut received: Vec<Option<Vec<u8>>> = Vec::with_capacity(k + m);
|
||||
@@ -128,7 +131,8 @@ mod tests {
|
||||
#[test]
|
||||
fn gf8_too_much_loss_errors() {
|
||||
let data: Vec<Vec<u8>> = (0..8).map(|_| vec![0u8; 64]).collect();
|
||||
let recovery = Gf8Coder.encode(&data, 2).unwrap();
|
||||
let refs: Vec<&[u8]> = data.iter().map(|s| s.as_slice()).collect();
|
||||
let recovery = Gf8Coder.encode(&refs, 2).unwrap();
|
||||
let mut received: Vec<Option<Vec<u8>>> = data
|
||||
.iter()
|
||||
.cloned()
|
||||
|
||||
Reference in New Issue
Block a user