From e14e449640474388dab69e26c649c85e92f6b720 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 8 Nov 2024 00:06:55 -0500 Subject: [PATCH 01/56] Add CompactEncodable trait --- src/types.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/types.rs b/src/types.rs index 68714ec..9cecbb2 100644 --- a/src/types.rs +++ b/src/types.rs @@ -744,6 +744,12 @@ impl State { } } + /// Encode a u16 + /// TODO should it be encode_uINT16? + pub fn encode_u16(&mut self, uint: u16, buffer: &mut [u8]) -> Result { + self.set_slice_to_buffer(&uint.to_le_bytes(), buffer) + } + /// Encode a 2 byte unsigned integer. NB: assumes `bytes` buffer large enough, hence not public! fn encode_uint16_bytes( &mut self, @@ -772,6 +778,27 @@ impl State { self.encode_uint32_bytes(bytes, buffer)?; self.set_slice_to_buffer(&bytes[4..8], buffer) } + + fn preencode_t(&mut self, value: &T) -> Result { + let size = value.encoded_size().map_err(|e| e.into())?; + let out = self.add_end(size)?; + Ok(out) + } + + fn encode_t( + &mut self, + value: &T, + buffer: &mut [u8], + ) -> Result { + let bytes = value.encoded_bytes().map_err(|e| e.into())?; + self.set_slice_to_buffer(&bytes, buffer) + } + + fn decode_t(&mut self, buffer: &[u8]) -> Result { + let out = T::decode(buffer).map_err(|e| e.into())?; + self.add_start(out.encoded_size().map_err(|e| e.into())?)?; + Ok(out) + } } /// Compact Encoding @@ -788,3 +815,33 @@ where /// Decode fn decode(&mut self, buffer: &[u8]) -> Result; } + +impl CompactEncoding for State { + fn preencode(&mut self, value: &T) -> Result { + self.preencode_t(value) + } + + fn encode(&mut self, value: &T, buffer: &mut [u8]) -> Result { + self.encode_t(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> Result { + self.decode_t(buffer) + } +} + +/// Implement this trait on a type and it can be used with: +/// State::preencode/encode/decode +pub trait CompactEncodable { + /// Error type that occors when pre/enc/decoding + type Error: std::error::Error + Into; + + /// The size required in the buffer for this time + fn encoded_size(&self) -> Result; + /// The bytes resulting from encoding this type + fn encoded_bytes(&self) -> Result, Self::Error>; + /// Decode a value from the buffer + fn decode(buffer: &[u8]) -> Result + where + Self: Sized; +} From dfaa143bb558d4b9d98f9bf51362c1d48bf1929b Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 30 Dec 2024 10:54:22 -0600 Subject: [PATCH 02/56] implement CompactEncodable for Vec --- src/types.rs | 159 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 144 insertions(+), 15 deletions(-) diff --git a/src/types.rs b/src/types.rs index 9cecbb2..ac0f469 100644 --- a/src/types.rs +++ b/src/types.rs @@ -306,15 +306,7 @@ impl State { &mut self, uint: &T, ) -> Result { - let increment: usize = if *uint < T::from(U16_SIGNIFIER.into()) { - 1 - } else if *uint <= T::from(0xffff) { - 3 - } else if *uint <= T::from(0xffffffff) { - 5 - } else { - 9 - }; + let increment = uint_var_encoded_size(uint); self.add_end(increment) } @@ -795,13 +787,19 @@ impl State { } fn decode_t(&mut self, buffer: &[u8]) -> Result { - let out = T::decode(buffer).map_err(|e| e.into())?; - self.add_start(out.encoded_size().map_err(|e| e.into())?)?; - Ok(out) + let (result, remaining_buffer) = T::decode(buffer).map_err(|e| e.into())?; + let before = buffer.len(); + let after = remaining_buffer.len(); + if after > before { + todo!() + } + self.add_start(before - after)?; + Ok(result) } } -/// Compact Encoding +/// Compact Encoding. You must implement `CompactEncoding` where T is the type to be encoded. +/// The trait must be implemented for [`State`]. pub trait CompactEncoding where T: fmt::Debug, @@ -830,18 +828,149 @@ impl CompactEncoding for State { } } +fn uint_var_encoded_size + Ord>(uint: &T) -> usize { + if *uint < T::from(U16_SIGNIFIER.into()) { + 1 + } else if *uint <= T::from(0xffff) { + 3 + } else if *uint <= T::from(0xffffffff) { + 5 + } else { + 9 + } +} + +fn usize_encoded_bytes(uint: usize) -> Vec { + if uint < U16_SIGNIFIER.into() { + uint.to_le_bytes().to_vec() + } else if uint <= 0xffff { + let mut out = vec![U16_SIGNIFIER]; + out.extend(uint.to_le_bytes()); + out + } else if uint <= 0xffffffff { + let mut out = vec![U32_SIGNIFIER]; + out.extend(uint.to_le_bytes()); + out + } else { + let mut out = vec![U64_SIGNIFIER]; + out.extend(uint.to_le_bytes()); + out + } +} + +fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { + let [one, two, rest @ ..] = buffer else { + todo!() + }; + let value: u16 = (*one as u16) | ((*two as u16) << 8); + Ok((value, rest)) +} + +fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { + let [one, two, three, four, rest @ ..] = buffer else { + return Err(EncodingError::new( + EncodingErrorKind::Overflow, + "Not enough bytes to decode u32", + )); + }; + let value: u32 = + (*one as u32) | ((*two as u32) << 8) | ((*three as u32) << 16) | ((*four as u32) << 24); + Ok((value, rest)) +} + +fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { + let [one, two, three, four, five, six, seven, eight, rest @ ..] = buffer else { + return Err(EncodingError::new( + EncodingErrorKind::Overflow, + "Not enough bytes to decode u64", + )); + }; + let value: u64 = (*one as u64) + | ((*two as u64) << 8) + | ((*three as u64) << 16) + | ((*four as u64) << 24) + | ((*five as u64) << 32) + | ((*six as u64) << 40) + | ((*seven as u64) << 48) + | ((*eight as u64) << 56); + Ok((value, rest)) +} + +fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { + let [first, rest @ ..] = buffer else { + todo!("silec had zero bytes") + }; + let first = *first; + if first < U16_SIGNIFIER { + Ok((first.into(), rest)) + } else if first == U16_SIGNIFIER { + let (out, rest) = decode_u16(buffer)?; + return Ok((out.into(), rest)); + } else if first == U32_SIGNIFIER { + let (out, rest) = decode_u32(buffer)?; + let out: usize = out.try_into().map_err(|_e| { + EncodingError::new(EncodingErrorKind::Overflow, "u32 is bigger than usize") + })?; + return Ok((out, rest)); + } else { + let (out, rest) = decode_u64(buffer)?; + let out: usize = out.try_into().map_err(|_e| { + EncodingError::new(EncodingErrorKind::Overflow, "u64 is bigger than usize") + })?; + return Ok((out, rest)); + } +} + /// Implement this trait on a type and it can be used with: /// State::preencode/encode/decode pub trait CompactEncodable { /// Error type that occors when pre/enc/decoding - type Error: std::error::Error + Into; + type Error: std::error::Error + From + Into; /// The size required in the buffer for this time fn encoded_size(&self) -> Result; /// The bytes resulting from encoding this type fn encoded_bytes(&self) -> Result, Self::Error>; /// Decode a value from the buffer - fn decode(buffer: &[u8]) -> Result + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), Self::Error> where Self: Sized; } + +impl CompactEncodable for Vec { + type Error = T::Error; + + fn encoded_size(&self) -> Result { + let mut size = uint_var_encoded_size(&(self.len() as u32)); + for item in self.iter() { + size += item.encoded_size()?; + } + Ok(size) + } + + fn encoded_bytes(&self) -> Result, Self::Error> { + let mut out = vec![]; + out.extend(usize_encoded_bytes(self.len())); + for item in self.iter() { + out.extend(item.encoded_bytes()?); + } + Ok(out) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), Self::Error> + where + Self: Sized, + { + let (length, mut rest) = usize_decode(buffer).map_err(|_e| todo!())?; + if length > 0x100000 { + todo!() + } + let mut out = vec![]; + for _ in 0..length { + let result: (T, &[u8]) = ::decode(rest)?; + rest = result.1; + out.push(result.0); + } + Ok((out, rest)) + } +} From f5bf11cbef58ad739232786f19a621196078849d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 30 Dec 2024 10:56:48 -0600 Subject: [PATCH 03/56] just EncodingError --- src/types.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/types.rs b/src/types.rs index ac0f469..4b0613e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -924,23 +924,18 @@ fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { /// Implement this trait on a type and it can be used with: /// State::preencode/encode/decode pub trait CompactEncodable { - /// Error type that occors when pre/enc/decoding - type Error: std::error::Error + From + Into; - /// The size required in the buffer for this time - fn encoded_size(&self) -> Result; + fn encoded_size(&self) -> Result; /// The bytes resulting from encoding this type - fn encoded_bytes(&self) -> Result, Self::Error>; + fn encoded_bytes(&self) -> Result, EncodingError>; /// Decode a value from the buffer - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), Self::Error> + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized; } impl CompactEncodable for Vec { - type Error = T::Error; - - fn encoded_size(&self) -> Result { + fn encoded_size(&self) -> Result { let mut size = uint_var_encoded_size(&(self.len() as u32)); for item in self.iter() { size += item.encoded_size()?; @@ -948,7 +943,7 @@ impl CompactEncodable for Vec { Ok(size) } - fn encoded_bytes(&self) -> Result, Self::Error> { + fn encoded_bytes(&self) -> Result, EncodingError> { let mut out = vec![]; out.extend(usize_encoded_bytes(self.len())); for item in self.iter() { @@ -957,7 +952,7 @@ impl CompactEncodable for Vec { Ok(out) } - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), Self::Error> + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, { From 945e8aa6cbe2fa9d04f270fb5c2abac6c64eac73 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 30 Dec 2024 12:26:50 -0600 Subject: [PATCH 04/56] make encoded_bytes take buffer --- src/types.rs | 83 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/src/types.rs b/src/types.rs index 4b0613e..6146d5c 100644 --- a/src/types.rs +++ b/src/types.rs @@ -782,8 +782,10 @@ impl State { value: &T, buffer: &mut [u8], ) -> Result { - let bytes = value.encoded_bytes().map_err(|e| e.into())?; - self.set_slice_to_buffer(&bytes, buffer) + let start_len = buffer.len(); + let rest = value.encoded_bytes(buffer).map_err(|e| e.into())?; + let offset = start_len - rest.len(); + self.add_start(offset) } fn decode_t(&mut self, buffer: &[u8]) -> Result { @@ -840,24 +842,47 @@ fn uint_var_encoded_size + Ord>(uint: &T) -> usize { } } -fn usize_encoded_bytes(uint: usize) -> Vec { +fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { if uint < U16_SIGNIFIER.into() { - uint.to_le_bytes().to_vec() + encode_u8(uint as u8, buffer) } else if uint <= 0xffff { - let mut out = vec![U16_SIGNIFIER]; - out.extend(uint.to_le_bytes()); - out + let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { + todo!() + }; + sig[0] = U16_SIGNIFIER; + encode_u16(uint as u16, rest) } else if uint <= 0xffffffff { - let mut out = vec![U32_SIGNIFIER]; - out.extend(uint.to_le_bytes()); - out + let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { + todo!() + }; + sig[0] = U32_SIGNIFIER; + encode_u32(uint as u32, rest) } else { - let mut out = vec![U64_SIGNIFIER]; - out.extend(uint.to_le_bytes()); - out + let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { + todo!() + }; + sig[0] = U64_SIGNIFIER; + encode_u64(uint as u64, rest) } } +fn encode_u8(val: u8, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + let b = val.to_le_bytes(); + let Some((target, rest)) = buffer.split_first_chunk_mut::<1>() else { + todo!() + }; + target.copy_from_slice(&b); + Ok(rest) +} +fn encode_u16(val: u16, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + let b = val.to_le_bytes(); + let Some((target, rest)) = buffer.split_first_chunk_mut::<2>() else { + todo!() + }; + target.copy_from_slice(&b); + Ok(rest) +} + fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { let [one, two, rest @ ..] = buffer else { todo!() @@ -866,6 +891,15 @@ fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { Ok((value, rest)) } +fn encode_u32(val: u32, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + let b = val.to_le_bytes(); + let Some((target, rest)) = buffer.split_first_chunk_mut::<4>() else { + todo!() + }; + target.copy_from_slice(&b); + Ok(rest) +} + fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { let [one, two, three, four, rest @ ..] = buffer else { return Err(EncodingError::new( @@ -878,6 +912,14 @@ fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { Ok((value, rest)) } +fn encode_u64(val: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + let b = val.to_le_bytes(); + let Some((target, rest)) = buffer.split_first_chunk_mut::<8>() else { + todo!() + }; + target.copy_from_slice(&b); + Ok(rest) +} fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { let [one, two, three, four, five, six, seven, eight, rest @ ..] = buffer else { return Err(EncodingError::new( @@ -927,8 +969,9 @@ pub trait CompactEncodable { /// The size required in the buffer for this time fn encoded_size(&self) -> Result; /// The bytes resulting from encoding this type - fn encoded_bytes(&self) -> Result, EncodingError>; - /// Decode a value from the buffer + // TODO add buffer argument. Change result to return remaining buffer + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; + /// Decode a value from the buffer. Returns the value and remaining undecoded bytes fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized; @@ -943,15 +986,13 @@ impl CompactEncodable for Vec { Ok(size) } - fn encoded_bytes(&self) -> Result, EncodingError> { - let mut out = vec![]; - out.extend(usize_encoded_bytes(self.len())); + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let mut rest = usize_encoded_bytes(self.len(), buffer)?; for item in self.iter() { - out.extend(item.encoded_bytes()?); + rest = item.encoded_bytes(rest)?; } - Ok(out) + Ok(rest) } - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, From def5baee6749045f37a48bab569f4c85ecfec44e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 1 Jan 2025 16:26:39 -0600 Subject: [PATCH 05/56] Add usize encoding for compactencodable --- src/types.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/types.rs b/src/types.rs index 6146d5c..1b44c47 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,7 @@ //! Basic types of compact_encoding. use std::convert::TryFrom; use std::fmt; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::ops::Range; const U16_SIGNIFIER: u8 = 0xfd; @@ -306,7 +307,15 @@ impl State { &mut self, uint: &T, ) -> Result { - let increment = uint_var_encoded_size(uint); + let increment = if uint < &T::from(u32::from(U16_SIGNIFIER)) { + 1 + } else if uint <= &(0xffff.into()) { + 3 + } else if uint <= &(0xffffffff.into()) { + 5 + } else { + 9 + }; self.add_end(increment) } @@ -830,19 +839,21 @@ impl CompactEncoding for State { } } -fn uint_var_encoded_size + Ord>(uint: &T) -> usize { - if *uint < T::from(U16_SIGNIFIER.into()) { +/// The number of bytes required to encode this number +pub fn usize_encoded_size(val: usize) -> usize { + if val < U16_SIGNIFIER.into() { 1 - } else if *uint <= T::from(0xffff) { + } else if val <= 0xffff { 3 - } else if *uint <= T::from(0xffffffff) { + } else if val <= 0xffffffff { 5 } else { 9 } } -fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { +/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. +pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { if uint < U16_SIGNIFIER.into() { encode_u8(uint as u8, buffer) } else if uint <= 0xffff { @@ -938,7 +949,8 @@ fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { Ok((value, rest)) } -fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { +/// decode a `usize` from `buffer` and return the remaining bytes +pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { let [first, rest @ ..] = buffer else { todo!("silec had zero bytes") }; @@ -979,7 +991,7 @@ pub trait CompactEncodable { impl CompactEncodable for Vec { fn encoded_size(&self) -> Result { - let mut size = uint_var_encoded_size(&(self.len() as u32)); + let mut size = usize_encoded_size(self.len()); for item in self.iter() { size += item.encoded_size()?; } From df76e71dd89120054436f6d29f9f108ff15d1bd1 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 1 Jan 2025 16:27:02 -0600 Subject: [PATCH 06/56] Add IpAddr encodings --- src/types.rs | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/types.rs b/src/types.rs index 1b44c47..fd5eb56 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1022,3 +1022,78 @@ impl CompactEncodable for Vec { Ok((out, rest)) } } + +impl CompactEncodable for Ipv4Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(4) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { + todo!() + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { + todo!() + }; + Ok((Ipv4Addr::from(*dest), rest)) + } +} +impl CompactEncodable for Ipv6Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(4) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { + todo!() + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { + todo!() + }; + Ok((Ipv6Addr::from(*dest), rest)) + } +} + +/// Write `source` to `buffer` and return the remainder of `buffer`. +pub fn write_array<'a, const N: usize>( + source: &[u8; N], + buffer: &'a mut [u8], +) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::() else { + todo!() + }; + dest.copy_from_slice(source); + Ok(rest) +} + +/// split the first `N` bytes of `buffer` off and return them +pub fn take_array( + buffer: &[u8], +) -> std::result::Result<([u8; N], &[u8]), EncodingError> { + let Some((out, rest)) = buffer.split_first_chunk::() else { + todo!() + }; + Ok((*out, rest)) +} From b81646f75e210ff77aeae4f4d29c81fff7571f0f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 3 Apr 2025 10:47:31 -0400 Subject: [PATCH 07/56] impl Clone, PartialEq, and Error for EncodingError --- src/types.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/types.rs b/src/types.rs index fd5eb56..ca3013b 100644 --- a/src/types.rs +++ b/src/types.rs @@ -9,7 +9,7 @@ const U32_SIGNIFIER: u8 = 0xfe; const U64_SIGNIFIER: u8 = 0xff; /// Specific type [EncodingError] -#[derive(fmt::Debug)] +#[derive(fmt::Debug, Clone, PartialEq)] pub enum EncodingErrorKind { /// Encoding or decoding did not stay between [State] `start` and `end`. OutOfBounds, @@ -20,7 +20,7 @@ pub enum EncodingErrorKind { } /// Encoding/decoding error. -#[derive(fmt::Debug)] +#[derive(fmt::Debug, Clone, PartialEq)] pub struct EncodingError { /// Specific type of error pub kind: EncodingErrorKind, @@ -28,6 +28,12 @@ pub struct EncodingError { pub message: String, } +impl std::error::Error for EncodingError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + None + } +} + impl EncodingError { /// Create EncodingError pub fn new(kind: EncodingErrorKind, message: &str) -> Self { From de6c4fd992077a699d49f69c72fa4a7a6fb97ef9 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 3 Apr 2025 10:48:06 -0400 Subject: [PATCH 08/56] docs --- src/types.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/types.rs b/src/types.rs index ca3013b..c18d4f1 100644 --- a/src/types.rs +++ b/src/types.rs @@ -981,8 +981,11 @@ pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { } } -/// Implement this trait on a type and it can be used with: -/// State::preencode/encode/decode +/// Instead of carrying around [`State`] we just use a buffer. +/// To track how much buffer is used (like we do with [`State::start`]) +/// we return a slice of the unused portion after encoding. +/// +/// If you Implement this trait on a type then it will automatically implement [`CompactEncoding`]. pub trait CompactEncodable { /// The size required in the buffer for this time fn encoded_size(&self) -> Result; @@ -993,6 +996,21 @@ pub trait CompactEncodable { fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized; + + /// Encode `self` into a `Vec`. This is just a helper method for: + /// ``` + /// # use std::net::Ipv4Addr; + /// # use compact_encoding::types::CompactEncodable; + /// let foo: Ipv4Addr = "0.0.0.0".parse()?; + /// let mut buff = vec![0; foo.encoded_size()?]; + /// foo.encoded_bytes(&mut buff)?; + /// # Ok::<(), Box>(()) + /// ``` + fn to_bytes(&self) -> Result, EncodingError> { + let mut buff = vec![0; self.encoded_size()?]; + self.encoded_bytes(&mut buff)?; + Ok(buff) + } } impl CompactEncodable for Vec { @@ -1011,6 +1029,7 @@ impl CompactEncodable for Vec { } Ok(rest) } + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, @@ -1083,6 +1102,7 @@ impl CompactEncodable for Ipv6Addr { } /// Write `source` to `buffer` and return the remainder of `buffer`. +/// Errors when `N < buffer.len()` pub fn write_array<'a, const N: usize>( source: &[u8; N], buffer: &'a mut [u8], From 23ddffcdc48430641652e604c84bcebb2319445b Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 3 Apr 2025 10:59:21 -0400 Subject: [PATCH 09/56] take `&[T]` instead of `&Vec` in funcs --- src/types.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/types.rs b/src/types.rs index c18d4f1..6908d99 100644 --- a/src/types.rs +++ b/src/types.rs @@ -455,7 +455,7 @@ impl State { } /// Preencode a vector byte buffer - pub fn preencode_buffer_vec(&mut self, value: &Vec) -> Result { + pub fn preencode_buffer_vec(&mut self, value: &[u8]) -> Result { let len = value.len(); self.preencode_usize_var(&len)?; self.add_end(len) @@ -488,7 +488,7 @@ impl State { /// Preencode a raw byte buffer. Only possible to use if this is the last value /// of the State. - pub fn preencode_raw_buffer(&mut self, value: &Vec) -> Result { + pub fn preencode_raw_buffer(&mut self, value: &[u8]) -> Result { self.add_end(value.len()) } @@ -562,7 +562,7 @@ impl State { } /// Preencode a string array - pub fn preencode_string_array(&mut self, value: &Vec) -> Result { + pub fn preencode_string_array(&mut self, value: &[String]) -> Result { let len = value.len(); self.preencode_usize_var(&len)?; for string_value in value.iter() { @@ -596,7 +596,7 @@ impl State { } /// Preencode an u32 array - pub fn preencode_u32_array(&mut self, value: &Vec) -> Result { + pub fn preencode_u32_array(&mut self, value: &[u32]) -> Result { let len = value.len(); self.preencode_usize_var(&len)?; let total_len = len.checked_mul(4).ok_or_else(|| { @@ -637,10 +637,7 @@ impl State { } /// Preencode a fixed 32 byte value array - pub fn preencode_fixed_32_array( - &mut self, - value: &Vec<[u8; 32]>, - ) -> Result { + pub fn preencode_fixed_32_array(&mut self, value: &[[u8; 32]]) -> Result { let len = value.len(); self.preencode(&len)?; let size = len.checked_mul(32).ok_or_else(|| { From 2a28141d3fa7d3b894ba81dab22f57cc50323a53 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 5 Apr 2025 16:05:36 -0500 Subject: [PATCH 10/56] handle errors, fix todo!() --- src/types.rs | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/types.rs b/src/types.rs index 6908d99..729a83a 100644 --- a/src/types.rs +++ b/src/types.rs @@ -784,7 +784,7 @@ impl State { } fn preencode_t(&mut self, value: &T) -> Result { - let size = value.encoded_size().map_err(|e| e.into())?; + let size = value.encoded_size()?; let out = self.add_end(size)?; Ok(out) } @@ -795,19 +795,16 @@ impl State { buffer: &mut [u8], ) -> Result { let start_len = buffer.len(); - let rest = value.encoded_bytes(buffer).map_err(|e| e.into())?; + let rest = value.encoded_bytes(buffer)?; let offset = start_len - rest.len(); self.add_start(offset) } fn decode_t(&mut self, buffer: &[u8]) -> Result { - let (result, remaining_buffer) = T::decode(buffer).map_err(|e| e.into())?; - let before = buffer.len(); + let start_len = buffer.len(); + let (result, remaining_buffer) = T::decode(buffer)?; let after = remaining_buffer.len(); - if after > before { - todo!() - } - self.add_start(before - after)?; + self.add_start(start_len - after)?; Ok(result) } } @@ -1104,8 +1101,15 @@ pub fn write_array<'a, const N: usize>( source: &[u8; N], buffer: &'a mut [u8], ) -> std::result::Result<&'a mut [u8], EncodingError> { + let blen = buffer.len(); let Some((dest, rest)) = buffer.split_first_chunk_mut::() else { - todo!() + return Err(EncodingError::new( + EncodingErrorKind::OutOfBounds, + &format!( + "Could not write [{}] bytes to buffer of length [{}]", + N, blen + ), + )); }; dest.copy_from_slice(source); Ok(rest) @@ -1116,7 +1120,14 @@ pub fn take_array( buffer: &[u8], ) -> std::result::Result<([u8; N], &[u8]), EncodingError> { let Some((out, rest)) = buffer.split_first_chunk::() else { - todo!() + return Err(EncodingError::new( + EncodingErrorKind::OutOfBounds, + &format!( + "Could not write [{}] bytes to buffer of length [{}]", + N, + buffer.len() + ), + )); }; Ok((*out, rest)) } From 0b6cadbef93fd257fb13e290c6830238ea933a1e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 09:44:43 -0500 Subject: [PATCH 11/56] Add helper funcs for building EncodingError --- src/types.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/types.rs b/src/types.rs index 729a83a..900ac13 100644 --- a/src/types.rs +++ b/src/types.rs @@ -42,6 +42,27 @@ impl EncodingError { message: message.to_string(), } } + /// Helper function for making an overflow error + pub fn overflow(message: &str) -> Self { + Self { + kind: EncodingErrorKind::Overflow, + message: message.to_string(), + } + } + /// Helper function for making an out of bounds error + pub fn out_of_bounds(message: &str) -> Self { + Self { + kind: EncodingErrorKind::OutOfBounds, + message: message.to_string(), + } + } + /// Helper function for making an invalid data error + pub fn invalid_data(message: &str) -> Self { + Self { + kind: EncodingErrorKind::InvalidData, + message: message.to_string(), + } + } } impl fmt::Display for EncodingError { From de293d93ffb055a5896cb912f16e31e76bd117d7 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 10:11:29 -0500 Subject: [PATCH 12/56] Add encodable module --- src/encodable.rs | 901 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/types.rs | 315 +---------------- 3 files changed, 905 insertions(+), 312 deletions(-) create mode 100644 src/encodable.rs diff --git a/src/encodable.rs b/src/encodable.rs new file mode 100644 index 0000000..dd570ee --- /dev/null +++ b/src/encodable.rs @@ -0,0 +1,901 @@ +#![allow(unused)] +//! CompactEncodable stuff +use std::net::{Ipv4Addr, Ipv6Addr}; + +use crate::{ + types::{U16_SIGNIFIER, U32_SIGNIFIER, U64_SIGNIFIER}, + EncodingError, +}; + +/* +impl CompactEncoding for State { + fn preencode(&mut self, value: &T) -> Result { + self.preencode_t(value) + } + + fn encode(&mut self, value: &T, buffer: &mut [u8]) -> Result { + self.encode_t(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> Result { + self.decode_t(buffer) + } +} +*/ + +/// Instead of carrying around [`State`] we just use a buffer. +/// To track how much buffer is used (like we do with [`State::start`]) +/// we return a slice of the unused portion after encoding. +/// +/// If you Implement this trait on a type then it will automatically implement [`CompactEncoding`]. +pub trait CompactEncodable { + /// The size required in the buffer for this time + fn encoded_size(&self) -> Result; + /// The bytes resulting from encoding this type + // TODO add buffer argument. Change result to return remaining buffer + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; + /// Decode a value from the buffer. Returns the value and remaining undecoded bytes + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized; + + /// Encode `self` into a `Vec`. This is just a helper method for: + /// ``` + /// # use std::net::Ipv4Addr; + /// # use compact_encoding::encodable::CompactEncodable; + /// let foo: Ipv4Addr = "0.0.0.0".parse()?; + /// let mut buff = vec![0; foo.encoded_size()?]; + /// foo.encoded_bytes(&mut buff)?; + /// # Ok::<(), Box>(()) + /// ``` + fn to_bytes(&self) -> Result, EncodingError> { + let mut buff = vec![0; self.encoded_size()?]; + self.encoded_bytes(&mut buff)?; + Ok(buff) + } +} + +/* +impl CompactEncodable for Vec { + fn encoded_size(&self) -> Result { + let mut size = usize_encoded_size(self.len()); + for item in self.iter() { + size += item.encoded_size()?; + } + Ok(size) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let mut rest = usize_encoded_bytes(self.len(), buffer)?; + for item in self.iter() { + rest = item.encoded_bytes(rest)?; + } + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (length, mut rest) = usize_decode(buffer).map_err(|_e| todo!())?; + if length > 0x100000 { + todo!() + } + let mut out = vec![]; + for _ in 0..length { + let result: (T, &[u8]) = ::decode(rest)?; + rest = result.1; + out.push(result.0); + } + Ok((out, rest)) + } +} +*/ + +impl CompactEncodable for Ipv4Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(4) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { + todo!() + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { + todo!() + }; + Ok((Ipv4Addr::from(*dest), rest)) + } +} +impl CompactEncodable for Ipv6Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(4) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { + todo!() + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { + todo!() + }; + Ok((Ipv6Addr::from(*dest), rest)) + } +} +/// helper for mapping the first element of a two eleent tuple +macro_rules! map_first { + ($res:expr, $f:expr) => {{ + let (one, two) = $res; + let mapped = $f(one)?; + (mapped, two) + }}; +} + +/// The number of bytes required to encode this number +pub fn usize_encoded_size(val: usize) -> usize { + if val < U16_SIGNIFIER.into() { + 1 + } else if val <= 0xffff { + 3 + } else if val <= 0xffffffff { + 5 + } else { + 9 + } +} + +/// The number of bytes required to encode this number. +/// We only need this for u64 because all other uints can be converted to usize reliably. +pub fn u64_var_encoded_size(val: u64) -> usize { + if val < U16_SIGNIFIER.into() { + 1 + } else if val <= 0xffff { + 3 + } else if val <= 0xffffffff { + 5 + } else { + 9 + } +} + +/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. +pub fn u64_encoded_bytes(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + if uint < U16_SIGNIFIER.into() { + encode_u8(uint as u8, buffer) + } else if uint <= 0xffff { + let rest = write_array(&[U16_SIGNIFIER], buffer)?; + encode_u16(uint as u16, rest) + } else if uint <= 0xffffffff { + let rest = write_array(&[U32_SIGNIFIER], buffer)?; + encode_u32(uint as u32, rest) + } else { + let rest = write_array(&[U64_SIGNIFIER], buffer)?; + encode_u64(uint, rest) + } +} + +/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. +pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + u64_encoded_bytes(uint as u64, buffer) +} + +/// decode a `usize` from `buffer` and return the remaining bytes +pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { + let [first, rest @ ..] = buffer else { + todo!("silec had zero bytes") + }; + let first = *first; + if first < U16_SIGNIFIER { + Ok((first.into(), rest)) + } else if first == U16_SIGNIFIER { + let (out, rest) = decode_u16(buffer)?; + return Ok((out.into(), rest)); + } else if first == U32_SIGNIFIER { + let (out, rest) = decode_u32(buffer)?; + let out: usize = out + .try_into() + .map_err(|_e| EncodingError::overflow("u32 is bigger than usize"))?; + return Ok((out, rest)); + } else { + let (out, rest) = decode_u64(buffer)?; + let out: usize = out + .try_into() + .map_err(|_e| EncodingError::overflow("u64 is bigger than usize"))?; + return Ok((out, rest)); + } +} + +/// Split a slice in two at `mid`. Returns encoding error when `mid` is out of bounds. +pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), EncodingError> { + buffer.split_at_checked(mid).ok_or_else(|| { + EncodingError::out_of_bounds(&format!( + "Cound not read [{mid}] bytes from buffer of length [{}]", + buffer.len() + )) + }) +} +fn get_slices_mut_checked( + buffer: &mut [u8], + mid: usize, +) -> Result<(&mut [u8], &mut [u8]), EncodingError> { + let len = buffer.len(); + buffer.split_at_mut_checked(mid).ok_or_else(|| { + EncodingError::out_of_bounds(&format!( + "Cound not read [{mid}] bytes from buffer of length [{len}]" + )) + }) +} + +/// Write `source` to `buffer` and return the remainder of `buffer`. +/// Errors when `N < buffer.len()` +pub fn write_array<'a, const N: usize>( + source: &[u8; N], + buffer: &'a mut [u8], +) -> std::result::Result<&'a mut [u8], EncodingError> { + let blen = buffer.len(); + let Some((dest, rest)) = buffer.split_first_chunk_mut::() else { + return Err(EncodingError::out_of_bounds(&format!( + "Could not write [{}] bytes to buffer of length [{}]", + N, blen + ))); + }; + dest.copy_from_slice(source); + Ok(rest) +} + +/// split the first `N` bytes of `buffer` off and return them +pub fn take_array( + buffer: &[u8], +) -> std::result::Result<([u8; N], &[u8]), EncodingError> { + let Some((out, rest)) = buffer.split_first_chunk::() else { + return Err(EncodingError::out_of_bounds(&format!( + "Could not write [{}] bytes to buffer of length [{}]", + N, + buffer.len() + ))); + }; + Ok((*out, rest)) +} +/// split the first `N` bytes of `buffer` off and return them +pub fn take_array_mut( + buffer: &mut [u8], +) -> std::result::Result<(&mut [u8; N], &mut [u8]), EncodingError> { + let blen = buffer.len(); + let Some((out, rest)) = buffer.split_first_chunk_mut::() else { + return Err(EncodingError::out_of_bounds(&format!( + "Could not write [{}] bytes to buffer of length [{blen}]", + N, + ))); + }; + Ok((out, rest)) +} + +/// write `source` to `buffer` and return remaining buffer +pub fn write_slice<'a>(source: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let mid = source.len(); + let (mut dest, rest) = get_slices_mut_checked(buffer, mid)?; + dest.copy_from_slice(source); + Ok(rest) +} + +/// Gives the encoded size of a type +pub trait EncodedSize { + /// The encoded size of a value in bytes + fn encoded_size(&self) -> Result; +} + +impl EncodedSize for usize { + fn encoded_size(&self) -> Result { + Ok(usize_encoded_size(*self)) + } +} + +/// Marker trait for types with sizes known at compile time +pub trait ConstantSize { + /// The encoded size of the value in bytes. + const ENCODED_SIZE: usize; +} + +impl EncodedSize for T { + fn encoded_size(&self) -> Result { + Ok(::ENCODED_SIZE) + } +} + +macro_rules! constant_sized { + // Process multiple type/size pairs separated by semicolons + ( + $($type:ty, $size:expr);+ $(;)? + ) => { + $( + impl ConstantSize for $type { + const ENCODED_SIZE: usize = $size; + } + )+ + }; +} + +constant_sized! { + u8, 1; + u16, 2; + u32, 4; + u64, 8; +} + +fn encoded_size_str(value: &str) -> Result { + Ok(usize_encoded_size(value.len()) + value.len()) +} + +/// Helper to convert a vec to an array, and fail with an encoding error when needed +pub fn bytes_fixed_from_vec(value: &[u8]) -> Result<[u8; N], EncodingError> { + <[u8; N]>::try_from(value).map_err(|e| { + EncodingError::invalid_data(&format!( + "Could not covert vec with length [{}] to array of length [{}]", + value.len(), + N + )) + }) +} + +/// Encoded a fixed sized array to a buffer +pub fn encode_bytes_fixed<'a, const N: usize>( + value: &[u8; N], + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], EncodingError> { + write_array(value, buffer) +} + +/// Encoded a fixed sized array to a buffer +pub fn decode_bytes_fixed( + buffer: &[u8], +) -> Result<([u8; N], &[u8]), EncodingError> { + take_array(buffer) + //write_array(value, buffer) +} + +impl CompactEncodable for [u8; N] { + fn encoded_size(&self) -> Result { + Ok(N) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + write_array(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + take_array(buffer) + } +} + +impl ConstantSize for [T; N] { + const ENCODED_SIZE: usize = N * T::ENCODED_SIZE; +} + +impl EncodedSize for &[T] { + fn encoded_size(&self) -> Result { + Ok(self.len().encoded_size()? + + self.iter().try_fold(0usize, |acc, x| { + acc.checked_add(x.encoded_size()?).ok_or_else(|| { + EncodingError::overflow(&format!( + "Could not calculate encoded size of `&[{}]`, total size overflowed", + std::any::type_name::() + )) + }) + })?) + } +} + +fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { + let (data, rest) = take_array::<2>(buffer)?; + Ok(((data[0] as u16) | ((data[1] as u16) << 8), rest)) +} + +fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { + let (data, rest) = take_array::<4>(buffer)?; + Ok(( + ((data[0] as u32) + | ((data[1] as u32) << 8) + | ((data[2] as u32) << 16) + | ((data[3] as u32) << 24)), + rest, + )) +} + +fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { + let (data, rest) = take_array::<8>(buffer)?; + Ok(( + ((data[0] as u64) + | ((data[1] as u64) << 8) + | ((data[2] as u64) << 16) + | ((data[3] as u64) << 24) + | ((data[4] as u64) << 32) + | ((data[5] as u64) << 40) + | ((data[6] as u64) << 48) + | ((data[7] as u64) << 56)), + rest, + )) +} + +fn decode_u8(buffer: &[u8]) -> Result<(u8, &[u8]), EncodingError> { + let (data, rest) = take_array::<1>(buffer)?; + Ok((data[0], rest)) +} + +fn decode_u32_var(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { + let ([first], rest) = take_array::<1>(buffer)?; + Ok(match first { + x if x < U16_SIGNIFIER => (x.into(), rest), + U16_SIGNIFIER => { + let (val, rest) = decode_u16(rest)?; + (val.into(), rest) + } + _ => decode_u32(rest)?, + }) +} + +fn decode_u64_var(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { + let ([first], rest) = take_array::<1>(buffer)?; + Ok(match first { + x if x < U16_SIGNIFIER => (x.into(), rest), + U16_SIGNIFIER => map_first!(decode_u16(rest)?, |x: u16| Ok(x.into())), + U32_SIGNIFIER => map_first!(decode_u32(rest)?, |x: u32| Ok(x.into())), + _ => decode_u64(rest)?, + }) +} + +fn decode_usize_var(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { + let ([first], rest) = take_array::<1>(buffer)?; + Ok(match first { + x if x < U16_SIGNIFIER => (x.into(), rest), + U16_SIGNIFIER => map_first!(decode_u16(rest)?, |x: u16| Ok(x.into())), + U32_SIGNIFIER => { + map_first!(decode_u32(rest)?, |val| usize::try_from(val).map_err( + |_| EncodingError::overflow("Could not convert u32 to usize") + )) + } + _ => { + map_first!(decode_u64(rest)?, |val| usize::try_from(val).map_err( + |_| EncodingError::overflow("Could not convert u64 to usize") + )) + } + }) +} + +fn decode_buffer_vec(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { + let (n_bytes, rest) = decode_usize_var(buffer)?; + let (out, rest) = get_slices_checked(rest, n_bytes)?; + Ok((out.to_vec(), rest)) +} + +fn decode_string(buffer: &[u8]) -> Result<(String, &[u8]), EncodingError> { + let (len, rest) = decode_usize_var(buffer)?; + let (str_buff, rest) = get_slices_checked(rest, len)?; + let out = String::from_utf8(str_buff.to_vec()) + .map_err(|e| EncodingError::invalid_data(&format!("String is invalid UTF-8, {e}")))?; + Ok((out, rest)) +} + +fn decode_string_array(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { + let (len, mut rest) = decode_usize_var(buffer)?; + let mut out = Vec::with_capacity(len); + for _ in 0..len { + let next = decode_string(rest)?; + out.push(next.0); + rest = next.1; + } + Ok((out, rest)) +} + +fn encode_u8(val: u8, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + write_array(&val.to_le_bytes(), buffer) +} +fn encode_u16(val: u16, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + write_array(&val.to_le_bytes(), buffer) +} +fn encode_u32(val: u32, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + write_array(&val.to_le_bytes(), buffer) +} +fn encode_u64(val: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + write_array(&val.to_le_bytes(), buffer) +} + +fn encode_usize_var<'a>( + value: &usize, + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], EncodingError> { + if *value < U16_SIGNIFIER.into() { + encode_u8(*value as u8, buffer) + } else if *value <= 0xffff { + encode_u16(*value as u16, write_array(&[U16_SIGNIFIER], buffer)?) + } else if *value <= 0xffffffff { + let value = u32::try_from(*value).map_err(|e| { + EncodingError::overflow(&format!("count not covert usize [{value}] to u32")) + })?; + encode_u32(value, write_array(&[U32_SIGNIFIER], buffer)?) + } else { + let value = u64::try_from(*value).map_err(|e| { + EncodingError::overflow(&format!("count not covert usize [{value}] to u64")) + })?; + encode_u64(value, write_array(&[U64_SIGNIFIER], buffer)?) + } +} + +fn encode_str<'a>(value: &str, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = encode_usize_var(&value.len(), buffer)?; + write_slice(value.as_bytes(), rest) +} + +fn encode_buffer<'a>(value: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let rest = encode_usize_var(&value.len(), buffer)?; + write_slice(value, rest) +} + +fn encode_string_array<'a>( + value: &[String], + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], EncodingError> { + let mut rest = encode_usize_var(&value.len(), buffer)?; + for s in value { + rest = encode_str(s, rest)?; + } + Ok(rest) +} + +impl CompactEncodable for u16 { + fn encoded_size(&self) -> Result { + ::encoded_size(self) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_u16(*self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode_u16(buffer) + } +} +impl CompactEncodable for u32 { + fn encoded_size(&self) -> Result { + Ok(usize_encoded_size(*self as usize)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_usize_var(&(*self as usize), buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode_u32(buffer) + } +} +impl CompactEncodable for u64 { + fn encoded_size(&self) -> Result { + Ok(u64_var_encoded_size(*self)) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + u64_encoded_bytes(*self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode_u64(buffer) + } +} + +impl CompactEncodable for String { + fn encoded_size(&self) -> Result { + encoded_size_str(self) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_str(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode_string(buffer) + } +} + +impl CompactEncodable for Vec { + fn encoded_size(&self) -> Result { + let mut out = usize_encoded_size(self.len()); + for s in self { + out += s.encoded_size()?; + } + Ok(out) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + let mut rest = usize_encoded_bytes(self.len(), buffer)?; + for s in self { + rest = s.encoded_bytes(rest)?; + } + Ok(rest) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let (len, mut rest) = decode_usize_var(buffer)?; + let mut out = Vec::with_capacity(len); + for _ in 0..len { + let result = String::decode(rest)?; + out.push(result.0); + rest = result.1; + } + Ok((out, rest)) + } +} + +impl CompactEncodable for Vec { + fn encoded_size(&self) -> Result { + Ok(usize_encoded_size(self.len()) + self.len()) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_buffer(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + decode_buffer_vec(buffer) + } +} + +/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` +pub trait VecEncodable: CompactEncodable { + /// Calculate the resulting size in bytes of `vec` + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized; + + /// Encode `vec` to `buffer` + fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + encode_vec(vec, buffer) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + where + Self: Sized, + { + decode_vec(buffer) + } +} + +fn encode_vec<'a, T: CompactEncodable + Sized>( + vec: &[T], + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], EncodingError> { + let mut rest = encode_usize_var(&vec.len(), buffer)?; + for x in vec { + rest = ::encoded_bytes(x, rest)?; + } + Ok(rest) +} + +fn decode_vec<'a, T: CompactEncodable + Sized>( + buffer: &[u8], +) -> Result<(Vec, &[u8]), EncodingError> { + let (len, mut rest) = decode_usize_var(buffer)?; + let mut out = Vec::with_capacity(len); + for i in 0..len { + let res = ::decode(rest)?; + out.push(res.0); + rest = res.1; + } + Ok((out, rest)) +} + +impl CompactEncodable for Vec { + fn encoded_size(&self) -> Result { + T::vec_encoded_size(self) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + ::encoded_bytes(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + ::decode(buffer) + } +} + +impl VecEncodable for u32 { + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized, + { + Ok(usize_encoded_size(vec.len()) + (vec.len() * u32::ENCODED_SIZE)) + } +} + +/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` +pub trait BoxArrayEncodable: CompactEncodable { + /// The encoded size in bytes + fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result + where + Self: Sized; + + /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer + fn encoded_bytes<'a>( + vec: &Box<[Self]>, + buffer: &'a mut [u8], + ) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + encode_vec(vec, buffer) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + where + Self: Sized, + { + let (result, rest) = decode_vec(buffer)?; + Ok((result.into_boxed_slice(), rest)) + } +} + +impl CompactEncodable for u8 { + fn encoded_size(&self) -> Result { + Ok(1) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + write_array(&[*self], buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([out], rest) = take_array::<1>(buffer)?; + Ok((out, rest)) + } +} + +impl BoxArrayEncodable for u8 { + fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result + where + Self: Sized, + { + Ok(usize_encoded_size(boxed.len()) + boxed.len()) + } + + fn encoded_bytes<'a>( + boxed: &Box<[Self]>, + buffer: &'a mut [u8], + ) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + let rest = encode_usize_var(&boxed.len(), buffer)?; + write_slice(boxed, rest) + } + + fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + where + Self: Sized, + { + let (len, rest) = decode_usize_var(buffer)?; + let (out, rest) = get_slices_checked(rest, len)?; + Ok((out.into(), rest)) + } +} + +impl CompactEncodable for Box<[T]> { + fn encoded_size(&self) -> Result { + T::boxed_array_encoded_size(self) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + ::encoded_bytes(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + ::decode(buffer) + } +} + +#[cfg(test)] +#[allow(unused)] +mod test { + use super::*; + + fn get_constant_encoded_size(_x: T) -> usize { + T::ENCODED_SIZE + } + + #[test] + fn decode_buff_vec() -> Result<(), EncodingError> { + let buf = &[1, 1]; + let (a, b) = decode_buffer_vec(buf)?; + assert_eq!(a, &[1]); + assert_eq!(b, &[]); + Ok(()) + } + macro_rules! check_usize_var_enc_dec { + ($size:expr, $value:expr) => { + let mut buffer = vec![0; usize_encoded_size($value)]; + assert_eq!(buffer.len(), $size); + let remaining = encode_usize_var(&$value, &mut buffer)?; + assert!(remaining.is_empty()); + let (result, rest) = decode_usize_var(&buffer)?; + assert!(rest.is_empty()); + assert_eq!(result, $value); + }; + } + + #[test] + fn usize_var_enc_dec() -> Result<(), EncodingError> { + check_usize_var_enc_dec!(1, 42); + check_usize_var_enc_dec!(1 + 2, 256); + check_usize_var_enc_dec!(1 + 4, 65536); + check_usize_var_enc_dec!(1 + 8, 4294967296); + + Ok(()) + } + + #[test] + fn constant_sized() { + let foo: u8 = 42; + let bar: u32 = 64; + let qux: u64 = 64; + let keylike = [2u8; 32]; + //let x = foo as ConstantSize; + assert_eq!(get_constant_encoded_size(foo), 1); + assert_eq!(get_constant_encoded_size(bar), 4); + assert_eq!(get_constant_encoded_size(qux), 8); + assert_eq!(get_constant_encoded_size(keylike), 32); + } +} diff --git a/src/lib.rs b/src/lib.rs index ee2e0ad..797ce1f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -143,6 +143,7 @@ //! If you export the struct, orphan rule will require you to //! implement a wrapper for [State], e.g. `struct MyState(State);` and implement //! [CompactEncoding] for the wrapper struct instead. +pub mod encodable; pub mod generic; pub mod types; pub use types::{CompactEncoding, EncodingError, EncodingErrorKind, State}; diff --git a/src/types.rs b/src/types.rs index 900ac13..18a5886 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,12 +1,11 @@ //! Basic types of compact_encoding. use std::convert::TryFrom; use std::fmt; -use std::net::{Ipv4Addr, Ipv6Addr}; use std::ops::Range; -const U16_SIGNIFIER: u8 = 0xfd; -const U32_SIGNIFIER: u8 = 0xfe; -const U64_SIGNIFIER: u8 = 0xff; +pub(crate) const U16_SIGNIFIER: u8 = 0xfd; +pub(crate) const U32_SIGNIFIER: u8 = 0xfe; +pub(crate) const U64_SIGNIFIER: u8 = 0xff; /// Specific type [EncodingError] #[derive(fmt::Debug, Clone, PartialEq)] @@ -831,7 +830,6 @@ impl State { } /// Compact Encoding. You must implement `CompactEncoding` where T is the type to be encoded. -/// The trait must be implemented for [`State`]. pub trait CompactEncoding where T: fmt::Debug, @@ -845,310 +843,3 @@ where /// Decode fn decode(&mut self, buffer: &[u8]) -> Result; } - -impl CompactEncoding for State { - fn preencode(&mut self, value: &T) -> Result { - self.preencode_t(value) - } - - fn encode(&mut self, value: &T, buffer: &mut [u8]) -> Result { - self.encode_t(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_t(buffer) - } -} - -/// The number of bytes required to encode this number -pub fn usize_encoded_size(val: usize) -> usize { - if val < U16_SIGNIFIER.into() { - 1 - } else if val <= 0xffff { - 3 - } else if val <= 0xffffffff { - 5 - } else { - 9 - } -} - -/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. -pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - if uint < U16_SIGNIFIER.into() { - encode_u8(uint as u8, buffer) - } else if uint <= 0xffff { - let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { - todo!() - }; - sig[0] = U16_SIGNIFIER; - encode_u16(uint as u16, rest) - } else if uint <= 0xffffffff { - let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { - todo!() - }; - sig[0] = U32_SIGNIFIER; - encode_u32(uint as u32, rest) - } else { - let Some((sig, rest)) = buffer.split_first_chunk_mut::<1>() else { - todo!() - }; - sig[0] = U64_SIGNIFIER; - encode_u64(uint as u64, rest) - } -} - -fn encode_u8(val: u8, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - let b = val.to_le_bytes(); - let Some((target, rest)) = buffer.split_first_chunk_mut::<1>() else { - todo!() - }; - target.copy_from_slice(&b); - Ok(rest) -} -fn encode_u16(val: u16, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - let b = val.to_le_bytes(); - let Some((target, rest)) = buffer.split_first_chunk_mut::<2>() else { - todo!() - }; - target.copy_from_slice(&b); - Ok(rest) -} - -fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { - let [one, two, rest @ ..] = buffer else { - todo!() - }; - let value: u16 = (*one as u16) | ((*two as u16) << 8); - Ok((value, rest)) -} - -fn encode_u32(val: u32, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - let b = val.to_le_bytes(); - let Some((target, rest)) = buffer.split_first_chunk_mut::<4>() else { - todo!() - }; - target.copy_from_slice(&b); - Ok(rest) -} - -fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { - let [one, two, three, four, rest @ ..] = buffer else { - return Err(EncodingError::new( - EncodingErrorKind::Overflow, - "Not enough bytes to decode u32", - )); - }; - let value: u32 = - (*one as u32) | ((*two as u32) << 8) | ((*three as u32) << 16) | ((*four as u32) << 24); - Ok((value, rest)) -} - -fn encode_u64(val: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - let b = val.to_le_bytes(); - let Some((target, rest)) = buffer.split_first_chunk_mut::<8>() else { - todo!() - }; - target.copy_from_slice(&b); - Ok(rest) -} -fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { - let [one, two, three, four, five, six, seven, eight, rest @ ..] = buffer else { - return Err(EncodingError::new( - EncodingErrorKind::Overflow, - "Not enough bytes to decode u64", - )); - }; - let value: u64 = (*one as u64) - | ((*two as u64) << 8) - | ((*three as u64) << 16) - | ((*four as u64) << 24) - | ((*five as u64) << 32) - | ((*six as u64) << 40) - | ((*seven as u64) << 48) - | ((*eight as u64) << 56); - Ok((value, rest)) -} - -/// decode a `usize` from `buffer` and return the remaining bytes -pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { - let [first, rest @ ..] = buffer else { - todo!("silec had zero bytes") - }; - let first = *first; - if first < U16_SIGNIFIER { - Ok((first.into(), rest)) - } else if first == U16_SIGNIFIER { - let (out, rest) = decode_u16(buffer)?; - return Ok((out.into(), rest)); - } else if first == U32_SIGNIFIER { - let (out, rest) = decode_u32(buffer)?; - let out: usize = out.try_into().map_err(|_e| { - EncodingError::new(EncodingErrorKind::Overflow, "u32 is bigger than usize") - })?; - return Ok((out, rest)); - } else { - let (out, rest) = decode_u64(buffer)?; - let out: usize = out.try_into().map_err(|_e| { - EncodingError::new(EncodingErrorKind::Overflow, "u64 is bigger than usize") - })?; - return Ok((out, rest)); - } -} - -/// Instead of carrying around [`State`] we just use a buffer. -/// To track how much buffer is used (like we do with [`State::start`]) -/// we return a slice of the unused portion after encoding. -/// -/// If you Implement this trait on a type then it will automatically implement [`CompactEncoding`]. -pub trait CompactEncodable { - /// The size required in the buffer for this time - fn encoded_size(&self) -> Result; - /// The bytes resulting from encoding this type - // TODO add buffer argument. Change result to return remaining buffer - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; - /// Decode a value from the buffer. Returns the value and remaining undecoded bytes - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> - where - Self: Sized; - - /// Encode `self` into a `Vec`. This is just a helper method for: - /// ``` - /// # use std::net::Ipv4Addr; - /// # use compact_encoding::types::CompactEncodable; - /// let foo: Ipv4Addr = "0.0.0.0".parse()?; - /// let mut buff = vec![0; foo.encoded_size()?]; - /// foo.encoded_bytes(&mut buff)?; - /// # Ok::<(), Box>(()) - /// ``` - fn to_bytes(&self) -> Result, EncodingError> { - let mut buff = vec![0; self.encoded_size()?]; - self.encoded_bytes(&mut buff)?; - Ok(buff) - } -} - -impl CompactEncodable for Vec { - fn encoded_size(&self) -> Result { - let mut size = usize_encoded_size(self.len()); - for item in self.iter() { - size += item.encoded_size()?; - } - Ok(size) - } - - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let mut rest = usize_encoded_bytes(self.len(), buffer)?; - for item in self.iter() { - rest = item.encoded_bytes(rest)?; - } - Ok(rest) - } - - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - let (length, mut rest) = usize_decode(buffer).map_err(|_e| todo!())?; - if length > 0x100000 { - todo!() - } - let mut out = vec![]; - for _ in 0..length { - let result: (T, &[u8]) = ::decode(rest)?; - rest = result.1; - out.push(result.0); - } - Ok((out, rest)) - } -} - -impl CompactEncodable for Ipv4Addr { - fn encoded_size(&self) -> std::result::Result { - Ok(4) - } - - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { - let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { - todo!() - }; - dest.copy_from_slice(&self.octets()); - Ok(rest) - } - - fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { - todo!() - }; - Ok((Ipv4Addr::from(*dest), rest)) - } -} -impl CompactEncodable for Ipv6Addr { - fn encoded_size(&self) -> std::result::Result { - Ok(4) - } - - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { - let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { - todo!() - }; - dest.copy_from_slice(&self.octets()); - Ok(rest) - } - - fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { - todo!() - }; - Ok((Ipv6Addr::from(*dest), rest)) - } -} - -/// Write `source` to `buffer` and return the remainder of `buffer`. -/// Errors when `N < buffer.len()` -pub fn write_array<'a, const N: usize>( - source: &[u8; N], - buffer: &'a mut [u8], -) -> std::result::Result<&'a mut [u8], EncodingError> { - let blen = buffer.len(); - let Some((dest, rest)) = buffer.split_first_chunk_mut::() else { - return Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "Could not write [{}] bytes to buffer of length [{}]", - N, blen - ), - )); - }; - dest.copy_from_slice(source); - Ok(rest) -} - -/// split the first `N` bytes of `buffer` off and return them -pub fn take_array( - buffer: &[u8], -) -> std::result::Result<([u8; N], &[u8]), EncodingError> { - let Some((out, rest)) = buffer.split_first_chunk::() else { - return Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "Could not write [{}] bytes to buffer of length [{}]", - N, - buffer.len() - ), - )); - }; - Ok((*out, rest)) -} From 2caaf37e8269664204ff6ad137edfb31dfb08f22 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 10:17:16 -0500 Subject: [PATCH 13/56] Add conversion for Encoding to Encodable --- src/types.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/types.rs b/src/types.rs index 18a5886..1c8caac 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,4 +1,5 @@ //! Basic types of compact_encoding. +use crate::encodable::CompactEncodable; use std::convert::TryFrom; use std::fmt; use std::ops::Range; @@ -803,13 +804,15 @@ impl State { self.set_slice_to_buffer(&bytes[4..8], buffer) } - fn preencode_t(&mut self, value: &T) -> Result { + /// Like [`State::preencode`] but for `CompactEncodable` types + pub fn preencode_t(&mut self, value: &T) -> Result { let size = value.encoded_size()?; let out = self.add_end(size)?; Ok(out) } - fn encode_t( + /// Like [`State::encode`] but for `CompactEncodable` types + pub fn encode_t( &mut self, value: &T, buffer: &mut [u8], @@ -820,7 +823,8 @@ impl State { self.add_start(offset) } - fn decode_t(&mut self, buffer: &[u8]) -> Result { + /// Like [`State::decode`] but for `CompactEncodable` types + pub fn decode_t(&mut self, buffer: &[u8]) -> Result { let start_len = buffer.len(); let (result, remaining_buffer) = T::decode(buffer)?; let after = remaining_buffer.len(); From ff969aa514cb056b7016fbc73ba65b2abb258259 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 10:22:42 -0500 Subject: [PATCH 14/56] lints --- src/encodable.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/encodable.rs b/src/encodable.rs index dd570ee..210c216 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -716,7 +716,7 @@ fn encode_vec<'a, T: CompactEncodable + Sized>( Ok(rest) } -fn decode_vec<'a, T: CompactEncodable + Sized>( +fn decode_vec( buffer: &[u8], ) -> Result<(Vec, &[u8]), EncodingError> { let (len, mut rest) = decode_usize_var(buffer)?; @@ -755,6 +755,8 @@ impl VecEncodable for u32 { } } +// NB: we DO want &Box<..> because we want the trait to work for boxed things +#[allow(clippy::borrowed_box)] /// Define this trait for `T` to get `Box<[T]>: CompactEncodable` pub trait BoxArrayEncodable: CompactEncodable { /// The encoded size in bytes From 363657490840fade178874fee002f8d4a63d0c11 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 10:46:16 -0500 Subject: [PATCH 15/56] refactoring --- src/encodable.rs | 141 +++++++++++++++-------------------------------- 1 file changed, 44 insertions(+), 97 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 210c216..4d58b8f 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -7,22 +7,6 @@ use crate::{ EncodingError, }; -/* -impl CompactEncoding for State { - fn preencode(&mut self, value: &T) -> Result { - self.preencode_t(value) - } - - fn encode(&mut self, value: &T, buffer: &mut [u8]) -> Result { - self.encode_t(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_t(buffer) - } -} -*/ - /// Instead of carrying around [`State`] we just use a buffer. /// To track how much buffer is used (like we do with [`State::start`]) /// we return a slice of the unused portion after encoding. @@ -55,42 +39,59 @@ pub trait CompactEncodable { } } -/* -impl CompactEncodable for Vec { - fn encoded_size(&self) -> Result { - let mut size = usize_encoded_size(self.len()); - for item in self.iter() { - size += item.encoded_size()?; - } - Ok(size) +/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` +pub trait VecEncodable: CompactEncodable { + /// Calculate the resulting size in bytes of `vec` + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized; + + /// Encode `vec` to `buffer` + fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + encode_vec(vec, buffer) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let mut rest = usize_encoded_bytes(self.len(), buffer)?; - for item in self.iter() { - rest = item.encoded_bytes(rest)?; - } - Ok(rest) + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + where + Self: Sized, + { + decode_vec(buffer) } +} - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> +// NB: we DO want &Box<..> because we want the trait to work for boxed things +#[allow(clippy::borrowed_box)] +/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` +pub trait BoxArrayEncodable: CompactEncodable { + /// The encoded size in bytes + fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result + where + Self: Sized; + + /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer + fn encoded_bytes<'a>( + vec: &Box<[Self]>, + buffer: &'a mut [u8], + ) -> Result<&'a mut [u8], EncodingError> where Self: Sized, { - let (length, mut rest) = usize_decode(buffer).map_err(|_e| todo!())?; - if length > 0x100000 { - todo!() - } - let mut out = vec![]; - for _ in 0..length { - let result: (T, &[u8]) = ::decode(rest)?; - rest = result.1; - out.push(result.0); - } - Ok((out, rest)) + encode_vec(vec, buffer) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + where + Self: Sized, + { + let (result, rest) = decode_vec(buffer)?; + Ok((result.into_boxed_slice(), rest)) } } -*/ impl CompactEncodable for Ipv4Addr { fn encoded_size(&self) -> std::result::Result { @@ -681,30 +682,6 @@ impl CompactEncodable for Vec { } } -/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` -pub trait VecEncodable: CompactEncodable { - /// Calculate the resulting size in bytes of `vec` - fn vec_encoded_size(vec: &[Self]) -> Result - where - Self: Sized; - - /// Encode `vec` to `buffer` - fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> - where - Self: Sized, - { - encode_vec(vec, buffer) - } - - /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> - where - Self: Sized, - { - decode_vec(buffer) - } -} - fn encode_vec<'a, T: CompactEncodable + Sized>( vec: &[T], buffer: &'a mut [u8], @@ -755,36 +732,6 @@ impl VecEncodable for u32 { } } -// NB: we DO want &Box<..> because we want the trait to work for boxed things -#[allow(clippy::borrowed_box)] -/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` -pub trait BoxArrayEncodable: CompactEncodable { - /// The encoded size in bytes - fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result - where - Self: Sized; - - /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer - fn encoded_bytes<'a>( - vec: &Box<[Self]>, - buffer: &'a mut [u8], - ) -> Result<&'a mut [u8], EncodingError> - where - Self: Sized, - { - encode_vec(vec, buffer) - } - - /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> - where - Self: Sized, - { - let (result, rest) = decode_vec(buffer)?; - Ok((result.into_boxed_slice(), rest)) - } -} - impl CompactEncodable for u8 { fn encoded_size(&self) -> Result { Ok(1) From d72f8bd47a2a462123d10b5d55a2ee4891744e28 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 10:57:59 -0500 Subject: [PATCH 16/56] Remove EncodedSize trai --- src/encodable.rs | 87 +++--------------------------------------------- 1 file changed, 5 insertions(+), 82 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 4d58b8f..dbdcee0 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -7,6 +7,8 @@ use crate::{ EncodingError, }; +const U16_SIZE: usize = 2; +const U32_SIZE: usize = 4; /// Instead of carrying around [`State`] we just use a buffer. /// To track how much buffer is used (like we do with [`State::start`]) /// we return a slice of the unused portion after encoding. @@ -95,7 +97,7 @@ pub trait BoxArrayEncodable: CompactEncodable { impl CompactEncodable for Ipv4Addr { fn encoded_size(&self) -> std::result::Result { - Ok(4) + Ok(U32_SIZE) } fn encoded_bytes<'a>( @@ -301,50 +303,6 @@ pub fn write_slice<'a>(source: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u Ok(rest) } -/// Gives the encoded size of a type -pub trait EncodedSize { - /// The encoded size of a value in bytes - fn encoded_size(&self) -> Result; -} - -impl EncodedSize for usize { - fn encoded_size(&self) -> Result { - Ok(usize_encoded_size(*self)) - } -} - -/// Marker trait for types with sizes known at compile time -pub trait ConstantSize { - /// The encoded size of the value in bytes. - const ENCODED_SIZE: usize; -} - -impl EncodedSize for T { - fn encoded_size(&self) -> Result { - Ok(::ENCODED_SIZE) - } -} - -macro_rules! constant_sized { - // Process multiple type/size pairs separated by semicolons - ( - $($type:ty, $size:expr);+ $(;)? - ) => { - $( - impl ConstantSize for $type { - const ENCODED_SIZE: usize = $size; - } - )+ - }; -} - -constant_sized! { - u8, 1; - u16, 2; - u32, 4; - u64, 8; -} - fn encoded_size_str(value: &str) -> Result { Ok(usize_encoded_size(value.len()) + value.len()) } @@ -393,24 +351,6 @@ impl CompactEncodable for [u8; N] { } } -impl ConstantSize for [T; N] { - const ENCODED_SIZE: usize = N * T::ENCODED_SIZE; -} - -impl EncodedSize for &[T] { - fn encoded_size(&self) -> Result { - Ok(self.len().encoded_size()? - + self.iter().try_fold(0usize, |acc, x| { - acc.checked_add(x.encoded_size()?).ok_or_else(|| { - EncodingError::overflow(&format!( - "Could not calculate encoded size of `&[{}]`, total size overflowed", - std::any::type_name::() - )) - }) - })?) - } -} - fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { let (data, rest) = take_array::<2>(buffer)?; Ok(((data[0] as u16) | ((data[1] as u16) << 8), rest)) @@ -569,7 +509,7 @@ fn encode_string_array<'a>( impl CompactEncodable for u16 { fn encoded_size(&self) -> Result { - ::encoded_size(self) + Ok(U16_SIZE) } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { @@ -728,7 +668,7 @@ impl VecEncodable for u32 { where Self: Sized, { - Ok(usize_encoded_size(vec.len()) + (vec.len() * u32::ENCODED_SIZE)) + Ok(usize_encoded_size(vec.len()) + (vec.len() * 4)) } } @@ -801,10 +741,6 @@ impl CompactEncodable for Box<[T]> { mod test { use super::*; - fn get_constant_encoded_size(_x: T) -> usize { - T::ENCODED_SIZE - } - #[test] fn decode_buff_vec() -> Result<(), EncodingError> { let buf = &[1, 1]; @@ -834,17 +770,4 @@ mod test { Ok(()) } - - #[test] - fn constant_sized() { - let foo: u8 = 42; - let bar: u32 = 64; - let qux: u64 = 64; - let keylike = [2u8; 32]; - //let x = foo as ConstantSize; - assert_eq!(get_constant_encoded_size(foo), 1); - assert_eq!(get_constant_encoded_size(bar), 4); - assert_eq!(get_constant_encoded_size(qux), 8); - assert_eq!(get_constant_encoded_size(keylike), 32); - } } From 75020e311e6b8fc14f233fb534010dcc69a711d3 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 11:17:57 -0500 Subject: [PATCH 17/56] fix todo!() --- src/encodable.rs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index dbdcee0..ecbc650 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -1,6 +1,9 @@ #![allow(unused)] //! CompactEncodable stuff -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::{ + any::type_name, + net::{Ipv4Addr, Ipv6Addr}, +}; use crate::{ types::{U16_SIGNIFIER, U32_SIGNIFIER, U64_SIGNIFIER}, @@ -9,11 +12,10 @@ use crate::{ const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; + /// Instead of carrying around [`State`] we just use a buffer. /// To track how much buffer is used (like we do with [`State::start`]) /// we return a slice of the unused portion after encoding. -/// -/// If you Implement this trait on a type then it will automatically implement [`CompactEncoding`]. pub trait CompactEncodable { /// The size required in the buffer for this time fn encoded_size(&self) -> Result; @@ -105,7 +107,10 @@ impl CompactEncodable for Ipv4Addr { buffer: &'a mut [u8], ) -> std::result::Result<&'a mut [u8], EncodingError> { let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { - todo!() + return Err(EncodingError::out_of_bounds(&format!( + "Colud not encode {}, not enough room in buffer", + std::any::type_name::() + ))); }; dest.copy_from_slice(&self.octets()); Ok(rest) @@ -116,7 +121,10 @@ impl CompactEncodable for Ipv4Addr { Self: Sized, { let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { - todo!() + return Err(EncodingError::out_of_bounds(&format!( + "Colud not decode {}, buffer not big enough", + std::any::type_name::() + ))); }; Ok((Ipv4Addr::from(*dest), rest)) } @@ -131,7 +139,10 @@ impl CompactEncodable for Ipv6Addr { buffer: &'a mut [u8], ) -> std::result::Result<&'a mut [u8], EncodingError> { let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { - todo!() + return Err(EncodingError::out_of_bounds(&format!( + "Colud not encode {}, not enough room in buffer", + std::any::type_name::() + ))); }; dest.copy_from_slice(&self.octets()); Ok(rest) @@ -142,7 +153,10 @@ impl CompactEncodable for Ipv6Addr { Self: Sized, { let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { - todo!() + return Err(EncodingError::out_of_bounds(&format!( + "Colud not decode {}, buffer not big enough", + std::any::type_name::() + ))); }; Ok((Ipv6Addr::from(*dest), rest)) } @@ -207,7 +221,9 @@ pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], /// decode a `usize` from `buffer` and return the remaining bytes pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { let [first, rest @ ..] = buffer else { - todo!("silec had zero bytes") + return Err(EncodingError::out_of_bounds( + "Colud not decode usize, empty buffer", + )); }; let first = *first; if first < U16_SIGNIFIER { @@ -737,7 +753,6 @@ impl CompactEncodable for Box<[T]> { } #[cfg(test)] -#[allow(unused)] mod test { use super::*; From 5a23165bf7d9e499d403675facc709dd8a4e3807 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 10 Apr 2025 13:15:51 -0500 Subject: [PATCH 18/56] begin adding encodable tests --- src/encodable.rs | 278 ++++++++++++++++++++++++++++----------------- tests/encodable.rs | 25 ++++ 2 files changed, 201 insertions(+), 102 deletions(-) create mode 100644 tests/encodable.rs diff --git a/src/encodable.rs b/src/encodable.rs index ecbc650..431e406 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -43,6 +43,61 @@ pub trait CompactEncodable { } } +#[macro_export] +/// Used for defining CompactEncodable::encoded_size. +/// Pass self and a list of fields to call encoded_size on +macro_rules! sum_encoded_size { + // Base case: single field + ($self:ident, $field:ident) => { + $self.$field.encoded_size()? + }; + // Recursive case: first field + rest + ($self: ident, $first:ident, $($rest:ident),+) => { + $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) + }; +} + +#[macro_export] +// TODO is this exported from the crate? +/// Used for defining CompactEncodable::encoded_bytes. +/// Pass self, the buffer and a list of fields to call encoded_size on +macro_rules! map_encodables { + // Base case: single field + ($buffer:expr, $field:ident) => { + $field.encoded_bytes($buffer)? + }; + // Recursive case: first field + rest + ($buffer:expr, $first:ident, $($rest:ident),+) => {{ + let rest = $first.encoded_bytes($buffer)?; + map_encodables!(rest, $($rest),+) + }}; +} + +#[macro_export] +/// Helper for decoding multiple types into a tuple and returning the remaining buffer. +/// It takes as arguments: `(&buffer, [type1, type2, type3, ...])` +/// And returns: `((decoded_type1, decoded_type2, ...), remaining_buffer)` +macro_rules! map_decode { + ($buffer:expr, [ + $($field_type:ty),* $(,)? + ]) => {{ + let mut current_buffer: &[u8] = $buffer; + + // Decode each type into the `result_tuple` + let result_tuple = ( + $( + match <$field_type>::decode(¤t_buffer)? { + (value, new_buf) => { + current_buffer = new_buf; + value + } + }, + )* + ); + Ok((result_tuple, current_buffer)) + }}; +} + /// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` pub trait VecEncodable: CompactEncodable { /// Calculate the resulting size in bytes of `vec` @@ -97,70 +152,6 @@ pub trait BoxArrayEncodable: CompactEncodable { } } -impl CompactEncodable for Ipv4Addr { - fn encoded_size(&self) -> std::result::Result { - Ok(U32_SIZE) - } - - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { - let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { - return Err(EncodingError::out_of_bounds(&format!( - "Colud not encode {}, not enough room in buffer", - std::any::type_name::() - ))); - }; - dest.copy_from_slice(&self.octets()); - Ok(rest) - } - - fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { - return Err(EncodingError::out_of_bounds(&format!( - "Colud not decode {}, buffer not big enough", - std::any::type_name::() - ))); - }; - Ok((Ipv4Addr::from(*dest), rest)) - } -} -impl CompactEncodable for Ipv6Addr { - fn encoded_size(&self) -> std::result::Result { - Ok(4) - } - - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { - let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { - return Err(EncodingError::out_of_bounds(&format!( - "Colud not encode {}, not enough room in buffer", - std::any::type_name::() - ))); - }; - dest.copy_from_slice(&self.octets()); - Ok(rest) - } - - fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { - return Err(EncodingError::out_of_bounds(&format!( - "Colud not decode {}, buffer not big enough", - std::any::type_name::() - ))); - }; - Ok((Ipv6Addr::from(*dest), rest)) - } -} /// helper for mapping the first element of a two eleent tuple macro_rules! map_first { ($res:expr, $f:expr) => {{ @@ -213,6 +204,32 @@ pub fn u64_encoded_bytes(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], Enco } } +/// Sum encoded sizes +pub fn encoded_size(arr: &[impl CompactEncodable]) -> Result { + let mut out = 0; + for x in arr { + out += x.encoded_size()?; + } + Ok(out) +} + +/// Create a buffer from an array of CompactEncodable objects +pub fn create_buffer(arr: &[impl CompactEncodable]) -> Result, EncodingError> { + Ok(vec![0; encoded_size(arr)?]) +} + +/// Encode an array of CompactEncodable objects +pub fn map_encode<'a>( + arr: &[impl CompactEncodable], + buffer: &'a mut [u8], +) -> Result<&'a mut [u8], EncodingError> { + let mut rest = buffer; + for x in arr { + rest = x.encoded_bytes(rest)?; + } + Ok(rest) +} + /// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { u64_encoded_bytes(uint as u64, buffer) @@ -367,40 +384,21 @@ impl CompactEncodable for [u8; N] { } } +fn decode_u8(buffer: &[u8]) -> Result<(u8, &[u8]), EncodingError> { + let (data, rest) = take_array::<1>(buffer)?; + Ok((u8::from_le_bytes(data), rest)) +} fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { let (data, rest) = take_array::<2>(buffer)?; - Ok(((data[0] as u16) | ((data[1] as u16) << 8), rest)) + Ok((u16::from_le_bytes(data), rest)) } - fn decode_u32(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { let (data, rest) = take_array::<4>(buffer)?; - Ok(( - ((data[0] as u32) - | ((data[1] as u32) << 8) - | ((data[2] as u32) << 16) - | ((data[3] as u32) << 24)), - rest, - )) + Ok((u32::from_le_bytes(data), rest)) } - fn decode_u64(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { let (data, rest) = take_array::<8>(buffer)?; - Ok(( - ((data[0] as u64) - | ((data[1] as u64) << 8) - | ((data[2] as u64) << 16) - | ((data[3] as u64) << 24) - | ((data[4] as u64) << 32) - | ((data[5] as u64) << 40) - | ((data[6] as u64) << 48) - | ((data[7] as u64) << 56)), - rest, - )) -} - -fn decode_u8(buffer: &[u8]) -> Result<(u8, &[u8]), EncodingError> { - let (data, rest) = take_array::<1>(buffer)?; - Ok((data[0], rest)) + Ok((u64::from_le_bytes(data), rest)) } fn decode_u32_var(buffer: &[u8]) -> Result<(u32, &[u8]), EncodingError> { @@ -523,6 +521,24 @@ fn encode_string_array<'a>( Ok(rest) } +impl CompactEncodable for u8 { + fn encoded_size(&self) -> Result { + Ok(1) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + write_array(&[*self], buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let ([out], rest) = take_array::<1>(buffer)?; + Ok((out, rest)) + } +} + impl CompactEncodable for u16 { fn encoded_size(&self) -> Result { Ok(U16_SIZE) @@ -539,6 +555,8 @@ impl CompactEncodable for u16 { decode_u16(buffer) } } + +// NB: we want u32 encoded and decoded as variable sized uint impl CompactEncodable for u32 { fn encoded_size(&self) -> Result { Ok(usize_encoded_size(*self as usize)) @@ -638,6 +656,71 @@ impl CompactEncodable for Vec { } } +impl CompactEncodable for Ipv4Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(U32_SIZE) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { + return Err(EncodingError::out_of_bounds(&format!( + "Colud not encode {}, not enough room in buffer", + std::any::type_name::() + ))); + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { + return Err(EncodingError::out_of_bounds(&format!( + "Colud not decode {}, buffer not big enough", + std::any::type_name::() + ))); + }; + Ok((Ipv4Addr::from(*dest), rest)) + } +} +impl CompactEncodable for Ipv6Addr { + fn encoded_size(&self) -> std::result::Result { + Ok(4) + } + + fn encoded_bytes<'a>( + &self, + buffer: &'a mut [u8], + ) -> std::result::Result<&'a mut [u8], EncodingError> { + let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { + return Err(EncodingError::out_of_bounds(&format!( + "Colud not encode {}, not enough room in buffer", + std::any::type_name::() + ))); + }; + dest.copy_from_slice(&self.octets()); + Ok(rest) + } + + fn decode(buffer: &[u8]) -> std::result::Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { + return Err(EncodingError::out_of_bounds(&format!( + "Colud not decode {}, buffer not big enough", + std::any::type_name::() + ))); + }; + Ok((Ipv6Addr::from(*dest), rest)) + } +} + fn encode_vec<'a, T: CompactEncodable + Sized>( vec: &[T], buffer: &'a mut [u8], @@ -688,21 +771,12 @@ impl VecEncodable for u32 { } } -impl CompactEncodable for u8 { - fn encoded_size(&self) -> Result { - Ok(1) - } - - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - write_array(&[*self], buffer) - } - - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> +impl VecEncodable for [u8; N] { + fn vec_encoded_size(vec: &[Self]) -> Result where Self: Sized, { - let ([out], rest) = take_array::<1>(buffer)?; - Ok((out, rest)) + Ok(usize_encoded_size(vec.len()) + (vec.len() * N)) } } diff --git a/tests/encodable.rs b/tests/encodable.rs new file mode 100644 index 0000000..5fdfdaf --- /dev/null +++ b/tests/encodable.rs @@ -0,0 +1,25 @@ +use compact_encoding::{ + encodable::{create_buffer, map_encode, CompactEncodable}, + map_decode, map_encodables, + types::EncodingError, +}; + +#[test] +fn cenc_32_byte_array() -> Result<(), EncodingError> { + let empty_array: Vec<[u8; 32]> = vec![]; + let one_array: Vec<[u8; 32]> = vec![[0; 32]]; + let many_array: Vec<[u8; 32]> = vec![[1; 32], [2; 32], [3; 32]]; + let data = vec![empty_array.clone(), one_array.clone(), many_array.clone()]; + let expected_size = 1 + 1 + 32 + 1 + (3 * 32); + let mut buffer = create_buffer(&data)?; + assert_eq!(buffer.len(), expected_size); + + let rest = map_encodables!(&mut buffer, empty_array, one_array, many_array); + assert!(rest.is_empty()); + + let (result, rest) = map_decode!(&buffer, [Vec<[u8; 32]>, Vec<[u8; 32]>, Vec<[u8;32]>])?; + assert_eq!(result.0, data[0]); + assert_eq!(result.1, data[1]); + assert_eq!(result.2, data[2]); + Ok(()) +} From 9c2f085a968e19ddf50d11d9e4116824f993a469 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 12 Apr 2025 17:03:34 -0400 Subject: [PATCH 19/56] wip new encoding tests --- src/encodable.rs | 34 ++++++++++++++++++++++++---------- tests/encodable.rs | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 431e406..afad4a7 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -46,19 +46,33 @@ pub trait CompactEncodable { #[macro_export] /// Used for defining CompactEncodable::encoded_size. /// Pass self and a list of fields to call encoded_size on -macro_rules! sum_encoded_size { - // Base case: single field - ($self:ident, $field:ident) => { - $self.$field.encoded_size()? - }; - // Recursive case: first field + rest - ($self: ident, $first:ident, $($rest:ident),+) => { - $self.$first.encoded_size()? + sum_encoded_size!($self, $($rest),+) - }; +macro_rules! sum_preencode { + ($($val:expr),+) => {{ + let out: usize = [ + $( + $val.encoded_size()?, + + )* + ].iter().sum(); + out + }} +} + +#[macro_export] +/// Create a buffer from a list of CompactEncodable things to be used for encoding +macro_rules! create_buffer { + ($($val:expr),+) => {{ + let len: usize = [ + $( + $val.encoded_size()?, + + )* + ].iter().sum(); + vec![0; len] + }} } #[macro_export] -// TODO is this exported from the crate? /// Used for defining CompactEncodable::encoded_bytes. /// Pass self, the buffer and a list of fields to call encoded_size on macro_rules! map_encodables { diff --git a/tests/encodable.rs b/tests/encodable.rs index 5fdfdaf..0470657 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,9 +1,16 @@ use compact_encoding::{ - encodable::{create_buffer, map_encode, CompactEncodable}, + create_buffer, + encodable::{create_buffer, CompactEncodable}, map_decode, map_encodables, types::EncodingError, }; +// The max value for 1 byte length is 252 +const MAX_ONE_BYTE_UINT: u8 = 252; + +// The min value for 2 byte length is 253 +const _MIN_TWO_BYTE_UINT: u8 = 253; + #[test] fn cenc_32_byte_array() -> Result<(), EncodingError> { let empty_array: Vec<[u8; 32]> = vec![]; @@ -18,8 +25,41 @@ fn cenc_32_byte_array() -> Result<(), EncodingError> { assert!(rest.is_empty()); let (result, rest) = map_decode!(&buffer, [Vec<[u8; 32]>, Vec<[u8; 32]>, Vec<[u8;32]>])?; + assert!(rest.is_empty()); assert_eq!(result.0, data[0]); assert_eq!(result.1, data[1]); assert_eq!(result.2, data[2]); Ok(()) } + +#[test] +fn cenc_basic() -> Result<(), EncodingError> { + let _str_value_1 = "foo"; + let str_value_2 = (0..MAX_ONE_BYTE_UINT).map(|_| "X").collect::(); + let u32_value_3: u32 = u32::MAX; + let u32_value_4: u32 = 0xF0E1D2C3; + + let mut _buff = create_buffer!(str_value_2, u32_value_3, u32_value_4); + assert_eq!(_buff.len(), 1 + 252 + 1 + 4 + 1 + 4); + + //let rest = map_encodables!(buf, str_value_1, str_value_2, u32_value_3, u32_value_4); + /* + // Strings: 1 byte for length, 3/252 bytes for content + // u32: 1 byte for u32 signifier, 4 bytes for data + assert_eq!(buffer.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); + enc_state.encode_str(str_value_1, &mut buffer)?; + enc_state.encode(&str_value_2, &mut buffer)?; + enc_state.encode(&u32_value_3, &mut buffer)?; + enc_state.encode(&u32_value_4, &mut buffer)?; + let mut dec_state = State::from_buffer(&buffer); + let str_value_1_ret: String = dec_state.decode(&buffer)?; + assert_eq!(str_value_1, str_value_1_ret); + let str_value_2_ret: String = dec_state.decode(&buffer)?; + assert_eq!(str_value_2, str_value_2_ret); + let u32_value_3_ret: u32 = dec_state.decode(&buffer)?; + assert_eq!(u32_value_3, u32_value_3_ret); + let u32_value_4_ret: u32 = dec_state.decode(&buffer)?; + assert_eq!(u32_value_4, u32_value_4_ret); + */ + Ok(()) +} From 108f7ca026b70106927609933d38d50d9b88fe6d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 12 Apr 2025 17:38:17 -0400 Subject: [PATCH 20/56] Fix issues with implementing for String & str --- src/encodable.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index afad4a7..349f3a6 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -16,16 +16,16 @@ const U32_SIZE: usize = 4; /// Instead of carrying around [`State`] we just use a buffer. /// To track how much buffer is used (like we do with [`State::start`]) /// we return a slice of the unused portion after encoding. -pub trait CompactEncodable { +pub trait CompactEncodable { /// The size required in the buffer for this time fn encoded_size(&self) -> Result; /// The bytes resulting from encoding this type // TODO add buffer argument. Change result to return remaining buffer fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; /// Decode a value from the buffer. Returns the value and remaining undecoded bytes - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + fn decode(buffer: &[u8]) -> Result<(Decode, &[u8]), EncodingError> where - Self: Sized; + Decode: Sized; /// Encode `self` into a `Vec`. This is just a helper method for: /// ``` @@ -621,6 +621,20 @@ impl CompactEncodable for String { } } +impl CompactEncodable for str { + fn encoded_size(&self) -> Result { + encoded_size_str(self) + } + + fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_str(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(String, &[u8]), EncodingError> { + decode_string(buffer) + } +} + impl CompactEncodable for Vec { fn encoded_size(&self) -> Result { let mut out = usize_encoded_size(self.len()); From 6625a606812bdcc65179f8fec47b423a975fcacd Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 12 Apr 2025 17:39:44 -0400 Subject: [PATCH 21/56] Fix bug in u32 --- src/encodable.rs | 4 ++-- tests/encodable.rs | 41 +++++++++++++++++++---------------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 349f3a6..18be897 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -584,7 +584,7 @@ impl CompactEncodable for u32 { where Self: Sized, { - decode_u32(buffer) + decode_u32_var(buffer) } } impl CompactEncodable for u64 { @@ -600,7 +600,7 @@ impl CompactEncodable for u64 { where Self: Sized, { - decode_u64(buffer) + decode_u64_var(buffer) } } diff --git a/tests/encodable.rs b/tests/encodable.rs index 0470657..8378ca4 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -34,32 +34,29 @@ fn cenc_32_byte_array() -> Result<(), EncodingError> { #[test] fn cenc_basic() -> Result<(), EncodingError> { - let _str_value_1 = "foo"; + let str_value_1 = "foo"; let str_value_2 = (0..MAX_ONE_BYTE_UINT).map(|_| "X").collect::(); let u32_value_3: u32 = u32::MAX; let u32_value_4: u32 = 0xF0E1D2C3; - let mut _buff = create_buffer!(str_value_2, u32_value_3, u32_value_4); - assert_eq!(_buff.len(), 1 + 252 + 1 + 4 + 1 + 4); + let mut buff = create_buffer!(str_value_1, str_value_2, u32_value_3, u32_value_4); + assert_eq!(buff.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); - //let rest = map_encodables!(buf, str_value_1, str_value_2, u32_value_3, u32_value_4); - /* - // Strings: 1 byte for length, 3/252 bytes for content - // u32: 1 byte for u32 signifier, 4 bytes for data - assert_eq!(buffer.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); - enc_state.encode_str(str_value_1, &mut buffer)?; - enc_state.encode(&str_value_2, &mut buffer)?; - enc_state.encode(&u32_value_3, &mut buffer)?; - enc_state.encode(&u32_value_4, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let str_value_1_ret: String = dec_state.decode(&buffer)?; - assert_eq!(str_value_1, str_value_1_ret); - let str_value_2_ret: String = dec_state.decode(&buffer)?; - assert_eq!(str_value_2, str_value_2_ret); - let u32_value_3_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value_3, u32_value_3_ret); - let u32_value_4_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value_4, u32_value_4_ret); - */ + let rest = map_encodables!( + &mut buff, + str_value_1, + str_value_2, + u32_value_3, + u32_value_4 + ); + assert!(rest.is_empty()); + let (result, remaning_buff) = map_decode!(&buff, [String, String, u32, u32])?; + dbg!(remaning_buff); + assert!(remaning_buff.is_empty()); + + assert_eq!(result.0, str_value_1); + assert_eq!(result.1, str_value_2); + assert_eq!(result.2, u32_value_3); + assert_eq!(result.3, u32_value_4); Ok(()) } From 37fe9c4df66833faf719e33000eb3ec8e132addb Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 12 Apr 2025 18:18:16 -0400 Subject: [PATCH 22/56] redo all tests --- tests/encodable.rs | 196 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 172 insertions(+), 24 deletions(-) diff --git a/tests/encodable.rs b/tests/encodable.rs index 8378ca4..2b73d41 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -9,28 +9,7 @@ use compact_encoding::{ const MAX_ONE_BYTE_UINT: u8 = 252; // The min value for 2 byte length is 253 -const _MIN_TWO_BYTE_UINT: u8 = 253; - -#[test] -fn cenc_32_byte_array() -> Result<(), EncodingError> { - let empty_array: Vec<[u8; 32]> = vec![]; - let one_array: Vec<[u8; 32]> = vec![[0; 32]]; - let many_array: Vec<[u8; 32]> = vec![[1; 32], [2; 32], [3; 32]]; - let data = vec![empty_array.clone(), one_array.clone(), many_array.clone()]; - let expected_size = 1 + 1 + 32 + 1 + (3 * 32); - let mut buffer = create_buffer(&data)?; - assert_eq!(buffer.len(), expected_size); - - let rest = map_encodables!(&mut buffer, empty_array, one_array, many_array); - assert!(rest.is_empty()); - - let (result, rest) = map_decode!(&buffer, [Vec<[u8; 32]>, Vec<[u8; 32]>, Vec<[u8;32]>])?; - assert!(rest.is_empty()); - assert_eq!(result.0, data[0]); - assert_eq!(result.1, data[1]); - assert_eq!(result.2, data[2]); - Ok(()) -} +const MIN_TWO_BYTE_UINT: u8 = 253; #[test] fn cenc_basic() -> Result<(), EncodingError> { @@ -50,8 +29,7 @@ fn cenc_basic() -> Result<(), EncodingError> { u32_value_4 ); assert!(rest.is_empty()); - let (result, remaning_buff) = map_decode!(&buff, [String, String, u32, u32])?; - dbg!(remaning_buff); + let (result, remaning_buff) = map_decode!(&buff, [String, String, u32, u32]); assert!(remaning_buff.is_empty()); assert_eq!(result.0, str_value_1); @@ -60,3 +38,173 @@ fn cenc_basic() -> Result<(), EncodingError> { assert_eq!(result.3, u32_value_4); Ok(()) } + +#[test] +fn cenc_string_long() -> Result<(), EncodingError> { + let value = (0..MIN_TWO_BYTE_UINT).map(|_| "X").collect::(); + assert_eq!(value.len(), 253); + let mut buffer = create_buffer!(value); + assert_eq!(buffer.len(), 1 + 2 + 253); + + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + + let ((result,), rest) = map_decode!(&buffer, [String]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_u32_as_u16() -> Result<(), EncodingError> { + let value: u32 = u16::MAX.into(); + let mut buffer = create_buffer!(value); + // 1 byte for u16 signifier, 2 bytes for length + assert_eq!(buffer.len(), 1 + 2); + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + + let ((result,), rest) = map_decode!(&buffer, [u32]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_u32_as_u8() -> Result<(), EncodingError> { + let value: u32 = MAX_ONE_BYTE_UINT.into(); + let mut buffer = create_buffer!(value); + // 1 byte for data + assert_eq!(buffer.len(), 1); + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + + let ((result,), rest) = map_decode!(&buffer, [u32]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_u64() -> Result<(), EncodingError> { + let value: u64 = 0xF0E1D2C3B4A59687; + let mut buffer = create_buffer!(value); + // 1 byte for u64 signifier, 8 bytes for length + assert_eq!(buffer.len(), 1 + 8); + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + let ((result,), rest) = map_decode!(&buffer, [u64]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_u64_as_u32() -> Result<(), EncodingError> { + let value: u64 = u32::MAX.into(); + let mut buffer = create_buffer!(value); + // 1 byte for u32 signifier, 4 bytes for length + assert_eq!(buffer.len(), 1 + 4); + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + let ((result,), rest) = map_decode!(&buffer, [u64]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_buffer() -> Result<(), EncodingError> { + let buf_value_1 = vec![0xFF, 0x00].into_boxed_slice(); + let buf_value_2 = vec![0xEE, 0x11, 0x22].into_boxed_slice(); + + let mut buffer = create_buffer!(buf_value_1, buf_value_2); + // 1 byte for length, 2 bytes for data + // 1 byte for length, 3 bytes for data + assert_eq!(buffer.len(), 1 + 2 + 1 + 3); + let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2); + assert!(rest.is_empty()); + let (result, rest) = map_decode!(&buffer, [Box<[u8]>, Box<[u8]>]); + assert!(rest.is_empty()); + assert_eq!(result.0, buf_value_1); + assert_eq!(result.1, buf_value_2); + Ok(()) +} + +#[test] +fn cenc_vec() -> Result<(), EncodingError> { + let buf_value_1: Vec = vec![0xFF, 0x00]; + let buf_value_2: Vec = vec![0xFFFFFFFF, 0x11223344, 0x99887766]; + + let mut buffer = create_buffer!(buf_value_1, buf_value_2); + // 1 byte for length, 2 bytes for data + // 1 byte for length, 4*3 bytes for data + assert_eq!(buffer.len(), 1 + 2 + 1 + 12); + + let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2); + assert!(rest.is_empty()); + let (result, rest) = map_decode!(&buffer, [Vec, Vec]); + assert!(rest.is_empty()); + assert_eq!(result.0, buf_value_1); + assert_eq!(result.1, buf_value_2); + Ok(()) +} + +#[test] +fn cenc_string_array() -> Result<(), EncodingError> { + let value = vec!["first".to_string(), "second".to_string()]; + let mut buffer = create_buffer!(value); + // 1 byte for array length, + // 1 byte for string length, 5 bytes for string, + // 1 byte for string length, 6 bytes for string + assert_eq!(buffer.len(), 1 + 1 + 5 + 1 + 6); + let rest = map_encodables!(&mut buffer, value); + assert!(rest.is_empty()); + let ((result,), rest) = map_decode!(&buffer, [Vec]); + assert!(rest.is_empty()); + assert_eq!(result, value); + Ok(()) +} + +#[test] +fn cenc_fixed_and_raw() -> Result<(), EncodingError> { + let buf_value_1: [u8; 16] = [0xEE; 16]; + let buf_value_2: [u8; 32] = [0xFF; 32]; + let buf_value_3: [u8; 3] = [0xFF, 0x11, 0x99]; + + let mut buffer = create_buffer!(buf_value_1, buf_value_2, buf_value_3); + // 1 byte for length, 2 bytes for data + // 1 byte for length, 4*3 bytes for data + assert_eq!(buffer.len(), 16 + 32 + 3); + + let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2, buf_value_3); + assert!(rest.is_empty()); + let (result, rest) = map_decode!(&buffer, [[u8; 16], [u8; 32], [u8; 3]]); + assert!(rest.is_empty()); + assert_eq!(result.0, buf_value_1); + assert_eq!(result.1, buf_value_2); + assert_eq!(result.2, buf_value_3); + Ok(()) +} + +#[test] +fn cenc_32_byte_array() -> Result<(), EncodingError> { + let empty_array: Vec<[u8; 32]> = vec![]; + let one_array: Vec<[u8; 32]> = vec![[0; 32]]; + let many_array: Vec<[u8; 32]> = vec![[1; 32], [2; 32], [3; 32]]; + let data = vec![empty_array.clone(), one_array.clone(), many_array.clone()]; + let expected_size = 1 + 1 + 32 + 1 + (3 * 32); + + let mut buffer = create_buffer(&data)?; + assert_eq!(buffer.len(), expected_size); + + let rest = map_encodables!(&mut buffer, empty_array, one_array, many_array); + assert!(rest.is_empty()); + + let (result, rest) = map_decode!(&buffer, [Vec<[u8; 32]>, Vec<[u8; 32]>, Vec<[u8;32]>]); + assert!(rest.is_empty()); + assert_eq!(result.0, data[0]); + assert_eq!(result.1, data[1]); + assert_eq!(result.2, data[2]); + Ok(()) +} From f24c45577620b3248cfe87690610a426ff7442b8 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 14:36:12 -0400 Subject: [PATCH 23/56] Add VecEncodable for u32 --- src/encodable.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 18be897..408ba10 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -13,9 +13,7 @@ use crate::{ const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; -/// Instead of carrying around [`State`] we just use a buffer. -/// To track how much buffer is used (like we do with [`State::start`]) -/// we return a slice of the unused portion after encoding. +/// Implement for types to get encoding and decoding. pub trait CompactEncodable { /// The size required in the buffer for this time fn encoded_size(&self) -> Result; @@ -108,7 +106,7 @@ macro_rules! map_decode { }, )* ); - Ok((result_tuple, current_buffer)) + (result_tuple, current_buffer) }}; } @@ -797,6 +795,33 @@ impl VecEncodable for u32 { { Ok(usize_encoded_size(vec.len()) + (vec.len() * 4)) } + /// Encode `vec` to `buffer` + fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + let mut rest = encode_usize_var(&vec.len(), buffer)?; + for x in vec { + rest = encode_u32(*x, rest)?; + } + Ok(rest) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + where + Self: Sized, + { + let (len, mut rest) = decode_usize_var(buffer)?; + let mut out = Vec::with_capacity(len); + + for _ in 0..len { + let result = decode_u32(rest)?; + out.push(result.0); + rest = result.1; + } + Ok((out, rest)) + } } impl VecEncodable for [u8; N] { From 36ae9eca867d2da75414bf0b6c459478ca956614 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 14:41:28 -0400 Subject: [PATCH 24/56] add CompactEncodable::create_buffer --- src/encodable.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/encodable.rs b/src/encodable.rs index 408ba10..cd22b5c 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -25,7 +25,8 @@ pub trait CompactEncodable { where Decode: Sized; - /// Encode `self` into a `Vec`. This is just a helper method for: + /// Encode `self` into a `Vec`. This is just a helper method for creating a buffer and + /// encoding to it in one step. /// ``` /// # use std::net::Ipv4Addr; /// # use compact_encoding::encodable::CompactEncodable; @@ -39,6 +40,18 @@ pub trait CompactEncodable { self.encoded_bytes(&mut buff)?; Ok(buff) } + /// Create an empty buffer of the correct size for encoding `self` to. This is just a helper + /// method for: encoding to it in one step. + /// ``` + /// # use std::net::Ipv4Addr; + /// # use compact_encoding::encodable::CompactEncodable; + /// let foo: Ipv4Addr = "0.0.0.0".parse()?; + /// vec![0; foo.encoded_size()?]; + /// # Ok::<(), Box>(()) + /// ``` + fn create_buffer(&self) -> Result, EncodingError> { + Ok(vec![0; self.encoded_size()?]) + } } #[macro_export] From 8ae21539e5eacf5acf3ee6416c65eced68774d14 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 14:43:15 -0400 Subject: [PATCH 25/56] lints --- src/encodable.rs | 49 +++++++++++++----------------------------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index cd22b5c..75843a8 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -1,4 +1,3 @@ -#![allow(unused)] //! CompactEncodable stuff use std::{ any::type_name, @@ -356,7 +355,7 @@ pub fn take_array_mut( /// write `source` to `buffer` and return remaining buffer pub fn write_slice<'a>(source: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let mid = source.len(); - let (mut dest, rest) = get_slices_mut_checked(buffer, mid)?; + let (dest, rest) = get_slices_mut_checked(buffer, mid)?; dest.copy_from_slice(source); Ok(rest) } @@ -369,7 +368,7 @@ fn encoded_size_str(value: &str) -> Result { pub fn bytes_fixed_from_vec(value: &[u8]) -> Result<[u8; N], EncodingError> { <[u8; N]>::try_from(value).map_err(|e| { EncodingError::invalid_data(&format!( - "Could not covert vec with length [{}] to array of length [{}]", + "Could not covert slice with length [{}] to array of length [{}]. Error: [{e}]", value.len(), N )) @@ -409,10 +408,6 @@ impl CompactEncodable for [u8; N] { } } -fn decode_u8(buffer: &[u8]) -> Result<(u8, &[u8]), EncodingError> { - let (data, rest) = take_array::<1>(buffer)?; - Ok((u8::from_le_bytes(data), rest)) -} fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { let (data, rest) = take_array::<2>(buffer)?; Ok((u16::from_le_bytes(data), rest)) @@ -480,17 +475,6 @@ fn decode_string(buffer: &[u8]) -> Result<(String, &[u8]), EncodingError> { Ok((out, rest)) } -fn decode_string_array(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { - let (len, mut rest) = decode_usize_var(buffer)?; - let mut out = Vec::with_capacity(len); - for _ in 0..len { - let next = decode_string(rest)?; - out.push(next.0); - rest = next.1; - } - Ok((out, rest)) -} - fn encode_u8(val: u8, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { write_array(&val.to_le_bytes(), buffer) } @@ -514,12 +498,16 @@ fn encode_usize_var<'a>( encode_u16(*value as u16, write_array(&[U16_SIGNIFIER], buffer)?) } else if *value <= 0xffffffff { let value = u32::try_from(*value).map_err(|e| { - EncodingError::overflow(&format!("count not covert usize [{value}] to u32")) + EncodingError::overflow(&format!( + "count not covert usize [{value}] to u32. Error: [{e}]" + )) })?; encode_u32(value, write_array(&[U32_SIGNIFIER], buffer)?) } else { let value = u64::try_from(*value).map_err(|e| { - EncodingError::overflow(&format!("count not covert usize [{value}] to u64")) + EncodingError::overflow(&format!( + "count not covert usize [{value}] to u64. Error: [{e}]" + )) })?; encode_u64(value, write_array(&[U64_SIGNIFIER], buffer)?) } @@ -535,17 +523,6 @@ fn encode_buffer<'a>(value: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], write_slice(value, rest) } -fn encode_string_array<'a>( - value: &[String], - buffer: &'a mut [u8], -) -> Result<&'a mut [u8], EncodingError> { - let mut rest = encode_usize_var(&value.len(), buffer)?; - for s in value { - rest = encode_str(s, rest)?; - } - Ok(rest) -} - impl CompactEncodable for u8 { fn encoded_size(&self) -> Result { Ok(1) @@ -707,7 +684,7 @@ impl CompactEncodable for Ipv4Addr { let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not encode {}, not enough room in buffer", - std::any::type_name::() + type_name::() ))); }; dest.copy_from_slice(&self.octets()); @@ -721,7 +698,7 @@ impl CompactEncodable for Ipv4Addr { let Some((dest, rest)) = buffer.split_first_chunk::<4>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not decode {}, buffer not big enough", - std::any::type_name::() + type_name::() ))); }; Ok((Ipv4Addr::from(*dest), rest)) @@ -739,7 +716,7 @@ impl CompactEncodable for Ipv6Addr { let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not encode {}, not enough room in buffer", - std::any::type_name::() + type_name::() ))); }; dest.copy_from_slice(&self.octets()); @@ -753,7 +730,7 @@ impl CompactEncodable for Ipv6Addr { let Some((dest, rest)) = buffer.split_first_chunk::<16>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not decode {}, buffer not big enough", - std::any::type_name::() + type_name::() ))); }; Ok((Ipv6Addr::from(*dest), rest)) @@ -776,7 +753,7 @@ fn decode_vec( ) -> Result<(Vec, &[u8]), EncodingError> { let (len, mut rest) = decode_usize_var(buffer)?; let mut out = Vec::with_capacity(len); - for i in 0..len { + for _ in 0..len { let res = ::decode(rest)?; out.push(res.0); rest = res.1; From 9d0b6bd77567ead899778f8b4112dc9f0a03d629 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 16:02:16 -0400 Subject: [PATCH 26/56] flush out macros and add doctests --- src/encodable.rs | 189 +++++++++++++++++++++++++++++++---------------- 1 file changed, 126 insertions(+), 63 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 75843a8..d28dd2c 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -53,9 +53,74 @@ pub trait CompactEncodable { } } +/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` +pub trait VecEncodable: CompactEncodable { + /// Calculate the resulting size in bytes of `vec` + fn vec_encoded_size(vec: &[Self]) -> Result + where + Self: Sized; + + /// Encode `vec` to `buffer` + fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + encode_vec(vec, buffer) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + where + Self: Sized, + { + decode_vec(buffer) + } +} + +// NB: we DO want &Box<..> because we want the trait to work for boxed things +#[allow(clippy::borrowed_box)] +/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` +pub trait BoxArrayEncodable: CompactEncodable { + /// The encoded size in bytes + fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result + where + Self: Sized; + + /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer + fn encoded_bytes<'a>( + vec: &Box<[Self]>, + buffer: &'a mut [u8], + ) -> Result<&'a mut [u8], EncodingError> + where + Self: Sized, + { + encode_vec(vec, buffer) + } + + /// Decode [`Vec`] from buffer + fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + where + Self: Sized, + { + let (result, rest) = decode_vec(buffer)?; + Ok((result.into_boxed_slice(), rest)) + } +} + #[macro_export] -/// Used for defining CompactEncodable::encoded_size. -/// Pass self and a list of fields to call encoded_size on +/// Given a list of [`CompactEncodable`] things, sum the result of calling +/// [`CompactEncodable::encoded_size`] on all of them. +/// Note this is macro is useful when your arguments have differing types. +/// ``` +/// # use crate::compact_encoding::{sum_preencode, encodable::CompactEncodable}; +/// # use std::net::Ipv4Addr; +/// let foo: Ipv4Addr = "0.0.0.0".parse()?; +/// let bar = 42u64; +/// let qux = "hello?"; +/// let result = sum_preencode!(foo, bar, qux); +/// assert_eq!(result, 12); +/// # Ok::<(), Box>(()) +/// ``` macro_rules! sum_preencode { ($($val:expr),+) => {{ let out: usize = [ @@ -69,7 +134,18 @@ macro_rules! sum_preencode { } #[macro_export] -/// Create a buffer from a list of CompactEncodable things to be used for encoding +/// Given a list of [`CompactEncodable`] things, create a zeroed buffer of the correct size for encoding. +/// Note this is macro is useful when your arguments have differing types. +/// ``` +/// # use crate::compact_encoding::{create_buffer, encodable::CompactEncodable}; +/// # use std::net::Ipv4Addr; +/// let foo: Ipv4Addr = "0.0.0.0".parse()?; +/// let bar = 42u64; +/// let qux = "hello?"; +/// let buff = create_buffer!(foo, bar, qux); +/// assert_eq!(buff.len(), 12); +/// # Ok::<(), Box>(()) +/// ``` macro_rules! create_buffer { ($($val:expr),+) => {{ let len: usize = [ @@ -83,31 +159,72 @@ macro_rules! create_buffer { } #[macro_export] -/// Used for defining CompactEncodable::encoded_bytes. -/// Pass self, the buffer and a list of fields to call encoded_size on +/// Given a buffer and a list of [`CompactEncodable`] things, encode the arguments to the buffer. +/// Note this is macro is useful when your arguments have differing types. +/// ``` +/// # use crate::compact_encoding::{create_buffer, map_encodables, encodable::CompactEncodable}; +/// let num = 42u64; +/// let word = "yo"; +/// let mut buff = create_buffer!(num, word); +/// let result = map_encodables!(&mut buff, num, word); +/// assert!(result.is_empty()); +/// assert_eq!(&buff, &[42, 2, 121, 111]); +/// # Ok::<(), Box>(()) +/// ``` macro_rules! map_encodables { + ($buffer:expr$(,)*) => { + $buffer + }; // Base case: single field - ($buffer:expr, $field:ident) => { + ($buffer:expr, $field:expr) => { $field.encoded_bytes($buffer)? }; // Recursive case: first field + rest - ($buffer:expr, $first:ident, $($rest:ident),+) => {{ + ($buffer:expr, $first:expr, $($rest:expr),+) => {{ let rest = $first.encoded_bytes($buffer)?; map_encodables!(rest, $($rest),+) }}; } #[macro_export] -/// Helper for decoding multiple types into a tuple and returning the remaining buffer. +/// Given a list of [`CompactEncodable`] things, encode the arguments to the buffer. +/// Note this is macro is useful when your arguments have differing types. +/// ``` +/// # use crate::compact_encoding::to_encoded_bytes; +/// let result = to_encoded_bytes!(42u64, "yo"); +/// assert_eq!(&result, &[42, 2, 121, 111]); +/// # Ok::<(), Box>(()) +/// ``` +macro_rules! to_encoded_bytes { + ($($val:expr),*) => {{ + use $crate::{map_encodables, create_buffer, encodable::CompactEncodable}; + let mut buffer = create_buffer!($($val),*); + map_encodables!(&mut buffer, $($val),*); + buffer + }} +} + +#[macro_export] +/// Decode a buffer to the list of types provided, returning the remaining buffer. /// It takes as arguments: `(&buffer, [type1, type2, type3, ...])` /// And returns: `((decoded_type1, decoded_type2, ...), remaining_buffer)` +/// ``` +/// # use crate::compact_encoding::{to_encoded_bytes, map_decode}; +/// let buffer = to_encoded_bytes!(42u64, "yo"); +/// let ((number, word), remaining_buffer) = map_decode!(&buffer, [u64, String]); +/// assert!(remaining_buffer.is_empty()); +/// assert_eq!(number, 42u64); +/// assert_eq!(word, "yo"); +/// # Ok::<(), Box>(()) +/// ``` macro_rules! map_decode { ($buffer:expr, [ $($field_type:ty),* $(,)? ]) => {{ + use $crate::encodable::CompactEncodable; let mut current_buffer: &[u8] = $buffer; - // Decode each type into the `result_tuple` + // Decode each type into `result_tuple` let result_tuple = ( $( match <$field_type>::decode(¤t_buffer)? { @@ -122,60 +239,6 @@ macro_rules! map_decode { }}; } -/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` -pub trait VecEncodable: CompactEncodable { - /// Calculate the resulting size in bytes of `vec` - fn vec_encoded_size(vec: &[Self]) -> Result - where - Self: Sized; - - /// Encode `vec` to `buffer` - fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> - where - Self: Sized, - { - encode_vec(vec, buffer) - } - - /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> - where - Self: Sized, - { - decode_vec(buffer) - } -} - -// NB: we DO want &Box<..> because we want the trait to work for boxed things -#[allow(clippy::borrowed_box)] -/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` -pub trait BoxArrayEncodable: CompactEncodable { - /// The encoded size in bytes - fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result - where - Self: Sized; - - /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer - fn encoded_bytes<'a>( - vec: &Box<[Self]>, - buffer: &'a mut [u8], - ) -> Result<&'a mut [u8], EncodingError> - where - Self: Sized, - { - encode_vec(vec, buffer) - } - - /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> - where - Self: Sized, - { - let (result, rest) = decode_vec(buffer)?; - Ok((result.into_boxed_slice(), rest)) - } -} - /// helper for mapping the first element of a two eleent tuple macro_rules! map_first { ($res:expr, $f:expr) => {{ From 2f530787686ff03791c323d92ad3cb6e698f2d69 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 16:34:54 -0400 Subject: [PATCH 27/56] better docs --- src/encodable.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index d28dd2c..02f2548 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -12,14 +12,13 @@ use crate::{ const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; -/// Implement for types to get encoding and decoding. +/// Implement for a type to get encoding and decoding. pub trait CompactEncodable { - /// The size required in the buffer for this time + /// The size in bytes required to encode `self`. fn encoded_size(&self) -> Result; - /// The bytes resulting from encoding this type - // TODO add buffer argument. Change result to return remaining buffer + /// Encode `self` into `buffer` returning the remainder of `buffer`. fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; - /// Decode a value from the buffer. Returns the value and remaining undecoded bytes + /// Decode a value of type [`Decode`] from `buffer`. Returns the decoded value and remaining undecoded bytes. fn decode(buffer: &[u8]) -> Result<(Decode, &[u8]), EncodingError> where Decode: Sized; From c7bf377815b8b24afeab141009d58e6752e4e6ce Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 17:00:25 -0400 Subject: [PATCH 28/56] add box_ & vec_ prefix to VecEnc and BoxEnc trait methods --- src/encodable.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 02f2548..17c22aa 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -60,7 +60,7 @@ pub trait VecEncodable: CompactEncodable { Self: Sized; /// Encode `vec` to `buffer` - fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + fn vec_encode<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> where Self: Sized, { @@ -68,7 +68,7 @@ pub trait VecEncodable: CompactEncodable { } /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + fn vec_decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> where Self: Sized, { @@ -85,8 +85,8 @@ pub trait BoxArrayEncodable: CompactEncodable { where Self: Sized; - /// Encode `Box<[T]>` to the buffer and return the remainder of the buffer - fn encoded_bytes<'a>( + /// Encode `Box<[Self]>` to the buffer and return the remainder of the buffer + fn box_encode<'a>( vec: &Box<[Self]>, buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> @@ -96,8 +96,8 @@ pub trait BoxArrayEncodable: CompactEncodable { encode_vec(vec, buffer) } - /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + /// Decode [`Box<[Self]>`] from buffer + fn box_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> where Self: Sized, { @@ -829,14 +829,14 @@ impl CompactEncodable for Vec { } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - ::encoded_bytes(self, buffer) + ::vec_encode(self, buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, { - ::decode(buffer) + ::vec_decode(buffer) } } @@ -848,7 +848,7 @@ impl VecEncodable for u32 { Ok(usize_encoded_size(vec.len()) + (vec.len() * 4)) } /// Encode `vec` to `buffer` - fn encoded_bytes<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> + fn vec_encode<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> where Self: Sized, { @@ -860,7 +860,7 @@ impl VecEncodable for u32 { } /// Decode [`Vec`] from buffer - fn decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> + fn vec_decode(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> where Self: Sized, { @@ -893,7 +893,7 @@ impl BoxArrayEncodable for u8 { Ok(usize_encoded_size(boxed.len()) + boxed.len()) } - fn encoded_bytes<'a>( + fn box_encode<'a>( boxed: &Box<[Self]>, buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> @@ -904,7 +904,7 @@ impl BoxArrayEncodable for u8 { write_slice(boxed, rest) } - fn decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + fn box_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> where Self: Sized, { @@ -920,14 +920,14 @@ impl CompactEncodable for Box<[T]> { } fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - ::encoded_bytes(self, buffer) + ::box_encode(self, buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, { - ::decode(buffer) + ::box_decode(buffer) } } From 9431df909913dc66b6e1e0dde5039f00779adc39 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 17:15:20 -0400 Subject: [PATCH 29/56] Rename encoded_bytes to encode --- src/encodable.rs | 51 +++++++++++++++++++++++------------------------- src/types.rs | 2 +- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 17c22aa..acbfa1d 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -17,7 +17,7 @@ pub trait CompactEncodable { /// The size in bytes required to encode `self`. fn encoded_size(&self) -> Result; /// Encode `self` into `buffer` returning the remainder of `buffer`. - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; /// Decode a value of type [`Decode`] from `buffer`. Returns the decoded value and remaining undecoded bytes. fn decode(buffer: &[u8]) -> Result<(Decode, &[u8]), EncodingError> where @@ -30,12 +30,12 @@ pub trait CompactEncodable { /// # use compact_encoding::encodable::CompactEncodable; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let mut buff = vec![0; foo.encoded_size()?]; - /// foo.encoded_bytes(&mut buff)?; + /// foo.encode(&mut buff)?; /// # Ok::<(), Box>(()) /// ``` fn to_bytes(&self) -> Result, EncodingError> { let mut buff = vec![0; self.encoded_size()?]; - self.encoded_bytes(&mut buff)?; + self.encode(&mut buff)?; Ok(buff) } /// Create an empty buffer of the correct size for encoding `self` to. This is just a helper @@ -176,11 +176,11 @@ macro_rules! map_encodables { }; // Base case: single field ($buffer:expr, $field:expr) => { - $field.encoded_bytes($buffer)? + $field.encode($buffer)? }; // Recursive case: first field + rest ($buffer:expr, $first:expr, $($rest:expr),+) => {{ - let rest = $first.encoded_bytes($buffer)?; + let rest = $first.encode($buffer)?; map_encodables!(rest, $($rest),+) }}; } @@ -311,7 +311,7 @@ pub fn map_encode<'a>( ) -> Result<&'a mut [u8], EncodingError> { let mut rest = buffer; for x in arr { - rest = x.encoded_bytes(rest)?; + rest = x.encode(rest)?; } Ok(rest) } @@ -358,6 +358,9 @@ pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), E )) }) } + +/// Split a mutable slice into two mutable slices around `mid`. +/// Returns encoding error when `mid` is out of bounds. fn get_slices_mut_checked( buffer: &mut [u8], mid: usize, @@ -458,7 +461,7 @@ impl CompactEncodable for [u8; N] { Ok(N) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { write_array(self, buffer) } @@ -590,7 +593,7 @@ impl CompactEncodable for u8 { Ok(1) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { write_array(&[*self], buffer) } @@ -608,7 +611,7 @@ impl CompactEncodable for u16 { Ok(U16_SIZE) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { encode_u16(*self, buffer) } @@ -626,7 +629,7 @@ impl CompactEncodable for u32 { Ok(usize_encoded_size(*self as usize)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { encode_usize_var(&(*self as usize), buffer) } @@ -642,7 +645,7 @@ impl CompactEncodable for u64 { Ok(u64_var_encoded_size(*self)) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { u64_encoded_bytes(*self, buffer) } @@ -659,7 +662,7 @@ impl CompactEncodable for String { encoded_size_str(self) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { encode_str(self, buffer) } @@ -676,7 +679,7 @@ impl CompactEncodable for str { encoded_size_str(self) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { encode_str(self, buffer) } @@ -694,10 +697,10 @@ impl CompactEncodable for Vec { Ok(out) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { let mut rest = usize_encoded_bytes(self.len(), buffer)?; for s in self { - rest = s.encoded_bytes(rest)?; + rest = s.encode(rest)?; } Ok(rest) } @@ -722,7 +725,7 @@ impl CompactEncodable for Vec { Ok(usize_encoded_size(self.len()) + self.len()) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { encode_buffer(self, buffer) } @@ -739,10 +742,7 @@ impl CompactEncodable for Ipv4Addr { Ok(U32_SIZE) } - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> std::result::Result<&'a mut [u8], EncodingError> { let Some((dest, rest)) = buffer.split_first_chunk_mut::<4>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not encode {}, not enough room in buffer", @@ -771,10 +771,7 @@ impl CompactEncodable for Ipv6Addr { Ok(4) } - fn encoded_bytes<'a>( - &self, - buffer: &'a mut [u8], - ) -> std::result::Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> std::result::Result<&'a mut [u8], EncodingError> { let Some((dest, rest)) = buffer.split_first_chunk_mut::<16>() else { return Err(EncodingError::out_of_bounds(&format!( "Colud not encode {}, not enough room in buffer", @@ -805,7 +802,7 @@ fn encode_vec<'a, T: CompactEncodable + Sized>( ) -> Result<&'a mut [u8], EncodingError> { let mut rest = encode_usize_var(&vec.len(), buffer)?; for x in vec { - rest = ::encoded_bytes(x, rest)?; + rest = ::encode(x, rest)?; } Ok(rest) } @@ -828,7 +825,7 @@ impl CompactEncodable for Vec { T::vec_encoded_size(self) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { ::vec_encode(self, buffer) } @@ -919,7 +916,7 @@ impl CompactEncodable for Box<[T]> { T::boxed_array_encoded_size(self) } - fn encoded_bytes<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { ::box_encode(self, buffer) } diff --git a/src/types.rs b/src/types.rs index 1c8caac..46fb400 100644 --- a/src/types.rs +++ b/src/types.rs @@ -818,7 +818,7 @@ impl State { buffer: &mut [u8], ) -> Result { let start_len = buffer.len(); - let rest = value.encoded_bytes(buffer)?; + let rest = value.encode(buffer)?; let offset = start_len - rest.len(); self.add_start(offset) } From 52185b2d9357778bea971f8717ea50f940f5509f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 14 Apr 2025 23:53:39 -0400 Subject: [PATCH 30/56] rename CompactEncodable to CompacEncoding --- src/encodable.rs | 100 +++++++++++++++++++++------------------------ src/types.rs | 8 ++-- tests/encodable.rs | 2 +- 3 files changed, 52 insertions(+), 58 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index acbfa1d..709650f 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -13,7 +13,7 @@ const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; /// Implement for a type to get encoding and decoding. -pub trait CompactEncodable { +pub trait CompactEncoding { /// The size in bytes required to encode `self`. fn encoded_size(&self) -> Result; /// Encode `self` into `buffer` returning the remainder of `buffer`. @@ -27,7 +27,7 @@ pub trait CompactEncodable { /// encoding to it in one step. /// ``` /// # use std::net::Ipv4Addr; - /// # use compact_encoding::encodable::CompactEncodable; + /// # use compact_encoding::encodable::CompactEncoding; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let mut buff = vec![0; foo.encoded_size()?]; /// foo.encode(&mut buff)?; @@ -42,7 +42,7 @@ pub trait CompactEncodable { /// method for: encoding to it in one step. /// ``` /// # use std::net::Ipv4Addr; - /// # use compact_encoding::encodable::CompactEncodable; + /// # use compact_encoding::encodable::CompactEncoding; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// vec![0; foo.encoded_size()?]; /// # Ok::<(), Box>(()) @@ -52,8 +52,8 @@ pub trait CompactEncodable { } } -/// Implement this for type `T` to have `CompactEncodable` implemented for `Vec` -pub trait VecEncodable: CompactEncodable { +/// Implement this for type `T` to have `CompactEncoding` implemented for `Vec` +pub trait VecEncodable: CompactEncoding { /// Calculate the resulting size in bytes of `vec` fn vec_encoded_size(vec: &[Self]) -> Result where @@ -78,8 +78,8 @@ pub trait VecEncodable: CompactEncodable { // NB: we DO want &Box<..> because we want the trait to work for boxed things #[allow(clippy::borrowed_box)] -/// Define this trait for `T` to get `Box<[T]>: CompactEncodable` -pub trait BoxArrayEncodable: CompactEncodable { +/// Define this trait for `T` to get `Box<[T]>: CompactEncoding` +pub trait BoxArrayEncodable: CompactEncoding { /// The encoded size in bytes fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result where @@ -107,11 +107,11 @@ pub trait BoxArrayEncodable: CompactEncodable { } #[macro_export] -/// Given a list of [`CompactEncodable`] things, sum the result of calling -/// [`CompactEncodable::encoded_size`] on all of them. +/// Given a list of [`CompactEncoding`] things, sum the result of calling +/// [`CompactEncoding::encoded_size`] on all of them. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{sum_preencode, encodable::CompactEncodable}; +/// # use crate::compact_encoding::{sum_preencode, encodable::CompactEncoding}; /// # use std::net::Ipv4Addr; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let bar = 42u64; @@ -133,10 +133,10 @@ macro_rules! sum_preencode { } #[macro_export] -/// Given a list of [`CompactEncodable`] things, create a zeroed buffer of the correct size for encoding. +/// Given a list of [`CompactEncoding`] things, create a zeroed buffer of the correct size for encoding. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{create_buffer, encodable::CompactEncodable}; +/// # use crate::compact_encoding::{create_buffer, encodable::CompactEncoding}; /// # use std::net::Ipv4Addr; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let bar = 42u64; @@ -158,10 +158,10 @@ macro_rules! create_buffer { } #[macro_export] -/// Given a buffer and a list of [`CompactEncodable`] things, encode the arguments to the buffer. +/// Given a buffer and a list of [`CompactEncoding`] things, encode the arguments to the buffer. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{create_buffer, map_encodables, encodable::CompactEncodable}; +/// # use crate::compact_encoding::{create_buffer, map_encodables, encodable::CompactEncoding}; /// let num = 42u64; /// let word = "yo"; /// let mut buff = create_buffer!(num, word); @@ -186,7 +186,7 @@ macro_rules! map_encodables { } #[macro_export] -/// Given a list of [`CompactEncodable`] things, encode the arguments to the buffer. +/// Given a list of [`CompactEncoding`] things, encode the arguments to the buffer. /// Note this is macro is useful when your arguments have differing types. /// ``` /// # use crate::compact_encoding::to_encoded_bytes; @@ -196,7 +196,7 @@ macro_rules! map_encodables { /// ``` macro_rules! to_encoded_bytes { ($($val:expr),*) => {{ - use $crate::{map_encodables, create_buffer, encodable::CompactEncodable}; + use $crate::{map_encodables, create_buffer, encodable::CompactEncoding}; let mut buffer = create_buffer!($($val),*); map_encodables!(&mut buffer, $($val),*); buffer @@ -220,7 +220,7 @@ macro_rules! map_decode { ($buffer:expr, [ $($field_type:ty),* $(,)? ]) => {{ - use $crate::encodable::CompactEncodable; + use $crate::encodable::CompactEncoding; let mut current_buffer: &[u8] = $buffer; // Decode each type into `result_tuple` @@ -247,7 +247,7 @@ macro_rules! map_first { }}; } -/// The number of bytes required to encode this number +/// The number of bytes required to encode this number. Note this is always variable width. pub fn usize_encoded_size(val: usize) -> usize { if val < U16_SIGNIFIER.into() { 1 @@ -262,7 +262,7 @@ pub fn usize_encoded_size(val: usize) -> usize { /// The number of bytes required to encode this number. /// We only need this for u64 because all other uints can be converted to usize reliably. -pub fn u64_var_encoded_size(val: u64) -> usize { +pub fn encoded_size_var_u64(val: u64) -> usize { if val < U16_SIGNIFIER.into() { 1 } else if val <= 0xffff { @@ -275,7 +275,7 @@ pub fn u64_var_encoded_size(val: u64) -> usize { } /// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. -pub fn u64_encoded_bytes(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { +pub fn encoded_bytes_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { if uint < U16_SIGNIFIER.into() { encode_u8(uint as u8, buffer) } else if uint <= 0xffff { @@ -290,8 +290,9 @@ pub fn u64_encoded_bytes(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], Enco } } +// TODO RMME - redundant with sum_encoded_size /// Sum encoded sizes -pub fn encoded_size(arr: &[impl CompactEncodable]) -> Result { +pub fn encoded_size(arr: &[impl CompactEncoding]) -> Result { let mut out = 0; for x in arr { out += x.encoded_size()?; @@ -299,14 +300,14 @@ pub fn encoded_size(arr: &[impl CompactEncodable]) -> Result Result, EncodingError> { +/// Create a buffer from an array of CompactEncoding objects +pub fn create_buffer(arr: &[impl CompactEncoding]) -> Result, EncodingError> { Ok(vec![0; encoded_size(arr)?]) } -/// Encode an array of CompactEncodable objects +/// Encode an array of CompactEncoding objects pub fn map_encode<'a>( - arr: &[impl CompactEncodable], + arr: &[impl CompactEncoding], buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> { let mut rest = buffer; @@ -316,13 +317,8 @@ pub fn map_encode<'a>( Ok(rest) } -/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. -pub fn usize_encoded_bytes(uint: usize, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - u64_encoded_bytes(uint as u64, buffer) -} - /// decode a `usize` from `buffer` and return the remaining bytes -pub fn usize_decode(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { +pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { let [first, rest @ ..] = buffer else { return Err(EncodingError::out_of_bounds( "Colud not decode usize, empty buffer", @@ -456,7 +452,7 @@ pub fn decode_bytes_fixed( //write_array(value, buffer) } -impl CompactEncodable for [u8; N] { +impl CompactEncoding for [u8; N] { fn encoded_size(&self) -> Result { Ok(N) } @@ -588,7 +584,7 @@ fn encode_buffer<'a>(value: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], write_slice(value, rest) } -impl CompactEncodable for u8 { +impl CompactEncoding for u8 { fn encoded_size(&self) -> Result { Ok(1) } @@ -606,7 +602,7 @@ impl CompactEncodable for u8 { } } -impl CompactEncodable for u16 { +impl CompactEncoding for u16 { fn encoded_size(&self) -> Result { Ok(U16_SIZE) } @@ -624,7 +620,7 @@ impl CompactEncodable for u16 { } // NB: we want u32 encoded and decoded as variable sized uint -impl CompactEncodable for u32 { +impl CompactEncoding for u32 { fn encoded_size(&self) -> Result { Ok(usize_encoded_size(*self as usize)) } @@ -640,13 +636,13 @@ impl CompactEncodable for u32 { decode_u32_var(buffer) } } -impl CompactEncodable for u64 { +impl CompactEncoding for u64 { fn encoded_size(&self) -> Result { - Ok(u64_var_encoded_size(*self)) + Ok(encoded_size_var_u64(*self)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - u64_encoded_bytes(*self, buffer) + encoded_bytes_var_u64(*self, buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -657,7 +653,7 @@ impl CompactEncodable for u64 { } } -impl CompactEncodable for String { +impl CompactEncoding for String { fn encoded_size(&self) -> Result { encoded_size_str(self) } @@ -674,7 +670,7 @@ impl CompactEncodable for String { } } -impl CompactEncodable for str { +impl CompactEncoding for str { fn encoded_size(&self) -> Result { encoded_size_str(self) } @@ -688,7 +684,7 @@ impl CompactEncodable for str { } } -impl CompactEncodable for Vec { +impl CompactEncoding for Vec { fn encoded_size(&self) -> Result { let mut out = usize_encoded_size(self.len()); for s in self { @@ -698,7 +694,7 @@ impl CompactEncodable for Vec { } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - let mut rest = usize_encoded_bytes(self.len(), buffer)?; + let mut rest = encode_usize_var(&self.len(), buffer)?; for s in self { rest = s.encode(rest)?; } @@ -720,7 +716,7 @@ impl CompactEncodable for Vec { } } -impl CompactEncodable for Vec { +impl CompactEncoding for Vec { fn encoded_size(&self) -> Result { Ok(usize_encoded_size(self.len()) + self.len()) } @@ -737,7 +733,7 @@ impl CompactEncodable for Vec { } } -impl CompactEncodable for Ipv4Addr { +impl CompactEncoding for Ipv4Addr { fn encoded_size(&self) -> std::result::Result { Ok(U32_SIZE) } @@ -766,7 +762,7 @@ impl CompactEncodable for Ipv4Addr { Ok((Ipv4Addr::from(*dest), rest)) } } -impl CompactEncodable for Ipv6Addr { +impl CompactEncoding for Ipv6Addr { fn encoded_size(&self) -> std::result::Result { Ok(4) } @@ -796,31 +792,29 @@ impl CompactEncodable for Ipv6Addr { } } -fn encode_vec<'a, T: CompactEncodable + Sized>( +fn encode_vec<'a, T: CompactEncoding + Sized>( vec: &[T], buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> { let mut rest = encode_usize_var(&vec.len(), buffer)?; for x in vec { - rest = ::encode(x, rest)?; + rest = ::encode(x, rest)?; } Ok(rest) } -fn decode_vec( - buffer: &[u8], -) -> Result<(Vec, &[u8]), EncodingError> { +fn decode_vec(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { let (len, mut rest) = decode_usize_var(buffer)?; let mut out = Vec::with_capacity(len); for _ in 0..len { - let res = ::decode(rest)?; + let res = ::decode(rest)?; out.push(res.0); rest = res.1; } Ok((out, rest)) } -impl CompactEncodable for Vec { +impl CompactEncoding for Vec { fn encoded_size(&self) -> Result { T::vec_encoded_size(self) } @@ -911,7 +905,7 @@ impl BoxArrayEncodable for u8 { } } -impl CompactEncodable for Box<[T]> { +impl CompactEncoding for Box<[T]> { fn encoded_size(&self) -> Result { T::boxed_array_encoded_size(self) } diff --git a/src/types.rs b/src/types.rs index 46fb400..b7f3ab7 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,5 +1,5 @@ //! Basic types of compact_encoding. -use crate::encodable::CompactEncodable; +use crate::encodable::CompactEncoding as Cenc; use std::convert::TryFrom; use std::fmt; use std::ops::Range; @@ -805,14 +805,14 @@ impl State { } /// Like [`State::preencode`] but for `CompactEncodable` types - pub fn preencode_t(&mut self, value: &T) -> Result { + pub fn preencode_t(&mut self, value: &T) -> Result { let size = value.encoded_size()?; let out = self.add_end(size)?; Ok(out) } /// Like [`State::encode`] but for `CompactEncodable` types - pub fn encode_t( + pub fn encode_t( &mut self, value: &T, buffer: &mut [u8], @@ -824,7 +824,7 @@ impl State { } /// Like [`State::decode`] but for `CompactEncodable` types - pub fn decode_t(&mut self, buffer: &[u8]) -> Result { + pub fn decode_t(&mut self, buffer: &[u8]) -> Result { let start_len = buffer.len(); let (result, remaining_buffer) = T::decode(buffer)?; let after = remaining_buffer.len(); diff --git a/tests/encodable.rs b/tests/encodable.rs index 2b73d41..bd4906c 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,6 +1,6 @@ use compact_encoding::{ create_buffer, - encodable::{create_buffer, CompactEncodable}, + encodable::{create_buffer, CompactEncoding}, map_decode, map_encodables, types::EncodingError, }; From ab43259d9486c856fe3f4dd46b19cc3a7d52e28e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 01:53:14 -0400 Subject: [PATCH 31/56] renames --- src/encodable.rs | 47 ++++++++++------------------------------------ tests/encodable.rs | 36 ++++++++++++++++------------------- 2 files changed, 26 insertions(+), 57 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 709650f..4fafae1 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -111,16 +111,16 @@ pub trait BoxArrayEncodable: CompactEncoding { /// [`CompactEncoding::encoded_size`] on all of them. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{sum_preencode, encodable::CompactEncoding}; +/// # use crate::compact_encoding::{sum_encoded_size, encodable::CompactEncoding}; /// # use std::net::Ipv4Addr; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let bar = 42u64; /// let qux = "hello?"; -/// let result = sum_preencode!(foo, bar, qux); +/// let result = sum_encoded_size!(foo, bar, qux); /// assert_eq!(result, 12); /// # Ok::<(), Box>(()) /// ``` -macro_rules! sum_preencode { +macro_rules! sum_encoded_size { ($($val:expr),+) => {{ let out: usize = [ $( @@ -161,16 +161,16 @@ macro_rules! create_buffer { /// Given a buffer and a list of [`CompactEncoding`] things, encode the arguments to the buffer. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{create_buffer, map_encodables, encodable::CompactEncoding}; +/// # use crate::compact_encoding::{create_buffer, map_encode, encodable::CompactEncoding}; /// let num = 42u64; /// let word = "yo"; /// let mut buff = create_buffer!(num, word); -/// let result = map_encodables!(&mut buff, num, word); +/// let result = map_encode!(&mut buff, num, word); /// assert!(result.is_empty()); -/// assert_eq!(&buff, &[42, 2, 121, 111]); +/// assert_eq!(&buff, &[42, 2, b'y', b'o']); /// # Ok::<(), Box>(()) /// ``` -macro_rules! map_encodables { +macro_rules! map_encode { ($buffer:expr$(,)*) => { $buffer }; @@ -181,7 +181,7 @@ macro_rules! map_encodables { // Recursive case: first field + rest ($buffer:expr, $first:expr, $($rest:expr),+) => {{ let rest = $first.encode($buffer)?; - map_encodables!(rest, $($rest),+) + map_encode!(rest, $($rest),+) }}; } @@ -196,9 +196,9 @@ macro_rules! map_encodables { /// ``` macro_rules! to_encoded_bytes { ($($val:expr),*) => {{ - use $crate::{map_encodables, create_buffer, encodable::CompactEncoding}; + use $crate::{map_encode, create_buffer, encodable::CompactEncoding}; let mut buffer = create_buffer!($($val),*); - map_encodables!(&mut buffer, $($val),*); + map_encode!(&mut buffer, $($val),*); buffer }} } @@ -290,33 +290,6 @@ pub fn encoded_bytes_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], } } -// TODO RMME - redundant with sum_encoded_size -/// Sum encoded sizes -pub fn encoded_size(arr: &[impl CompactEncoding]) -> Result { - let mut out = 0; - for x in arr { - out += x.encoded_size()?; - } - Ok(out) -} - -/// Create a buffer from an array of CompactEncoding objects -pub fn create_buffer(arr: &[impl CompactEncoding]) -> Result, EncodingError> { - Ok(vec![0; encoded_size(arr)?]) -} - -/// Encode an array of CompactEncoding objects -pub fn map_encode<'a>( - arr: &[impl CompactEncoding], - buffer: &'a mut [u8], -) -> Result<&'a mut [u8], EncodingError> { - let mut rest = buffer; - for x in arr { - rest = x.encode(rest)?; - } - Ok(rest) -} - /// decode a `usize` from `buffer` and return the remaining bytes pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { let [first, rest @ ..] = buffer else { diff --git a/tests/encodable.rs b/tests/encodable.rs index bd4906c..eb9637a 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,8 +1,5 @@ use compact_encoding::{ - create_buffer, - encodable::{create_buffer, CompactEncoding}, - map_decode, map_encodables, - types::EncodingError, + create_buffer, encodable::CompactEncoding, map_decode, map_encode, types::EncodingError, }; // The max value for 1 byte length is 252 @@ -21,7 +18,7 @@ fn cenc_basic() -> Result<(), EncodingError> { let mut buff = create_buffer!(str_value_1, str_value_2, u32_value_3, u32_value_4); assert_eq!(buff.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); - let rest = map_encodables!( + let rest = map_encode!( &mut buff, str_value_1, str_value_2, @@ -46,7 +43,7 @@ fn cenc_string_long() -> Result<(), EncodingError> { let mut buffer = create_buffer!(value); assert_eq!(buffer.len(), 1 + 2 + 253); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [String]); @@ -61,7 +58,7 @@ fn cenc_u32_as_u16() -> Result<(), EncodingError> { let mut buffer = create_buffer!(value); // 1 byte for u16 signifier, 2 bytes for length assert_eq!(buffer.len(), 1 + 2); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [u32]); @@ -76,7 +73,7 @@ fn cenc_u32_as_u8() -> Result<(), EncodingError> { let mut buffer = create_buffer!(value); // 1 byte for data assert_eq!(buffer.len(), 1); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [u32]); @@ -91,7 +88,7 @@ fn cenc_u64() -> Result<(), EncodingError> { let mut buffer = create_buffer!(value); // 1 byte for u64 signifier, 8 bytes for length assert_eq!(buffer.len(), 1 + 8); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [u64]); assert!(rest.is_empty()); @@ -105,7 +102,7 @@ fn cenc_u64_as_u32() -> Result<(), EncodingError> { let mut buffer = create_buffer!(value); // 1 byte for u32 signifier, 4 bytes for length assert_eq!(buffer.len(), 1 + 4); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [u64]); assert!(rest.is_empty()); @@ -122,7 +119,7 @@ fn cenc_buffer() -> Result<(), EncodingError> { // 1 byte for length, 2 bytes for data // 1 byte for length, 3 bytes for data assert_eq!(buffer.len(), 1 + 2 + 1 + 3); - let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2); + let rest = map_encode!(&mut buffer, buf_value_1, buf_value_2); assert!(rest.is_empty()); let (result, rest) = map_decode!(&buffer, [Box<[u8]>, Box<[u8]>]); assert!(rest.is_empty()); @@ -141,7 +138,7 @@ fn cenc_vec() -> Result<(), EncodingError> { // 1 byte for length, 4*3 bytes for data assert_eq!(buffer.len(), 1 + 2 + 1 + 12); - let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2); + let rest = map_encode!(&mut buffer, buf_value_1, buf_value_2); assert!(rest.is_empty()); let (result, rest) = map_decode!(&buffer, [Vec, Vec]); assert!(rest.is_empty()); @@ -158,7 +155,7 @@ fn cenc_string_array() -> Result<(), EncodingError> { // 1 byte for string length, 5 bytes for string, // 1 byte for string length, 6 bytes for string assert_eq!(buffer.len(), 1 + 1 + 5 + 1 + 6); - let rest = map_encodables!(&mut buffer, value); + let rest = map_encode!(&mut buffer, value); assert!(rest.is_empty()); let ((result,), rest) = map_decode!(&buffer, [Vec]); assert!(rest.is_empty()); @@ -177,7 +174,7 @@ fn cenc_fixed_and_raw() -> Result<(), EncodingError> { // 1 byte for length, 4*3 bytes for data assert_eq!(buffer.len(), 16 + 32 + 3); - let rest = map_encodables!(&mut buffer, buf_value_1, buf_value_2, buf_value_3); + let rest = map_encode!(&mut buffer, buf_value_1, buf_value_2, buf_value_3); assert!(rest.is_empty()); let (result, rest) = map_decode!(&buffer, [[u8; 16], [u8; 32], [u8; 3]]); assert!(rest.is_empty()); @@ -192,19 +189,18 @@ fn cenc_32_byte_array() -> Result<(), EncodingError> { let empty_array: Vec<[u8; 32]> = vec![]; let one_array: Vec<[u8; 32]> = vec![[0; 32]]; let many_array: Vec<[u8; 32]> = vec![[1; 32], [2; 32], [3; 32]]; - let data = vec![empty_array.clone(), one_array.clone(), many_array.clone()]; let expected_size = 1 + 1 + 32 + 1 + (3 * 32); - let mut buffer = create_buffer(&data)?; + let mut buffer = create_buffer!(empty_array, one_array, many_array); assert_eq!(buffer.len(), expected_size); - let rest = map_encodables!(&mut buffer, empty_array, one_array, many_array); + let rest = map_encode!(&mut buffer, empty_array, one_array, many_array); assert!(rest.is_empty()); let (result, rest) = map_decode!(&buffer, [Vec<[u8; 32]>, Vec<[u8; 32]>, Vec<[u8;32]>]); assert!(rest.is_empty()); - assert_eq!(result.0, data[0]); - assert_eq!(result.1, data[1]); - assert_eq!(result.2, data[2]); + assert_eq!(result.0, empty_array); + assert_eq!(result.1, one_array); + assert_eq!(result.2, many_array); Ok(()) } From 5171bb8f943f5440f3a1b0d451aca0cbb88bab14 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 01:53:19 -0400 Subject: [PATCH 32/56] rewrite module documentation --- src/encodable.rs | 165 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 3 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 4fafae1..62cdaf2 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -1,4 +1,159 @@ -//! CompactEncodable stuff +#![forbid(unsafe_code, missing_docs)] +#![cfg_attr(test, deny(warnings))] + +//! # Series of compact encoding schemes for building small and fast parsers and serializers +//! +//! Binary compatible with the +//! [original Javascript compact-encoding library](https://github.com/compact-encoding/compact-encoding/). +//! +//! ## Usage +//! +//! ### Quick start +//! ``` +//! use compact_encoding::{to_encoded_bytes, map_decode}; +//! +//! let number = 41_u32; +//! let word = "hi"; +//! +//! let encoded_buffer = to_encoded_bytes!(number, word); +//! let ((number_dec, word_dec), remaining_buffer) = map_decode!(&encoded_buffer, [u32, String]); +//! +//! assert!(remaining_buffer.is_empty()); +//! assert_eq!(number_dec, number); +//! assert_eq!(word_dec, word); +//! # Ok::<(), Box>(()) +//! ``` +//! ### Manually implement encoding and decoding +//! +//! This example shows how to manually calculate the needed buffer size, create the buffer, encode +//! data, and decode it. Using only the types which include a default [`CompactEncoding`] +//! implementation. A more ergonomic pattern is demonstrated in other examples +//! ``` +//! use compact_encoding::encodable::CompactEncoding; +//! +//! let number = 41_u32; +//! let word = "hi"; +//! +//! // Use `encoded_size` to figure out how big a buffer should be. +//! let size = number.encoded_size()? + word.encoded_size()?; +//! +//! // Create a buffer with the calculated size +//! let mut buffer = vec![0; size]; +//! assert_eq!(buffer.len(), 1 + 1 + 2); +//! +//! // Then actually encode the values +//! let mut remaining_buffer = number.encode(&mut buffer)?; +//! remaining_buffer = word.encode(remaining_buffer)?; +//! assert!(remaining_buffer.is_empty()); +//! assert_eq!(buffer.to_vec(), vec![41_u8, 2_u8, b'h', b'i']); +//! +//! // `buffer` now contains all the encoded data, and we can decode from it +//! let (number_dec, remaining_buffer) = u32::decode(&buffer)?; +//! let (word_dec, remaining_buffer) = String::decode(remaining_buffer)?; +//! assert!(remaining_buffer.is_empty()); +//! assert_eq!(number_dec, number); +//! assert_eq!(word_dec, word); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! ### Implementing CompactEncoding for new types +//! +//! You can implement [`CompactEncoding`]` for your own structs like below: +//! ``` +//! use compact_encoding::{ +//! to_encoded_bytes, map_encode, map_decode, sum_encoded_size, +//! {encodable::CompactEncoding, EncodingError} +//! }; +//! +//! #[derive(Debug, PartialEq)] +//! struct MyStruct { +//! some_flag: bool, +//! values: Option>, +//! other: String, +//! stuff: u64, +//! } +//! +//! impl CompactEncoding for MyStruct { +//! fn encoded_size(&self) -> Result { +//! Ok(1 /* flags */ + { +//! /* handle option values */ +//! if let Some(values) = &self.values { +//! values.encoded_size()? +//! } else { +//! 0 +//! } +//! } + dbg!(sum_encoded_size!(&self.other, &self.stuff))) +//! } +//! +//! fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { +//! let mut flags: u8 = 0; +//! if self.some_flag { +//! flags |= 1; +//! } +//! if self.values.is_some() { +//! flags |= 2; +//! } +//! let mut rest = flags.encode(buffer)?; +//! if let Some(values) = &self.values { +//! rest = values.encode(rest)?; +//! } +//! Ok(map_encode!(rest, self.other, self.stuff)) +//! } +//! +//! fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> { +//! let (flags, rest) = u8::decode(buffer)?; +//! let some_flag: bool = flags & 1 != 0; +//! let (values, rest) = if flags & 2 != 0 { +//! let (vec, rest) = >::decode(rest)?; +//! (Some(vec), rest) +//! } else { +//! (None, rest) +//! }; +//! let ((other, stuff), rest) = map_decode!(rest, [String, u64]); +//! Ok((Self { +//! some_flag, +//! values, +//! other, +//! stuff +//! }, rest)) +//! } +//! } +//! +//! // Test values +//! let foo = MyStruct { +//! some_flag: false, +//! values: None, +//! other: "hi".to_string(), +//! stuff: 42, +//! }; +//! +//! let bar = MyStruct { +//! some_flag: true, +//! values: Some(vec![[1; 32], [2; 32]]), +//! other: "yo".to_string(), +//! stuff: 0 +//! }; +//! +//! // Encode `foo` and `bar` to a buffer +//! let buffer = to_encoded_bytes!(&foo, &bar); +//! +//! // With the above use of a flags byte, the empty value encodes to only one byte +//! assert_eq!(buffer.len(), +//! // flags + string + u64 +//! (1 + 3 + 1) + +//! // "" + (values.len().encoded_size() + (values.len() * <[u8;32]>::encoded_size()) + "" +//! (1 + (1 + (2 * 32)) + 3 + 1) +//! ); +//! +//! // And decode directly to your own struct +//! let (foo_dec, rest) = MyStruct::decode(&buffer)?; +//! let (bar_dec, rest) = MyStruct::decode(&rest)?; +//! // Ensure all bytes were used +//! assert!(rest.is_empty()); +//! assert_eq!(foo_dec, foo); +//! assert_eq!(bar_dec, bar); +//! # Ok::<(), Box>(()) +//! ``` use std::{ any::type_name, net::{Ipv4Addr, Ipv6Addr}, @@ -16,9 +171,12 @@ const U32_SIZE: usize = 4; pub trait CompactEncoding { /// The size in bytes required to encode `self`. fn encoded_size(&self) -> Result; + /// Encode `self` into `buffer` returning the remainder of `buffer`. fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; - /// Decode a value of type [`Decode`] from `buffer`. Returns the decoded value and remaining undecoded bytes. + + /// Decode a value from the given `buffer` of the type specified by the `Decode` type parameter + /// (which defaults to `Self`). Returns the decoded value and remaining undecoded bytes. fn decode(buffer: &[u8]) -> Result<(Decode, &[u8]), EncodingError> where Decode: Sized; @@ -290,7 +448,8 @@ pub fn encoded_bytes_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], } } -/// decode a `usize` from `buffer` and return the remaining bytes +/// Decode a `usize` from `buffer` and return the remaining bytes. +/// This will fail, when we are decoding a `usize` on a usize = u32 machine for data that was originally encoded on a `usize = u64` machine whenever the value is over `u32::MAX`. pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { let [first, rest @ ..] = buffer else { return Err(EncodingError::out_of_bounds( From 23b1359e8639be5b1ac590fda45dfeeab8c028dc Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:16:01 -0400 Subject: [PATCH 33/56] cargo +nightly rustfmt --- src/encodable.rs | 44 ++++++++++++++++++++++++-------------------- src/lib.rs | 26 +++++++++++++------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/encodable.rs b/src/encodable.rs index 62cdaf2..d3226dc 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -10,7 +10,7 @@ //! //! ### Quick start //! ``` -//! use compact_encoding::{to_encoded_bytes, map_decode}; +//! use compact_encoding::{map_decode, to_encoded_bytes}; //! //! let number = 41_u32; //! let word = "hi"; @@ -61,8 +61,8 @@ //! You can implement [`CompactEncoding`]` for your own structs like below: //! ``` //! use compact_encoding::{ -//! to_encoded_bytes, map_encode, map_decode, sum_encoded_size, -//! {encodable::CompactEncoding, EncodingError} +//! map_decode, map_encode, sum_encoded_size, to_encoded_bytes, +//! {encodable::CompactEncoding, EncodingError}, //! }; //! //! #[derive(Debug, PartialEq)] @@ -101,21 +101,24 @@ //! } //! //! fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> { -//! let (flags, rest) = u8::decode(buffer)?; -//! let some_flag: bool = flags & 1 != 0; -//! let (values, rest) = if flags & 2 != 0 { -//! let (vec, rest) = >::decode(rest)?; -//! (Some(vec), rest) -//! } else { -//! (None, rest) -//! }; -//! let ((other, stuff), rest) = map_decode!(rest, [String, u64]); -//! Ok((Self { -//! some_flag, -//! values, -//! other, -//! stuff -//! }, rest)) +//! let (flags, rest) = u8::decode(buffer)?; +//! let some_flag: bool = flags & 1 != 0; +//! let (values, rest) = if flags & 2 != 0 { +//! let (vec, rest) = >::decode(rest)?; +//! (Some(vec), rest) +//! } else { +//! (None, rest) +//! }; +//! let ((other, stuff), rest) = map_decode!(rest, [String, u64]); +//! Ok(( +//! Self { +//! some_flag, +//! values, +//! other, +//! stuff, +//! }, +//! rest, +//! )) //! } //! } //! @@ -131,14 +134,15 @@ //! some_flag: true, //! values: Some(vec![[1; 32], [2; 32]]), //! other: "yo".to_string(), -//! stuff: 0 +//! stuff: 0, //! }; //! //! // Encode `foo` and `bar` to a buffer //! let buffer = to_encoded_bytes!(&foo, &bar); //! //! // With the above use of a flags byte, the empty value encodes to only one byte -//! assert_eq!(buffer.len(), +//! assert_eq!( +//! buffer.len(), //! // flags + string + u64 //! (1 + 3 + 1) + //! // "" + (values.len().encoded_size() + (values.len() * <[u8;32]>::encoded_size()) + "" diff --git a/src/lib.rs b/src/lib.rs index 797ce1f..d9d42a4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -88,19 +88,19 @@ //! } //! //! fn decode(&mut self, buffer: &[u8]) -> Result { -//! let flags: u8 = self.decode(buffer)?; -//! let my_flag_1: bool = flags & 1 != 0; -//! let my_flag_2: bool = flags & 2 != 0; -//! let my_values: Vec<[u8; 32]> = if flags & 4 != 0 { -//! self.decode(buffer)? -//! } else { -//! vec![] -//! }; -//! Ok(MyStruct { +//! let flags: u8 = self.decode(buffer)?; +//! let my_flag_1: bool = flags & 1 != 0; +//! let my_flag_2: bool = flags & 2 != 0; +//! let my_values: Vec<[u8; 32]> = if flags & 4 != 0 { +//! self.decode(buffer)? +//! } else { +//! vec![] +//! }; +//! Ok(MyStruct { //! my_flag_1, //! my_flag_2, -//! my_values -//! }) +//! my_values, +//! }) //! } //! } //! @@ -108,12 +108,12 @@ //! let empty = MyStruct { //! my_flag_1: false, //! my_flag_2: true, -//! my_values: vec![] +//! my_values: vec![], //! }; //! let non_empty = MyStruct { //! my_flag_1: true, //! my_flag_2: false, -//! my_values: vec![[1; 32], [2; 32]] +//! my_values: vec![[1; 32], [2; 32]], //! }; //! //! // Start with an empty state From 58c54b55aa44b19cbe1ef2bc134bf6e3fd49edeb Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:21:55 -0400 Subject: [PATCH 34/56] replace lib.rs --- src/encodable.rs | 6 +- src/lib.rs | 149 ----------------------------------------------- 2 files changed, 2 insertions(+), 153 deletions(-) delete mode 100644 src/lib.rs diff --git a/src/encodable.rs b/src/encodable.rs index d3226dc..cbb7260 100644 --- a/src/encodable.rs +++ b/src/encodable.rs @@ -158,15 +158,13 @@ //! assert_eq!(bar_dec, bar); //! # Ok::<(), Box>(()) //! ``` +mod types; use std::{ any::type_name, net::{Ipv4Addr, Ipv6Addr}, }; -use crate::{ - types::{U16_SIGNIFIER, U32_SIGNIFIER, U64_SIGNIFIER}, - EncodingError, -}; +use crate::types::{EncodingError, U16_SIGNIFIER, U32_SIGNIFIER, U64_SIGNIFIER}; const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index d9d42a4..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,149 +0,0 @@ -#![forbid(unsafe_code, missing_docs)] -#![cfg_attr(test, deny(warnings))] -#![doc(test(attr(deny(warnings))))] - -//! # Series of compact encoding schemes for building small and fast parsers and serializers -//! -//! Binary compatible with the -//! [original Javascript compact-encoding library](https://github.com/compact-encoding/compact-encoding/). -//! -//! ## Usage -//! -//! ### Basic -//! -//! Using only the types implemented here (replace `unwrap()` with proper -//! handling of [EncodingError]): -//! -//! ``` -//! use compact_encoding::{CompactEncoding, State}; -//! -//! // Start with an empty state -//! let mut enc_state = State::new(); -//! -//! let number = 41_u32; -//! let str = "hi".to_string(); -//! -//! // Use preencode to figure out how big a buffer is needed -//! enc_state.preencode(&number).unwrap(); -//! enc_state.preencode(&str).unwrap(); -//! -//! // Create buffer of pre-encoded size -//! let mut buffer = enc_state.create_buffer(); -//! assert_eq!(buffer.len(), 1 + 1 + 2); -//! -//! // Then actually encode the values -//! enc_state.encode(&number, &mut buffer).unwrap(); -//! enc_state.encode(&str, &mut buffer).unwrap(); -//! assert_eq!(buffer.to_vec(), vec![41_u8, 2_u8, b'h', b'i']); -//! -//! // On the decoding end, create a state from byte buffer -//! let mut dec_state = State::from_buffer(&buffer); -//! let number_dec: u32 = dec_state.decode(&buffer).unwrap(); -//! let str_dec: String = dec_state.decode(&buffer).unwrap(); -//! assert_eq!(number_dec, number); -//! assert_eq!(str_dec, str); -//! ``` -//! -//! ### Custom -//! -//! If you want to encode your own structs directly, you can do that -//! by implementing [CompactEncoding] yourself (replace `unwrap()` with proper -//! handling of [EncodingError]): -//! -//! ``` -//! use compact_encoding::{CompactEncoding, EncodingError, State}; -//! -//! #[derive(Debug, PartialEq)] -//! struct MyStruct { -//! my_flag_1: bool, -//! my_flag_2: bool, -//! my_values: Vec<[u8; 32]>, -//! } -//! -//! impl CompactEncoding for State { -//! fn preencode(&mut self, value: &MyStruct) -> Result { -//! self.add_end(1)?; // flags -//! if !value.my_values.is_empty() { -//! self.preencode(&value.my_values)?; -//! } -//! Ok(self.end()) -//! } -//! -//! fn encode(&mut self, value: &MyStruct, buffer: &mut [u8]) -> Result { -//! let mut flags: u8 = 0; -//! if value.my_flag_1 { -//! flags |= 1; -//! } -//! if value.my_flag_2 { -//! flags |= 2; -//! } -//! if !value.my_values.is_empty() { -//! flags |= 4; -//! } -//! self.encode(&flags, buffer)?; -//! if !value.my_values.is_empty() { -//! self.encode(&value.my_values, buffer)?; -//! } -//! Ok(self.start()) -//! } -//! -//! fn decode(&mut self, buffer: &[u8]) -> Result { -//! let flags: u8 = self.decode(buffer)?; -//! let my_flag_1: bool = flags & 1 != 0; -//! let my_flag_2: bool = flags & 2 != 0; -//! let my_values: Vec<[u8; 32]> = if flags & 4 != 0 { -//! self.decode(buffer)? -//! } else { -//! vec![] -//! }; -//! Ok(MyStruct { -//! my_flag_1, -//! my_flag_2, -//! my_values, -//! }) -//! } -//! } -//! -//! // Test values -//! let empty = MyStruct { -//! my_flag_1: false, -//! my_flag_2: true, -//! my_values: vec![], -//! }; -//! let non_empty = MyStruct { -//! my_flag_1: true, -//! my_flag_2: false, -//! my_values: vec![[1; 32], [2; 32]], -//! }; -//! -//! // Start with an empty state -//! let mut enc_state = State::new(); -//! enc_state.preencode(&empty).unwrap(); -//! enc_state.preencode(&non_empty).unwrap(); -//! let mut buffer = enc_state.create_buffer(); -//! -//! // With the above use of a flags byte, the empty value encodes to only one byte -//! assert_eq!(buffer.len(), 1 + 1 + 1 + 2 * 32); -//! -//! // Then actually encode the values -//! enc_state.encode(&empty, &mut buffer).unwrap(); -//! enc_state.encode(&non_empty, &mut buffer).unwrap(); -//! -//! // On the decoding end, create a state from byte buffer -//! let mut dec_state = State::from_buffer(&buffer); -//! -//! // And decode directly to your own struct -//! let empty_dec: MyStruct = dec_state.decode(&buffer).unwrap(); -//! let non_empty_dec: MyStruct = dec_state.decode(&buffer).unwrap(); -//! assert_eq!(empty_dec, empty); -//! assert_eq!(non_empty_dec, non_empty); -//! ``` -//! -//! **NB**: This only works if you don't export your struct out of your crate. -//! If you export the struct, orphan rule will require you to -//! implement a wrapper for [State], e.g. `struct MyState(State);` and implement -//! [CompactEncoding] for the wrapper struct instead. -pub mod encodable; -pub mod generic; -pub mod types; -pub use types::{CompactEncoding, EncodingError, EncodingErrorKind, State}; From ff5a6140a05714463d19984cc9dcc5967f31c035 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:24:16 -0400 Subject: [PATCH 35/56] rm unused --- src/{encodable.rs => lib.rs} | 0 src/types.rs | 764 ----------------------------------- 2 files changed, 764 deletions(-) rename src/{encodable.rs => lib.rs} (100%) diff --git a/src/encodable.rs b/src/lib.rs similarity index 100% rename from src/encodable.rs rename to src/lib.rs diff --git a/src/types.rs b/src/types.rs index b7f3ab7..5c28307 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,8 +1,5 @@ //! Basic types of compact_encoding. -use crate::encodable::CompactEncoding as Cenc; -use std::convert::TryFrom; use std::fmt; -use std::ops::Range; pub(crate) const U16_SIGNIFIER: u8 = 0xfd; pub(crate) const U32_SIGNIFIER: u8 = 0xfe; @@ -86,764 +83,3 @@ impl From for std::io::Error { } } } - -/// State. -#[derive(Debug, Clone)] -pub struct State { - /// Start position - start: usize, - /// End position - end: usize, -} - -impl Default for State { - /// Create empty state - fn default() -> Self { - Self::new() - } -} - -impl State { - /// Create empty state - pub fn new() -> State { - State::new_with_start_and_end(0, 0) - } - - /// Create a state and buffer with an already known size. - /// With this, you can/must skip the preencode step. - pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { - ( - State::new_with_start_and_end(0, size), - vec![0; size].into_boxed_slice(), - ) - } - - /// Create a state with a start and end already known. - pub fn new_with_start_and_end(start: usize, end: usize) -> State { - State { start, end } - } - - /// Create a state from existing buffer. - pub fn from_buffer(buffer: &[u8]) -> State { - State::new_with_start_and_end(0, buffer.len()) - } - - /// Start value - pub fn start(&self) -> usize { - self.start - } - - /// Set start value - pub fn set_start(&mut self, value: usize) -> Result<(), EncodingError> { - if value > self.end { - return Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!("Value exceeded state.end: {} > {}", value, self.end), - )); - } - self.start = value; - Ok(()) - } - - /// End value - pub fn end(&self) -> usize { - self.end - } - - /// Set end value - pub fn set_end(&mut self, value: usize) { - self.end = value; - } - - /// Add to start handling overflow and out of bounds. - pub fn add_start(&mut self, increment: usize) -> Result { - self.start = self.start.checked_add(increment).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "State.start overflowed: {} + {} > {}", - self.start, - increment, - usize::MAX - ), - ) - })?; - if self.start > self.end { - Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "State.start exceeded state.end: {} > {}", - self.start, self.end - ), - )) - } else { - Ok(self.start) - } - } - - /// Add to end handling overflow - pub fn add_end(&mut self, increment: usize) -> Result { - self.end = self.end.checked_add(increment).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "State.end overflowed: {} + {} > {}", - self.end, - increment, - usize::MAX - ), - ) - })?; - Ok(self.end) - } - - /// After calling preencode(), this allocates the right size buffer to the heap. - /// Follow this with the same number of encode() steps to fill the created buffer. - pub fn create_buffer(&self) -> Box<[u8]> { - vec![0; self.end].into_boxed_slice() - } - - /// Safely set single byte to buffer at state.start and then increment state.start, returning - /// new state.start. - pub fn set_byte_to_buffer( - &mut self, - value: u8, - buffer: &mut [u8], - ) -> Result { - if buffer.len() <= self.start { - Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "Length of buffer {} too small to fit single byte", - buffer.len() - ), - )) - } else { - buffer[self.start] = value; - self.add_start(1) - } - } - - /// Safely set byte slice to buffer at state.start and then increment state.start with slice - /// length, returning new state.start. - pub fn set_slice_to_buffer( - &mut self, - value: &[u8], - buffer: &mut [u8], - ) -> Result { - self.set_slice_to_buffer_fixed(value, buffer, value.len()) - } - - /// Safely set byte slice of fixed len to buffer at state.start and then increment state.start with slice - /// length, returning new state.start. - pub fn set_slice_to_buffer_fixed( - &mut self, - value: &[u8], - buffer: &mut [u8], - size: usize, - ) -> Result { - if value.len() < size { - return Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "Length of value {} too small to fit fixed size {}", - value.len(), - size - ), - )); - } - let value_end = size.checked_add(self.start).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "Value end overflowed: {} + {} > {}", - size, - self.start, - usize::MAX - ), - ) - })?; - if buffer.len() < value_end { - Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!( - "Length of buffer {} too small to fit slice of length {}", - buffer.len(), - size - ), - )) - } else { - buffer[self.start..value_end].copy_from_slice(value); - self.add_start(size) - } - } - - /// Validate `size` can be decoded from `buffer`, return current start. - pub fn validate(&mut self, size: usize, buffer: &[u8]) -> Result, EncodingError> { - let value_end = size.checked_add(self.start).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "Value end overflowed during validate: {} + {} > {}", - size, - self.start, - usize::MAX - ), - ) - })?; - - if buffer.len() < value_end { - Err(EncodingError::new( - EncodingErrorKind::Overflow, - &format!("Buffer length {} too small for size {}", buffer.len(), size,), - )) - } else { - Ok(self.start..value_end) - } - } - - /// Preencode a string slice - pub fn preencode_str(&mut self, value: &str) -> Result { - self.preencode_usize_var(&value.len())?; - self.add_end(value.len()) - } - - /// Encode a string slice - pub fn encode_str(&mut self, value: &str, buffer: &mut [u8]) -> Result { - let len = value.len(); - self.encode_usize_var(&len, buffer)?; - self.set_slice_to_buffer(value.as_bytes(), buffer) - } - - /// Decode a String - pub fn decode_string(&mut self, buffer: &[u8]) -> Result { - let len = self.decode_usize_var(buffer)?; - let range = self.validate(len, buffer)?; - let value = std::str::from_utf8(&buffer[range]).map_err(|err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("String is invalid UTF-8, {err}"), - ) - })?; - self.add_start(len)?; - Ok(value.to_string()) - } - - /// Preencode a variable length usigned int - pub fn preencode_uint_var + Ord>( - &mut self, - uint: &T, - ) -> Result { - let increment = if uint < &T::from(u32::from(U16_SIGNIFIER)) { - 1 - } else if uint <= &(0xffff.into()) { - 3 - } else if uint <= &(0xffffffff.into()) { - 5 - } else { - 9 - }; - self.add_end(increment) - } - - /// Decode a fixed length u8 - pub fn decode_u8(&mut self, buffer: &[u8]) -> Result { - self.validate(1, buffer)?; - let value: u8 = buffer[self.start]; - self.add_start(1)?; - Ok(value) - } - - /// Decode a fixed length u16 - pub fn decode_u16(&mut self, buffer: &[u8]) -> Result { - self.validate(2, buffer)?; - let value: u16 = (buffer[self.start] as u16) | ((buffer[self.start + 1] as u16) << 8); - self.add_start(2)?; - Ok(value) - } - - /// Encode a variable length u32 - pub fn encode_u32_var( - &mut self, - value: &u32, - buffer: &mut [u8], - ) -> Result { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - self.set_byte_to_buffer(bytes[0], buffer) - } else if *value <= 0xffff { - self.set_byte_to_buffer(U16_SIGNIFIER, buffer)?; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer) - } else { - self.set_byte_to_buffer(U32_SIGNIFIER, buffer)?; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer) - } - } - - /// Encode u32 to 4 LE bytes. - pub fn encode_u32(&mut self, uint: u32, buffer: &mut [u8]) -> Result { - self.encode_uint32_bytes(&uint.to_le_bytes(), buffer) - } - - /// Decode a variable length u32 - #[allow(clippy::comparison_chain)] - pub fn decode_u32_var(&mut self, buffer: &[u8]) -> Result { - self.validate(1, buffer)?; - let first = buffer[self.start]; - self.add_start(1)?; - if first < U16_SIGNIFIER { - Ok(first.into()) - } else if first == U16_SIGNIFIER { - Ok(self.decode_u16(buffer)?.into()) - } else { - self.decode_u32(buffer) - } - } - - /// Decode a fixed length u32 - pub fn decode_u32(&mut self, buffer: &[u8]) -> Result { - self.validate(4, buffer)?; - let value: u32 = (buffer[self.start] as u32) - | ((buffer[self.start + 1] as u32) << 8) - | ((buffer[self.start + 2] as u32) << 16) - | ((buffer[self.start + 3] as u32) << 24); - self.add_start(4)?; - Ok(value) - } - - /// Encode a variable length u64 - pub fn encode_u64_var( - &mut self, - value: &u64, - buffer: &mut [u8], - ) -> Result { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - self.set_byte_to_buffer(bytes[0], buffer) - } else if *value <= 0xffff { - self.set_byte_to_buffer(U16_SIGNIFIER, buffer)?; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer) - } else if *value <= 0xffffffff { - self.set_byte_to_buffer(U32_SIGNIFIER, buffer)?; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer) - } else { - self.set_byte_to_buffer(U64_SIGNIFIER, buffer)?; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer) - } - } - - /// Encode u64 to 8 LE bytes. - pub fn encode_u64(&mut self, uint: u64, buffer: &mut [u8]) -> Result { - self.encode_uint64_bytes(&uint.to_le_bytes(), buffer) - } - - /// Decode a variable length u64 - pub fn decode_u64_var(&mut self, buffer: &[u8]) -> Result { - self.validate(1, buffer)?; - let first = buffer[self.start]; - self.add_start(1)?; - if first < U16_SIGNIFIER { - Ok(first.into()) - } else if first == U16_SIGNIFIER { - Ok(self.decode_u16(buffer)?.into()) - } else if first == U32_SIGNIFIER { - Ok(self.decode_u32(buffer)?.into()) - } else { - self.decode_u64(buffer) - } - } - - /// Decode a fixed length u64 - pub fn decode_u64(&mut self, buffer: &[u8]) -> Result { - self.validate(8, buffer)?; - let value: u64 = (buffer[self.start] as u64) - | ((buffer[self.start + 1] as u64) << 8) - | ((buffer[self.start + 2] as u64) << 16) - | ((buffer[self.start + 3] as u64) << 24) - | ((buffer[self.start + 4] as u64) << 32) - | ((buffer[self.start + 5] as u64) << 40) - | ((buffer[self.start + 6] as u64) << 48) - | ((buffer[self.start + 7] as u64) << 56); - self.add_start(8)?; - Ok(value) - } - - /// Preencode a byte buffer - pub fn preencode_buffer(&mut self, value: &[u8]) -> Result { - let len = value.len(); - self.preencode_usize_var(&len)?; - self.add_end(len) - } - - /// Preencode a vector byte buffer - pub fn preencode_buffer_vec(&mut self, value: &[u8]) -> Result { - let len = value.len(); - self.preencode_usize_var(&len)?; - self.add_end(len) - } - - /// Encode a byte buffer - pub fn encode_buffer( - &mut self, - value: &[u8], - buffer: &mut [u8], - ) -> Result { - let len = value.len(); - self.encode_usize_var(&len, buffer)?; - self.set_slice_to_buffer(value, buffer) - } - - /// Decode a byte buffer - pub fn decode_buffer(&mut self, buffer: &[u8]) -> Result, EncodingError> { - Ok(self.decode_buffer_vec(buffer)?.into_boxed_slice()) - } - - /// Decode a vector byte buffer - pub fn decode_buffer_vec(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len = self.decode_usize_var(buffer)?; - let range = self.validate(len, buffer)?; - let value = buffer[range].to_vec(); - self.add_start(value.len())?; - Ok(value) - } - - /// Preencode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn preencode_raw_buffer(&mut self, value: &[u8]) -> Result { - self.add_end(value.len()) - } - - /// Encode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn encode_raw_buffer( - &mut self, - value: &[u8], - buffer: &mut [u8], - ) -> Result { - self.set_slice_to_buffer(value, buffer) - } - - /// Decode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn decode_raw_buffer(&mut self, buffer: &[u8]) -> Result, EncodingError> { - if self.start >= self.end { - return Err(EncodingError::new( - EncodingErrorKind::OutOfBounds, - &format!("State.start {} >= state.end {}", self.start, self.end), - )); - } - let range = self.validate(self.end - self.start, buffer)?; - let value = buffer[range].to_vec(); - self.start = self.end; - Ok(value) - } - - /// Preencode a fixed 16 byte buffer - pub fn preencode_fixed_16(&mut self) -> Result { - self.add_end(16) - } - - /// Encode a fixed 16 byte buffer - pub fn encode_fixed_16( - &mut self, - value: &[u8], - buffer: &mut [u8], - ) -> Result { - self.set_slice_to_buffer_fixed(value, buffer, 16) - } - - /// Decode a fixed 16 byte buffer - pub fn decode_fixed_16(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let range = self.validate(16, buffer)?; - let value = buffer[range].to_vec().into_boxed_slice(); - self.add_start(16)?; - Ok(value) - } - - /// Preencode a fixed 32 byte buffer - pub fn preencode_fixed_32(&mut self) -> Result { - self.add_end(32) - } - - /// Encode a fixed 32 byte buffer - pub fn encode_fixed_32( - &mut self, - value: &[u8], - buffer: &mut [u8], - ) -> Result { - self.set_slice_to_buffer_fixed(value, buffer, 32) - } - - /// Decode a fixed 32 byte buffer - pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let range = self.validate(32, buffer)?; - let value = buffer[range].to_vec().into_boxed_slice(); - self.add_start(32)?; - Ok(value) - } - - /// Preencode a string array - pub fn preencode_string_array(&mut self, value: &[String]) -> Result { - let len = value.len(); - self.preencode_usize_var(&len)?; - for string_value in value.iter() { - self.preencode_str(string_value)?; - } - Ok(self.end) - } - - /// Encode a String array - pub fn encode_string_array( - &mut self, - value: &Vec, - buffer: &mut [u8], - ) -> Result { - let len = value.len(); - self.encode_usize_var(&len, buffer)?; - for string_value in value { - self.encode_str(string_value, buffer)?; - } - Ok(self.end) - } - - /// Decode a String array - pub fn decode_string_array(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len = self.decode_usize_var(buffer)?; - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode_string(buffer)?); - } - Ok(value) - } - - /// Preencode an u32 array - pub fn preencode_u32_array(&mut self, value: &[u32]) -> Result { - let len = value.len(); - self.preencode_usize_var(&len)?; - let total_len = len.checked_mul(4).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "Vec total length overflowed: {} * 4 > {}", - len, - usize::MAX - ), - ) - })?; - self.add_end(total_len) - } - - /// Encode an u32 array - pub fn encode_u32_array( - &mut self, - value: &Vec, - buffer: &mut [u8], - ) -> Result { - let len = value.len(); - self.encode_usize_var(&len, buffer)?; - for entry in value { - self.encode_u32(*entry, buffer)?; - } - Ok(self.start()) - } - - /// Decode an u32 array - pub fn decode_u32_array(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len = self.decode_usize_var(buffer)?; - let mut value: Vec = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode_u32(buffer)?); - } - Ok(value) - } - - /// Preencode a fixed 32 byte value array - pub fn preencode_fixed_32_array(&mut self, value: &[[u8; 32]]) -> Result { - let len = value.len(); - self.preencode(&len)?; - let size = len.checked_mul(32).ok_or_else(|| { - EncodingError::new( - EncodingErrorKind::Overflow, - &format!( - "Vec<[u8; 32]> byte size overflowed: {} * 32 > {}", - len, - usize::MAX - ), - ) - })?; - self.add_end(size)?; - Ok(self.end()) - } - - /// Encode a fixed 32 byte value array - pub fn encode_fixed_32_array( - &mut self, - value: &Vec<[u8; 32]>, - buffer: &mut [u8], - ) -> Result { - self.encode(&value.len(), buffer)?; - for entry in value { - self.set_slice_to_buffer_fixed(entry, buffer, 32)?; - } - Ok(self.start()) - } - - /// Decode a fixed 32 byte value array - pub fn decode_fixed_32_array(&mut self, buffer: &[u8]) -> Result, EncodingError> { - let len: usize = self.decode(buffer)?; - let mut entries: Vec<[u8; 32]> = Vec::with_capacity(len); - for _ in 0..len { - let range = self.validate(32, buffer)?; - entries.push(buffer[range].try_into().map_err(|err| { - EncodingError::new( - EncodingErrorKind::InvalidData, - &format!("Could not convert byte slice to [u8; 32], {err}"), - ) - })?); - self.add_start(32)?; - } - Ok(entries) - } - - /// Preencode a variable length usize - pub fn preencode_usize_var(&mut self, value: &usize) -> Result { - // This repeats the logic from above that works for u8 -> u64, but sadly not usize - let increment: usize = if *value < U16_SIGNIFIER.into() { - 1 - } else if *value <= 0xffff { - 3 - } else if *value <= 0xffffffff { - 5 - } else { - 9 - }; - self.add_end(increment) - } - - /// Encode a variable length usize - pub fn encode_usize_var( - &mut self, - value: &usize, - buffer: &mut [u8], - ) -> Result { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - self.set_byte_to_buffer(bytes[0], buffer) - } else if *value <= 0xffff { - self.set_byte_to_buffer(U16_SIGNIFIER, buffer)?; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer) - } else if *value <= 0xffffffff { - self.set_byte_to_buffer(U32_SIGNIFIER, buffer)?; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer) - } else { - self.set_byte_to_buffer(U64_SIGNIFIER, buffer)?; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer) - } - } - - /// Decode a variable length usize. - pub fn decode_usize_var(&mut self, buffer: &[u8]) -> Result { - self.validate(1, buffer)?; - let first = buffer[self.start]; - self.add_start(1)?; - // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently - // created from a byte slice. - if first < U16_SIGNIFIER { - Ok(first.into()) - } else if first == U16_SIGNIFIER { - Ok(self.decode_u16(buffer)?.into()) - } else if first == U32_SIGNIFIER { - usize::try_from(self.decode_u32(buffer)?).map_err(|_| { - EncodingError::new( - EncodingErrorKind::Overflow, - "Attempted converting to a 32 bit usize on below 32 bit system", - ) - }) - } else { - usize::try_from(self.decode_u64(buffer)?).map_err(|_| { - EncodingError::new( - EncodingErrorKind::Overflow, - "Attempted converting to a 64 bit usize on below 64 bit system", - ) - }) - } - } - - /// Encode a u16 - /// TODO should it be encode_uINT16? - pub fn encode_u16(&mut self, uint: u16, buffer: &mut [u8]) -> Result { - self.set_slice_to_buffer(&uint.to_le_bytes(), buffer) - } - - /// Encode a 2 byte unsigned integer. NB: assumes `bytes` buffer large enough, hence not public! - fn encode_uint16_bytes( - &mut self, - bytes: &[u8], - buffer: &mut [u8], - ) -> Result { - self.set_slice_to_buffer(&bytes[..2], buffer) - } - - /// Encode a 4 byte unsigned integer. NB: assumes `bytes` buffer large enough, hence not public! - fn encode_uint32_bytes( - &mut self, - bytes: &[u8], - buffer: &mut [u8], - ) -> Result { - self.encode_uint16_bytes(bytes, buffer)?; - self.set_slice_to_buffer(&bytes[2..4], buffer) - } - - /// Encode an 8 byte unsigned integer. NB: assumes `bytes` buffer large enough, hence not public! - fn encode_uint64_bytes( - &mut self, - bytes: &[u8], - buffer: &mut [u8], - ) -> Result { - self.encode_uint32_bytes(bytes, buffer)?; - self.set_slice_to_buffer(&bytes[4..8], buffer) - } - - /// Like [`State::preencode`] but for `CompactEncodable` types - pub fn preencode_t(&mut self, value: &T) -> Result { - let size = value.encoded_size()?; - let out = self.add_end(size)?; - Ok(out) - } - - /// Like [`State::encode`] but for `CompactEncodable` types - pub fn encode_t( - &mut self, - value: &T, - buffer: &mut [u8], - ) -> Result { - let start_len = buffer.len(); - let rest = value.encode(buffer)?; - let offset = start_len - rest.len(); - self.add_start(offset) - } - - /// Like [`State::decode`] but for `CompactEncodable` types - pub fn decode_t(&mut self, buffer: &[u8]) -> Result { - let start_len = buffer.len(); - let (result, remaining_buffer) = T::decode(buffer)?; - let after = remaining_buffer.len(); - self.add_start(start_len - after)?; - Ok(result) - } -} - -/// Compact Encoding. You must implement `CompactEncoding` where T is the type to be encoded. -pub trait CompactEncoding -where - T: fmt::Debug, -{ - /// Preencode - fn preencode(&mut self, value: &T) -> Result; - - /// Encode - fn encode(&mut self, value: &T, buffer: &mut [u8]) -> Result; - - /// Decode - fn decode(&mut self, buffer: &[u8]) -> Result; -} From e089aa9e71086ee3e675444929e97d8eea47b47b Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:24:28 -0400 Subject: [PATCH 36/56] rm unused --- tests/generic.rs | 228 ----------------------------------------------- 1 file changed, 228 deletions(-) delete mode 100644 tests/generic.rs diff --git a/tests/generic.rs b/tests/generic.rs deleted file mode 100644 index d1228c6..0000000 --- a/tests/generic.rs +++ /dev/null @@ -1,228 +0,0 @@ -use compact_encoding::{CompactEncoding, EncodingError, State}; - -// The max value for 1 byte length is 252 -const MAX_ONE_BYTE_UINT: u8 = 252; - -// The min value for 2 byte length is 253 -const MIN_TWO_BYTE_UINT: u8 = 253; - -#[test] -fn cenc_basic() -> Result<(), EncodingError> { - let str_value_1 = "foo"; - let str_value_2 = (0..MAX_ONE_BYTE_UINT).map(|_| "X").collect::(); - let u32_value_3: u32 = u32::MAX; - let u32_value_4: u32 = 0xF0E1D2C3; - - let mut enc_state = State::new(); - enc_state.preencode_str(str_value_1)?; - enc_state.preencode(&str_value_2)?; - enc_state.preencode(&u32_value_3)?; - enc_state.preencode(&u32_value_4)?; - let mut buffer = enc_state.create_buffer(); - // Strings: 1 byte for length, 3/252 bytes for content - // u32: 1 byte for u32 signifier, 4 bytes for data - assert_eq!(buffer.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); - enc_state.encode_str(str_value_1, &mut buffer)?; - enc_state.encode(&str_value_2, &mut buffer)?; - enc_state.encode(&u32_value_3, &mut buffer)?; - enc_state.encode(&u32_value_4, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let str_value_1_ret: String = dec_state.decode(&buffer)?; - assert_eq!(str_value_1, str_value_1_ret); - let str_value_2_ret: String = dec_state.decode(&buffer)?; - assert_eq!(str_value_2, str_value_2_ret); - let u32_value_3_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value_3, u32_value_3_ret); - let u32_value_4_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value_4, u32_value_4_ret); - Ok(()) -} - -#[test] -fn cenc_string_long() -> Result<(), EncodingError> { - let str_value = (0..MIN_TWO_BYTE_UINT).map(|_| "X").collect::(); - assert_eq!(str_value.len(), 253); - let mut enc_state = State::new(); - enc_state.preencode(&str_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for u16 signifier, 2 bytes for length, 256 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 253); - enc_state.encode(&str_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let str_value_ret: String = dec_state.decode(&buffer)?; - assert_eq!(str_value, str_value_ret); - Ok(()) -} - -#[test] -fn cenc_u32_as_u16() -> Result<(), EncodingError> { - let u32_value: u32 = u16::MAX.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u32_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for u16 signifier, 2 bytes for length - assert_eq!(buffer.len(), 1 + 2); - enc_state.encode(&u32_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let u32_value_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value, u32_value_ret); - Ok(()) -} - -#[test] -fn cenc_u32_as_u8() -> Result<(), EncodingError> { - let u32_value: u32 = MAX_ONE_BYTE_UINT.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u32_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for data - assert_eq!(buffer.len(), 1); - enc_state.encode(&u32_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let u32_value_ret: u32 = dec_state.decode(&buffer)?; - assert_eq!(u32_value, u32_value_ret); - Ok(()) -} - -#[test] -fn cenc_u64() -> Result<(), EncodingError> { - let u64_value: u64 = 0xF0E1D2C3B4A59687; - let mut enc_state = State::new(); - enc_state.preencode(&u64_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for u64 signifier, 8 bytes for length - assert_eq!(buffer.len(), 1 + 8); - enc_state.encode(&u64_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let u64_value_ret: u64 = dec_state.decode(&buffer)?; - assert_eq!(u64_value, u64_value_ret); - Ok(()) -} - -#[test] -fn cenc_u64_as_u32() -> Result<(), EncodingError> { - let u64_value: u64 = u32::MAX.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u64_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for u32 signifier, 4 bytes for length - assert_eq!(buffer.len(), 1 + 4); - enc_state.encode(&u64_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let u64_value_ret: u64 = dec_state.decode(&buffer)?; - assert_eq!(u64_value, u64_value_ret); - Ok(()) -} - -#[test] -fn cenc_buffer() -> Result<(), EncodingError> { - let buf_value_1 = vec![0xFF, 0x00].into_boxed_slice(); - let buf_value_2 = vec![0xEE, 0x11, 0x22].into_boxed_slice(); - let mut enc_state = State::new(); - enc_state.preencode(&buf_value_1)?; - enc_state.preencode(&buf_value_2)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for length, 2 bytes for data - // 1 byte for length, 3 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 1 + 3); - enc_state.encode(&buf_value_1, &mut buffer)?; - enc_state.encode(&buf_value_2, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Box<[u8]> = dec_state.decode(&buffer)?; - let buf_value_2_ret: Box<[u8]> = dec_state.decode(&buffer)?; - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); - - Ok(()) -} - -#[test] -fn cenc_vec() -> Result<(), EncodingError> { - let buf_value_1: Vec = vec![0xFF, 0x00]; - let buf_value_2: Vec = vec![0xFFFFFFFF, 0x11223344, 0x99887766]; - let mut enc_state = State::new(); - enc_state.preencode(&buf_value_1)?; - enc_state.preencode(&buf_value_2)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for length, 2 bytes for data - // 1 byte for length, 4*3 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 1 + 12); - enc_state.encode(&buf_value_1, &mut buffer)?; - enc_state.encode(&buf_value_2, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Vec = dec_state.decode(&buffer)?; - let buf_value_2_ret: Vec = dec_state.decode(&buffer)?; - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); - Ok(()) -} - -#[test] -fn cenc_string_array() -> Result<(), EncodingError> { - let string_array_value = vec!["first".to_string(), "second".to_string()]; - let mut enc_state = State::new(); - enc_state.preencode(&string_array_value)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for array length, - // 1 byte for string length, 5 bytes for string, - // 1 byte for string length, 6 bytes for string - assert_eq!(buffer.len(), 1 + 1 + 5 + 1 + 6); - enc_state.encode(&string_array_value, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let string_array_value_ret: Vec = dec_state.decode(&buffer)?; - assert_eq!(string_array_value, string_array_value_ret); - Ok(()) -} - -#[test] -fn cenc_fixed_and_raw() -> Result<(), EncodingError> { - let buf_value_1: Vec = vec![0xEE; 16]; - let buf_value_2: Vec = vec![0xFF; 32]; - let buf_value_3: Vec = vec![0xFF, 0x11, 0x99]; - let mut enc_state = State::new(); - enc_state.preencode_fixed_16()?; - enc_state.preencode_fixed_32()?; - enc_state.preencode_raw_buffer(&buf_value_3)?; - let mut buffer = enc_state.create_buffer(); - // 16 + 32 bytes for data - // 3 bytes for data - assert_eq!(buffer.len(), 16 + 32 + 3); - enc_state.encode_fixed_16(&buf_value_1, &mut buffer)?; - enc_state.encode_fixed_32(&buf_value_2, &mut buffer)?; - enc_state.encode_raw_buffer(&buf_value_3, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Vec = dec_state.decode_fixed_16(&buffer)?.to_vec(); - let buf_value_2_ret: Vec = dec_state.decode_fixed_32(&buffer)?.to_vec(); - let buf_value_3_ret: Vec = dec_state.decode_raw_buffer(&buffer)?; - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); - assert_eq!(buf_value_3, buf_value_3_ret); - Ok(()) -} - -#[test] -fn cenc_32_byte_array() -> Result<(), EncodingError> { - let empty_array: Vec<[u8; 32]> = vec![]; - let one_array: Vec<[u8; 32]> = vec![[0; 32]]; - let many_array: Vec<[u8; 32]> = vec![[1; 32], [2; 32], [3; 32]]; - let mut enc_state = State::new(); - - enc_state.preencode(&empty_array)?; - enc_state.preencode(&one_array)?; - enc_state.preencode(&many_array)?; - let mut buffer = enc_state.create_buffer(); - // 1 byte for array length, - // 32 bytes for content - assert_eq!(buffer.len(), 1 + 1 + 32 + 1 + 3 * 32); - enc_state.encode(&empty_array, &mut buffer)?; - enc_state.encode(&one_array, &mut buffer)?; - enc_state.encode(&many_array, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - let empty_array_ret: Vec<[u8; 32]> = dec_state.decode(&buffer)?; - let one_array_ret: Vec<[u8; 32]> = dec_state.decode(&buffer)?; - let many_array_ret: Vec<[u8; 32]> = dec_state.decode(&buffer)?; - assert_eq!(empty_array, empty_array_ret); - assert_eq!(one_array, one_array_ret); - assert_eq!(many_array, many_array_ret); - Ok(()) -} From abcca34514e6f3bd970d176b15f41bc5648c466e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:30:34 -0400 Subject: [PATCH 37/56] fix problems from move --- src/lib.rs | 29 ++++++++++++++++++----------- src/types.rs | 4 ---- tests/encodable.rs | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cbb7260..32727ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,7 @@ //! data, and decode it. Using only the types which include a default [`CompactEncoding`] //! implementation. A more ergonomic pattern is demonstrated in other examples //! ``` -//! use compact_encoding::encodable::CompactEncoding; +//! use compact_encoding::CompactEncoding; //! //! let number = 41_u32; //! let word = "hi"; @@ -62,7 +62,7 @@ //! ``` //! use compact_encoding::{ //! map_decode, map_encode, sum_encoded_size, to_encoded_bytes, -//! {encodable::CompactEncoding, EncodingError}, +//! CompactEncoding, EncodingError, //! }; //! //! #[derive(Debug, PartialEq)] @@ -158,13 +158,20 @@ //! assert_eq!(bar_dec, bar); //! # Ok::<(), Box>(()) //! ``` -mod types; +pub mod types; use std::{ any::type_name, net::{Ipv4Addr, Ipv6Addr}, }; -use crate::types::{EncodingError, U16_SIGNIFIER, U32_SIGNIFIER, U64_SIGNIFIER}; +pub use crate::types::{EncodingError, EncodingErrorKind}; + +/// indicates a variable width unsigned integer fits in u16 +pub const U16_SIGNIFIER: u8 = 0xfd; +/// indicates a variable width unsigned integer fits in u32 +pub const U32_SIGNIFIER: u8 = 0xfe; +/// indicates a variable width unsigned integer fits in u64 +pub const U64_SIGNIFIER: u8 = 0xff; const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; @@ -187,7 +194,7 @@ pub trait CompactEncoding { /// encoding to it in one step. /// ``` /// # use std::net::Ipv4Addr; - /// # use compact_encoding::encodable::CompactEncoding; + /// # use compact_encoding::CompactEncoding; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let mut buff = vec![0; foo.encoded_size()?]; /// foo.encode(&mut buff)?; @@ -202,7 +209,7 @@ pub trait CompactEncoding { /// method for: encoding to it in one step. /// ``` /// # use std::net::Ipv4Addr; - /// # use compact_encoding::encodable::CompactEncoding; + /// # use compact_encoding::CompactEncoding; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// vec![0; foo.encoded_size()?]; /// # Ok::<(), Box>(()) @@ -271,7 +278,7 @@ pub trait BoxArrayEncodable: CompactEncoding { /// [`CompactEncoding::encoded_size`] on all of them. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{sum_encoded_size, encodable::CompactEncoding}; +/// # use crate::compact_encoding::{sum_encoded_size, CompactEncoding}; /// # use std::net::Ipv4Addr; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let bar = 42u64; @@ -296,7 +303,7 @@ macro_rules! sum_encoded_size { /// Given a list of [`CompactEncoding`] things, create a zeroed buffer of the correct size for encoding. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{create_buffer, encodable::CompactEncoding}; +/// # use crate::compact_encoding::{create_buffer, CompactEncoding}; /// # use std::net::Ipv4Addr; /// let foo: Ipv4Addr = "0.0.0.0".parse()?; /// let bar = 42u64; @@ -321,7 +328,7 @@ macro_rules! create_buffer { /// Given a buffer and a list of [`CompactEncoding`] things, encode the arguments to the buffer. /// Note this is macro is useful when your arguments have differing types. /// ``` -/// # use crate::compact_encoding::{create_buffer, map_encode, encodable::CompactEncoding}; +/// # use crate::compact_encoding::{create_buffer, map_encode, CompactEncoding}; /// let num = 42u64; /// let word = "yo"; /// let mut buff = create_buffer!(num, word); @@ -356,7 +363,7 @@ macro_rules! map_encode { /// ``` macro_rules! to_encoded_bytes { ($($val:expr),*) => {{ - use $crate::{map_encode, create_buffer, encodable::CompactEncoding}; + use $crate::{map_encode, create_buffer, CompactEncoding}; let mut buffer = create_buffer!($($val),*); map_encode!(&mut buffer, $($val),*); buffer @@ -380,7 +387,7 @@ macro_rules! map_decode { ($buffer:expr, [ $($field_type:ty),* $(,)? ]) => {{ - use $crate::encodable::CompactEncoding; + use $crate::CompactEncoding; let mut current_buffer: &[u8] = $buffer; // Decode each type into `result_tuple` diff --git a/src/types.rs b/src/types.rs index 5c28307..be8301a 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,10 +1,6 @@ //! Basic types of compact_encoding. use std::fmt; -pub(crate) const U16_SIGNIFIER: u8 = 0xfd; -pub(crate) const U32_SIGNIFIER: u8 = 0xfe; -pub(crate) const U64_SIGNIFIER: u8 = 0xff; - /// Specific type [EncodingError] #[derive(fmt::Debug, Clone, PartialEq)] pub enum EncodingErrorKind { diff --git a/tests/encodable.rs b/tests/encodable.rs index eb9637a..218b0d3 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,5 +1,5 @@ use compact_encoding::{ - create_buffer, encodable::CompactEncoding, map_decode, map_encode, types::EncodingError, + create_buffer, map_decode, map_encode, types::EncodingError, CompactEncoding, }; // The max value for 1 byte length is 252 From 4b29f6de05ad1a03aaf6ea39fc15eca99c93a764 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:30:54 -0400 Subject: [PATCH 38/56] rm unused --- src/generic.rs | 146 ------------------------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 src/generic.rs diff --git a/src/generic.rs b/src/generic.rs deleted file mode 100644 index a3b3a99..0000000 --- a/src/generic.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Generic compact encodings - -use super::{CompactEncoding, EncodingError, State}; - -impl CompactEncoding for State { - fn preencode(&mut self, value: &String) -> Result { - self.preencode_str(value) - } - - fn encode(&mut self, value: &String, buffer: &mut [u8]) -> Result { - self.encode_str(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_string(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, _: &u8) -> Result { - self.add_end(1) - } - - fn encode(&mut self, value: &u8, buffer: &mut [u8]) -> Result { - buffer[self.start()] = *value; - self.add_start(1) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let value = buffer[self.start()]; - self.add_start(1)?; - Ok(value) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u32) -> Result { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u32, buffer: &mut [u8]) -> Result { - self.encode_u32_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_u32_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u64) -> Result { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u64, buffer: &mut [u8]) -> Result { - self.encode_u64_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_u64_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &usize) -> Result { - self.preencode_usize_var(value) - } - - fn encode(&mut self, value: &usize, buffer: &mut [u8]) -> Result { - self.encode_usize_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - self.decode_usize_var(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Box<[u8]>) -> Result { - self.preencode_buffer(value) - } - - fn encode(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) -> Result { - self.encode_buffer(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - self.decode_buffer(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) -> Result { - self.preencode_buffer_vec(value) - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { - self.encode_buffer(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - self.decode_buffer_vec(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) -> Result { - self.preencode_u32_array(value) - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { - self.encode_u32_array(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - self.decode_u32_array(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) -> Result { - self.preencode_string_array(value) - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { - self.encode_string_array(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - self.decode_string_array(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec<[u8; 32]>) -> Result { - self.preencode_fixed_32_array(value) - } - - fn encode(&mut self, value: &Vec<[u8; 32]>, buffer: &mut [u8]) -> Result { - self.encode_fixed_32_array(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { - self.decode_fixed_32_array(buffer) - } -} From a5204541f396d4c894dcfe2a7fa1cc91fe4fd97e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 02:31:39 -0400 Subject: [PATCH 39/56] rename types -> errors --- src/{types.rs => error.rs} | 0 src/lib.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/{types.rs => error.rs} (100%) diff --git a/src/types.rs b/src/error.rs similarity index 100% rename from src/types.rs rename to src/error.rs diff --git a/src/lib.rs b/src/lib.rs index 32727ec..85bcee9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -158,13 +158,13 @@ //! assert_eq!(bar_dec, bar); //! # Ok::<(), Box>(()) //! ``` -pub mod types; +pub mod error; use std::{ any::type_name, net::{Ipv4Addr, Ipv6Addr}, }; -pub use crate::types::{EncodingError, EncodingErrorKind}; +pub use crate::error::{EncodingError, EncodingErrorKind}; /// indicates a variable width unsigned integer fits in u16 pub const U16_SIGNIFIER: u8 = 0xfd; From c633b5c97f43e2bf55e375b221db2f2597195afa Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 15 Apr 2025 14:22:38 -0400 Subject: [PATCH 40/56] rename and reorder --- src/lib.rs | 205 ++++++++++++++++++++++----------------------- tests/encodable.rs | 2 +- 2 files changed, 103 insertions(+), 104 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 85bcee9..50dc023 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,7 +200,7 @@ pub trait CompactEncoding { /// foo.encode(&mut buff)?; /// # Ok::<(), Box>(()) /// ``` - fn to_bytes(&self) -> Result, EncodingError> { + fn to_encoded_bytes(&self) -> Result, EncodingError> { let mut buff = vec![0; self.encoded_size()?]; self.encode(&mut buff)?; Ok(buff) @@ -413,79 +413,6 @@ macro_rules! map_first { (mapped, two) }}; } - -/// The number of bytes required to encode this number. Note this is always variable width. -pub fn usize_encoded_size(val: usize) -> usize { - if val < U16_SIGNIFIER.into() { - 1 - } else if val <= 0xffff { - 3 - } else if val <= 0xffffffff { - 5 - } else { - 9 - } -} - -/// The number of bytes required to encode this number. -/// We only need this for u64 because all other uints can be converted to usize reliably. -pub fn encoded_size_var_u64(val: u64) -> usize { - if val < U16_SIGNIFIER.into() { - 1 - } else if val <= 0xffff { - 3 - } else if val <= 0xffffffff { - 5 - } else { - 9 - } -} - -/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. -pub fn encoded_bytes_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { - if uint < U16_SIGNIFIER.into() { - encode_u8(uint as u8, buffer) - } else if uint <= 0xffff { - let rest = write_array(&[U16_SIGNIFIER], buffer)?; - encode_u16(uint as u16, rest) - } else if uint <= 0xffffffff { - let rest = write_array(&[U32_SIGNIFIER], buffer)?; - encode_u32(uint as u32, rest) - } else { - let rest = write_array(&[U64_SIGNIFIER], buffer)?; - encode_u64(uint, rest) - } -} - -/// Decode a `usize` from `buffer` and return the remaining bytes. -/// This will fail, when we are decoding a `usize` on a usize = u32 machine for data that was originally encoded on a `usize = u64` machine whenever the value is over `u32::MAX`. -pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { - let [first, rest @ ..] = buffer else { - return Err(EncodingError::out_of_bounds( - "Colud not decode usize, empty buffer", - )); - }; - let first = *first; - if first < U16_SIGNIFIER { - Ok((first.into(), rest)) - } else if first == U16_SIGNIFIER { - let (out, rest) = decode_u16(buffer)?; - return Ok((out.into(), rest)); - } else if first == U32_SIGNIFIER { - let (out, rest) = decode_u32(buffer)?; - let out: usize = out - .try_into() - .map_err(|_e| EncodingError::overflow("u32 is bigger than usize"))?; - return Ok((out, rest)); - } else { - let (out, rest) = decode_u64(buffer)?; - let out: usize = out - .try_into() - .map_err(|_e| EncodingError::overflow("u64 is bigger than usize"))?; - return Ok((out, rest)); - } -} - /// Split a slice in two at `mid`. Returns encoding error when `mid` is out of bounds. pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), EncodingError> { buffer.split_at_checked(mid).ok_or_else(|| { @@ -562,10 +489,6 @@ pub fn write_slice<'a>(source: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u Ok(rest) } -fn encoded_size_str(value: &str) -> Result { - Ok(usize_encoded_size(value.len()) + value.len()) -} - /// Helper to convert a vec to an array, and fail with an encoding error when needed pub fn bytes_fixed_from_vec(value: &[u8]) -> Result<[u8; N], EncodingError> { <[u8; N]>::try_from(value).map_err(|e| { @@ -577,6 +500,82 @@ pub fn bytes_fixed_from_vec(value: &[u8]) -> Result<[u8; N], Enc }) } +fn encoded_size_str(value: &str) -> Result { + Ok(encoded_size_usize(value.len()) + value.len()) +} + +/// The number of bytes required to encode this number. Note this is always variable width. +pub fn encoded_size_usize(val: usize) -> usize { + if val < U16_SIGNIFIER.into() { + 1 + } else if val <= 0xffff { + 3 + } else if val <= 0xffffffff { + 5 + } else { + 9 + } +} + +/// The number of bytes required to encode this number. +/// We only need this for u64 because all other uints can be converted to usize reliably. +pub fn encoded_size_var_u64(val: u64) -> usize { + if val < U16_SIGNIFIER.into() { + 1 + } else if val <= 0xffff { + 3 + } else if val <= 0xffffffff { + 5 + } else { + 9 + } +} + +/// Write `uint` to the start of `buffer` and return the remaining part of `buffer`. +pub fn encode_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], EncodingError> { + if uint < U16_SIGNIFIER.into() { + encode_u8(uint as u8, buffer) + } else if uint <= 0xffff { + let rest = write_array(&[U16_SIGNIFIER], buffer)?; + encode_u16(uint as u16, rest) + } else if uint <= 0xffffffff { + let rest = write_array(&[U32_SIGNIFIER], buffer)?; + encode_u32(uint as u32, rest) + } else { + let rest = write_array(&[U64_SIGNIFIER], buffer)?; + encode_u64(uint, rest) + } +} + +/// Decode a `usize` from `buffer` and return the remaining bytes. +/// This will fail, when we are decoding a `usize` on a usize = u32 machine for data that was originally encoded on a `usize = u64` machine whenever the value is over `u32::MAX`. +pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { + let [first, rest @ ..] = buffer else { + return Err(EncodingError::out_of_bounds( + "Colud not decode usize, empty buffer", + )); + }; + let first = *first; + if first < U16_SIGNIFIER { + Ok((first.into(), rest)) + } else if first == U16_SIGNIFIER { + let (out, rest) = decode_u16(buffer)?; + return Ok((out.into(), rest)); + } else if first == U32_SIGNIFIER { + let (out, rest) = decode_u32(buffer)?; + let out: usize = out + .try_into() + .map_err(|_e| EncodingError::overflow("u32 is bigger than usize"))?; + return Ok((out, rest)); + } else { + let (out, rest) = decode_u64(buffer)?; + let out: usize = out + .try_into() + .map_err(|_e| EncodingError::overflow("u64 is bigger than usize"))?; + return Ok((out, rest)); + } +} + /// Encoded a fixed sized array to a buffer pub fn encode_bytes_fixed<'a, const N: usize>( value: &[u8; N], @@ -593,23 +592,6 @@ pub fn decode_bytes_fixed( //write_array(value, buffer) } -impl CompactEncoding for [u8; N] { - fn encoded_size(&self) -> Result { - Ok(N) - } - - fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - write_array(self, buffer) - } - - fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> - where - Self: Sized, - { - take_array(buffer) - } -} - fn decode_u16(buffer: &[u8]) -> Result<(u16, &[u8]), EncodingError> { let (data, rest) = take_array::<2>(buffer)?; Ok((u16::from_le_bytes(data), rest)) @@ -725,6 +707,23 @@ fn encode_buffer<'a>(value: &[u8], buffer: &'a mut [u8]) -> Result<&'a mut [u8], write_slice(value, rest) } +impl CompactEncoding for [u8; N] { + fn encoded_size(&self) -> Result { + Ok(N) + } + + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + write_array(self, buffer) + } + + fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> + where + Self: Sized, + { + take_array(buffer) + } +} + impl CompactEncoding for u8 { fn encoded_size(&self) -> Result { Ok(1) @@ -763,7 +762,7 @@ impl CompactEncoding for u16 { // NB: we want u32 encoded and decoded as variable sized uint impl CompactEncoding for u32 { fn encoded_size(&self) -> Result { - Ok(usize_encoded_size(*self as usize)) + Ok(encoded_size_usize(*self as usize)) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { @@ -783,7 +782,7 @@ impl CompactEncoding for u64 { } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - encoded_bytes_var_u64(*self, buffer) + encode_var_u64(*self, buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> @@ -827,7 +826,7 @@ impl CompactEncoding for str { impl CompactEncoding for Vec { fn encoded_size(&self) -> Result { - let mut out = usize_encoded_size(self.len()); + let mut out = encoded_size_usize(self.len()); for s in self { out += s.encoded_size()?; } @@ -859,7 +858,7 @@ impl CompactEncoding for Vec { impl CompactEncoding for Vec { fn encoded_size(&self) -> Result { - Ok(usize_encoded_size(self.len()) + self.len()) + Ok(encoded_size_usize(self.len()) + self.len()) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { @@ -977,7 +976,7 @@ impl VecEncodable for u32 { where Self: Sized, { - Ok(usize_encoded_size(vec.len()) + (vec.len() * 4)) + Ok(encoded_size_usize(vec.len()) + (vec.len() * 4)) } /// Encode `vec` to `buffer` fn vec_encode<'a>(vec: &[Self], buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> @@ -1013,7 +1012,7 @@ impl VecEncodable for [u8; N] { where Self: Sized, { - Ok(usize_encoded_size(vec.len()) + (vec.len() * N)) + Ok(encoded_size_usize(vec.len()) + (vec.len() * N)) } } @@ -1022,7 +1021,7 @@ impl BoxArrayEncodable for u8 { where Self: Sized, { - Ok(usize_encoded_size(boxed.len()) + boxed.len()) + Ok(encoded_size_usize(boxed.len()) + boxed.len()) } fn box_encode<'a>( @@ -1077,7 +1076,7 @@ mod test { } macro_rules! check_usize_var_enc_dec { ($size:expr, $value:expr) => { - let mut buffer = vec![0; usize_encoded_size($value)]; + let mut buffer = vec![0; encoded_size_usize($value)]; assert_eq!(buffer.len(), $size); let remaining = encode_usize_var(&$value, &mut buffer)?; assert!(remaining.is_empty()); diff --git a/tests/encodable.rs b/tests/encodable.rs index 218b0d3..ab739bf 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,5 +1,5 @@ use compact_encoding::{ - create_buffer, map_decode, map_encode, types::EncodingError, CompactEncoding, + create_buffer, error::EncodingError, map_decode, map_encode, CompactEncoding, }; // The max value for 1 byte length is 252 From 9dc20dfca3065c3c816230a7151308578c7ca791 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 16 Apr 2025 15:30:46 -0400 Subject: [PATCH 41/56] Add FixedWidthENcoding for uints --- src/fixedwidth.rs | 167 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 + 2 files changed, 170 insertions(+) create mode 100644 src/fixedwidth.rs diff --git a/src/fixedwidth.rs b/src/fixedwidth.rs new file mode 100644 index 0000000..f076f01 --- /dev/null +++ b/src/fixedwidth.rs @@ -0,0 +1,167 @@ +//! Allow encoding of unsigned ints in a fixed width way, instead of the default variable width. +//! +//! # Why? +//! +//! By default, unsigned integrers are variable width encoded with [`CompactEncoding`]. +//! However we sometimes want them fixed width encoded. +//! The [`FixedWidthEncoding`] lets us do this. To fixed width encode an unsigned integrer simply +//! call [`FixedWidthEncoding::as_fixed_width`] on it. Like this: +//! +//! ``` +//! # use compact_encoding::EncodingError; +//! use compact_encoding::{to_encoded_bytes, FixedWidthEncoding}; +//! let buff = to_encoded_bytes!(42u32.as_fixed_width()); +//! assert_eq!(buff, vec![42, 0, 0, 0]); +//! // vs variable width +//! let buff = to_encoded_bytes!(42u32); +//! assert_eq!(buff, vec![42]); +//! # Ok::<(), Box>(()) +//! ``` +//! +//! Likewise when decoding decoding from a fixed width encoded buffer you use +//! [`FixedWidthUint::decode`] which will decode to the underlying unsigned integer type. +//! So: `FixedWidthUint::decode(buffer) -> u32`. +//! Note that we also provide type aliases to make this more ergonomic: +//! `FixedWidthU64 = FixedWidthUint]); +//! assert_eq!(decoded, 42); // NOT! FixedWidthUint(42_u32) +//! // or using the alias +//!assert_eq!(map_decode!(&buff, [FixedWidthU32]).0.0, 42); +//! # Ok::<(), Box>(()) +//! ``` + +use crate::{ + decode_u32, decode_u64, encode_u32, encode_u64, CompactEncoding, EncodingError, U32_SIZE, + U64_SIZE, +}; + +/// Implents functionality needed to encode unisegned integrer in a fixed width way with +/// [`CompactEncoding`] +pub trait FixedWidthEncoding { + /// The type we decode to + // TODO could we just use T? + type Decode; + /// The size in bytes required to encode `self`. + fn fw_encoded_size(&self) -> Result; + + /// Encode `self` into `buffer` returning the remainder of `buffer`. + fn fw_encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError>; + + /// Decode a value from the given `buffer` of the type specified by the `Decode` type parameter + /// (which defaults to `Self`). Returns the decoded value and remaining undecoded bytes. + fn fw_decode(buffer: &[u8]) -> Result<(Self::Decode, &[u8]), EncodingError> + where + Self: Sized; + + /// Get a uint in a form that encodes to a fixed width + fn as_fixed_width(&self) -> FixedWidthUint { + FixedWidthUint(self) + } +} + +/// A fixed width encodable unsigned integer +#[derive(Debug)] +pub struct FixedWidthUint<'a, T: FixedWidthEncoding + ?Sized>(&'a T); + +impl<'t, T: FixedWidthEncoding> CompactEncoding for FixedWidthUint<'t, T> { + fn encoded_size(&self) -> Result { + self.0.fw_encoded_size() + } + + fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + self.0.fw_encode(buffer) + } + + fn decode(buffer: &[u8]) -> Result<(T::Decode, &[u8]), EncodingError> + where + Self: Sized, + { + ::fw_decode(buffer) + } +} + +impl FixedWidthEncoding for u32 { + type Decode = u32; + + fn fw_encoded_size(&self) -> Result { + Ok(U32_SIZE) + } + + fn fw_encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_u32(*self, buffer) + } + + fn fw_decode(buffer: &[u8]) -> Result<(Self::Decode, &[u8]), EncodingError> + where + Self: Sized, + { + decode_u32(buffer) + } +} +impl FixedWidthEncoding for u64 { + type Decode = u64; + + fn fw_encoded_size(&self) -> Result { + Ok(U64_SIZE) + } + + fn fw_encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { + encode_u64(*self, buffer) + } + + fn fw_decode(buffer: &[u8]) -> Result<(Self::Decode, &[u8]), EncodingError> + where + Self: Sized, + { + decode_u64(buffer) + } +} + +/// A wrapper around [`u32`] to let us encoded/decode to/from a fixed width +pub type FixedWidthU32<'a> = FixedWidthUint<'a, u32>; +/// A wrapper around [`u64`] to let us encoded/decode to/from a fixed width +pub type FixedWidthU64<'a> = FixedWidthUint<'a, u64>; +#[cfg(test)] +mod test { + use crate::{map_decode, to_encoded_bytes}; + + use super::*; + + #[test] + fn fixed_width_u32() -> Result<(), EncodingError> { + let x = 42u32; + let fixed_buff = to_encoded_bytes!(x.as_fixed_width()); + let var_buff = to_encoded_bytes!(x); + assert_eq!(fixed_buff, vec![42, 0, 0, 0]); + assert_eq!(var_buff, vec![42]); + + let ((fixed_dec,), rest) = map_decode!(&fixed_buff, [FixedWidthU32]); + assert!(rest.is_empty()); + assert_eq!(fixed_dec, x); + + let ((var_dec,), rest) = map_decode!(&var_buff, [u32]); + assert!(rest.is_empty()); + assert_eq!(var_dec, x); + Ok(()) + } + + #[test] + fn fixed_width_u64() -> Result<(), EncodingError> { + let x = 42u64; + let fixed_buff = to_encoded_bytes!(x.as_fixed_width()); + let var_buff = to_encoded_bytes!(x); + assert_eq!(fixed_buff, vec![42, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(var_buff, vec![42]); + + let ((fixed_dec,), rest) = map_decode!(&fixed_buff, [FixedWidthU64]); + assert!(rest.is_empty()); + assert_eq!(fixed_dec, x); + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 50dc023..2322b6e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -159,6 +159,8 @@ //! # Ok::<(), Box>(()) //! ``` pub mod error; +mod fixedwidth; +pub use fixedwidth::{FixedWidthEncoding, FixedWidthU32, FixedWidthU64, FixedWidthUint}; use std::{ any::type_name, net::{Ipv4Addr, Ipv6Addr}, @@ -175,6 +177,7 @@ pub const U64_SIGNIFIER: u8 = 0xff; const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; +const U64_SIZE: usize = 8; /// Implement for a type to get encoding and decoding. pub trait CompactEncoding { From eda12e1f61bf18766f736bac8aeb5807df7d938e Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 17 Apr 2025 15:00:24 -0400 Subject: [PATCH 42/56] docs & rm debug & add as_array --- src/error.rs | 2 +- src/lib.rs | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/error.rs b/src/error.rs index be8301a..e1c68e3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -4,7 +4,7 @@ use std::fmt; /// Specific type [EncodingError] #[derive(fmt::Debug, Clone, PartialEq)] pub enum EncodingErrorKind { - /// Encoding or decoding did not stay between [State] `start` and `end`. + /// Encoding or decoding did not stay within the bounds of the buffer OutOfBounds, /// Buffer data overflowed type during encoding or decoding. Overflow, diff --git a/src/lib.rs b/src/lib.rs index 2322b6e..92f6754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,7 +82,7 @@ //! } else { //! 0 //! } -//! } + dbg!(sum_encoded_size!(&self.other, &self.stuff))) +//! } + sum_encoded_size!(&self.other, &self.stuff)) //! } //! //! fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { @@ -440,6 +440,17 @@ fn get_slices_mut_checked( }) } +/// Get a slice as an array of size `N`. Errors when `slice.len() != N`. +pub fn as_array(buffer: &[u8]) -> Result<&[u8; N], EncodingError> { + let blen = buffer.len(); + if blen != N { + return Err(EncodingError::out_of_bounds(&format!( + "Could get a [{N}] byte array from a slice of length [{blen}]" + ))); + } + Ok(buffer.split_first_chunk::().expect("checked above").0) +} + /// Write `source` to `buffer` and return the remainder of `buffer`. /// Errors when `N < buffer.len()` pub fn write_array<'a, const N: usize>( From bab285f5ab4b848a0eb9fc03300d1abc95892f64 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 20 Apr 2025 20:13:47 -0400 Subject: [PATCH 43/56] dedup decode_usize --- src/lib.rs | 91 +++++++++++++++++++++++------------------------------- 1 file changed, 39 insertions(+), 52 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 92f6754..80d84d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -420,7 +420,7 @@ macro_rules! map_first { pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), EncodingError> { buffer.split_at_checked(mid).ok_or_else(|| { EncodingError::out_of_bounds(&format!( - "Cound not read [{mid}] bytes from buffer of length [{}]", + "Could not split slice at [{mid}] slice.len() = [{}]", buffer.len() )) }) @@ -428,14 +428,14 @@ pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), E /// Split a mutable slice into two mutable slices around `mid`. /// Returns encoding error when `mid` is out of bounds. -fn get_slices_mut_checked( +pub fn get_slices_mut_checked( buffer: &mut [u8], mid: usize, ) -> Result<(&mut [u8], &mut [u8]), EncodingError> { let len = buffer.len(); buffer.split_at_mut_checked(mid).ok_or_else(|| { EncodingError::out_of_bounds(&format!( - "Cound not read [{mid}] bytes from buffer of length [{len}]" + "Could not split mut slice at [{mid}] slice.len() = [{len}]" )) }) } @@ -451,6 +451,20 @@ pub fn as_array(buffer: &[u8]) -> Result<&[u8; N], EncodingError Ok(buffer.split_first_chunk::().expect("checked above").0) } +/// Get a slice as an array of size `N`. Errors when `slice.len() != N`. +pub fn as_array_mut(buffer: &mut [u8]) -> Result<&mut [u8; N], EncodingError> { + let blen = buffer.len(); + if blen != N { + return Err(EncodingError::out_of_bounds(&format!( + "Could get a [{N}] byte array from a slice of length [{blen}]" + ))); + } + Ok(buffer + .split_first_chunk_mut::() + .expect("checked above") + .0) +} + /// Write `source` to `buffer` and return the remainder of `buffer`. /// Errors when `N < buffer.len()` pub fn write_array<'a, const N: usize>( @@ -564,30 +578,21 @@ pub fn encode_var_u64(uint: u64, buffer: &mut [u8]) -> Result<&mut [u8], Encodin /// Decode a `usize` from `buffer` and return the remaining bytes. /// This will fail, when we are decoding a `usize` on a usize = u32 machine for data that was originally encoded on a `usize = u64` machine whenever the value is over `u32::MAX`. pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { - let [first, rest @ ..] = buffer else { - return Err(EncodingError::out_of_bounds( - "Colud not decode usize, empty buffer", - )); - }; - let first = *first; - if first < U16_SIGNIFIER { - Ok((first.into(), rest)) - } else if first == U16_SIGNIFIER { - let (out, rest) = decode_u16(buffer)?; - return Ok((out.into(), rest)); - } else if first == U32_SIGNIFIER { - let (out, rest) = decode_u32(buffer)?; - let out: usize = out - .try_into() - .map_err(|_e| EncodingError::overflow("u32 is bigger than usize"))?; - return Ok((out, rest)); - } else { - let (out, rest) = decode_u64(buffer)?; - let out: usize = out - .try_into() - .map_err(|_e| EncodingError::overflow("u64 is bigger than usize"))?; - return Ok((out, rest)); - } + let ([first], rest) = take_array::<1>(buffer)?; + Ok(match first { + x if x < U16_SIGNIFIER => (x.into(), rest), + U16_SIGNIFIER => map_first!(decode_u16(rest)?, |x: u16| Ok(x.into())), + U32_SIGNIFIER => { + map_first!(decode_u32(rest)?, |val| usize::try_from(val).map_err( + |_| EncodingError::overflow("Could not convert u32 to usize") + )) + } + _ => { + map_first!(decode_u64(rest)?, |val| usize::try_from(val).map_err( + |_| EncodingError::overflow("Could not convert u64 to usize") + )) + } + }) } /// Encoded a fixed sized array to a buffer @@ -641,32 +646,14 @@ fn decode_u64_var(buffer: &[u8]) -> Result<(u64, &[u8]), EncodingError> { }) } -fn decode_usize_var(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { - let ([first], rest) = take_array::<1>(buffer)?; - Ok(match first { - x if x < U16_SIGNIFIER => (x.into(), rest), - U16_SIGNIFIER => map_first!(decode_u16(rest)?, |x: u16| Ok(x.into())), - U32_SIGNIFIER => { - map_first!(decode_u32(rest)?, |val| usize::try_from(val).map_err( - |_| EncodingError::overflow("Could not convert u32 to usize") - )) - } - _ => { - map_first!(decode_u64(rest)?, |val| usize::try_from(val).map_err( - |_| EncodingError::overflow("Could not convert u64 to usize") - )) - } - }) -} - fn decode_buffer_vec(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { - let (n_bytes, rest) = decode_usize_var(buffer)?; + let (n_bytes, rest) = decode_usize(buffer)?; let (out, rest) = get_slices_checked(rest, n_bytes)?; Ok((out.to_vec(), rest)) } fn decode_string(buffer: &[u8]) -> Result<(String, &[u8]), EncodingError> { - let (len, rest) = decode_usize_var(buffer)?; + let (len, rest) = decode_usize(buffer)?; let (str_buff, rest) = get_slices_checked(rest, len)?; let out = String::from_utf8(str_buff.to_vec()) .map_err(|e| EncodingError::invalid_data(&format!("String is invalid UTF-8, {e}")))?; @@ -859,7 +846,7 @@ impl CompactEncoding for Vec { where Self: Sized, { - let (len, mut rest) = decode_usize_var(buffer)?; + let (len, mut rest) = decode_usize(buffer)?; let mut out = Vec::with_capacity(len); for _ in 0..len { let result = String::decode(rest)?; @@ -958,7 +945,7 @@ fn encode_vec<'a, T: CompactEncoding + Sized>( } fn decode_vec(buffer: &[u8]) -> Result<(Vec, &[u8]), EncodingError> { - let (len, mut rest) = decode_usize_var(buffer)?; + let (len, mut rest) = decode_usize(buffer)?; let mut out = Vec::with_capacity(len); for _ in 0..len { let res = ::decode(rest)?; @@ -1009,7 +996,7 @@ impl VecEncodable for u32 { where Self: Sized, { - let (len, mut rest) = decode_usize_var(buffer)?; + let (len, mut rest) = decode_usize(buffer)?; let mut out = Vec::with_capacity(len); for _ in 0..len { @@ -1053,7 +1040,7 @@ impl BoxArrayEncodable for u8 { where Self: Sized, { - let (len, rest) = decode_usize_var(buffer)?; + let (len, rest) = decode_usize(buffer)?; let (out, rest) = get_slices_checked(rest, len)?; Ok((out.into(), rest)) } @@ -1094,7 +1081,7 @@ mod test { assert_eq!(buffer.len(), $size); let remaining = encode_usize_var(&$value, &mut buffer)?; assert!(remaining.is_empty()); - let (result, rest) = decode_usize_var(&buffer)?; + let (result, rest) = decode_usize(&buffer)?; assert!(rest.is_empty()); assert_eq!(result, $value); }; From f3ec0fb3a6d5f2048eb64a4aa3ff882c3102effa Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sun, 20 Apr 2025 20:22:55 -0400 Subject: [PATCH 44/56] more docs and clippy --- src/fixedwidth.rs | 2 +- src/lib.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/fixedwidth.rs b/src/fixedwidth.rs index f076f01..4bdb638 100644 --- a/src/fixedwidth.rs +++ b/src/fixedwidth.rs @@ -69,7 +69,7 @@ pub trait FixedWidthEncoding { #[derive(Debug)] pub struct FixedWidthUint<'a, T: FixedWidthEncoding + ?Sized>(&'a T); -impl<'t, T: FixedWidthEncoding> CompactEncoding for FixedWidthUint<'t, T> { +impl CompactEncoding for FixedWidthUint<'_, T> { fn encoded_size(&self) -> Result { self.0.fw_encoded_size() } diff --git a/src/lib.rs b/src/lib.rs index 80d84d2..6ed2ecc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -290,12 +290,18 @@ pub trait BoxArrayEncodable: CompactEncoding { /// assert_eq!(result, 12); /// # Ok::<(), Box>(()) /// ``` +/// If you want to use this within a non-result context you can do +/// ``` +/// # use crate::compact_encoding::{sum_encoded_size, CompactEncoding, EncodingError}; +/// let bar = 42u64; +/// let result = (|| { Ok::(sum_encoded_size!(bar)) })().unwrap(); +/// assert_eq!(result, 1); +/// ``` macro_rules! sum_encoded_size { ($($val:expr),+) => {{ let out: usize = [ $( $val.encoded_size()?, - )* ].iter().sum(); out From 2ed7ae30d548c7cbb7c8811672431385f1736555 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 12:47:41 -0400 Subject: [PATCH 45/56] Add EncodingError::external --- src/error.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index e1c68e3..28f3ee7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,8 @@ pub enum EncodingErrorKind { Overflow, /// Buffer contained invalid data during decoding. InvalidData, + /// Some external error occurred causing a [`CompactEncoding`] method to fail. + External, } /// Encoding/decoding error. @@ -56,6 +58,13 @@ impl EncodingError { message: message.to_string(), } } + /// Helper function for making an invalid data error + pub fn external(message: &str) -> Self { + Self { + kind: EncodingErrorKind::External, + message: message.to_string(), + } + } } impl fmt::Display for EncodingError { @@ -64,8 +73,11 @@ impl fmt::Display for EncodingError { EncodingErrorKind::OutOfBounds => "Compact encoding failed, out of bounds", EncodingErrorKind::Overflow => "Compact encoding failed, overflow", EncodingErrorKind::InvalidData => "Compact encoding failed, invalid data", + EncodingErrorKind::External => { + "An external error caused a compact encoding operation to fail" + } }; - write!(f, "{}: {}", prefix, self.message) + write!(f, "{}: {}", prefix, self.message,) } } From 8ec77273e8e71f41cb231edd799255b8a1a4cc9d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 12:48:00 -0400 Subject: [PATCH 46/56] Add Cenc::encode/decode_with_len The same as Cenc::encode/decode except it also returns the # of bytes encoded and decoded --- src/lib.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 6ed2ecc..0c0e9f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -220,6 +220,26 @@ pub trait CompactEncoding { fn create_buffer(&self) -> Result, EncodingError> { Ok(vec![0; self.encoded_size()?]) } + + /// Like [`CompactEncoding::encode`] but also return the number of bytes encoded. + fn encode_with_len<'a>( + &self, + buffer: &'a mut [u8], + ) -> Result<(&'a mut [u8], usize), EncodingError> { + let before_len = buffer.len(); + let rest = self.encode(buffer)?; + let num_encoded_bytes = before_len - rest.len(); + Ok((rest, num_encoded_bytes)) + } + + /// Like [`CompactEncoding::decode`] but also return the number of bytes decoded. + fn decode_with_len(buffer: &[u8]) -> Result<(Decode, &[u8], usize), EncodingError> + where + Decode: Sized, + { + let (out, rest) = Self::decode(buffer)?; + Ok((out, rest, buffer.len() - rest.len())) + } } /// Implement this for type `T` to have `CompactEncoding` implemented for `Vec` From d398358bd1e5d6d2af6c09b9a7bf6ceb8e088c3b Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 12:59:23 -0400 Subject: [PATCH 47/56] simplify summing in macros --- src/lib.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0c0e9f9..6f59a85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -319,12 +319,10 @@ pub trait BoxArrayEncodable: CompactEncoding { /// ``` macro_rules! sum_encoded_size { ($($val:expr),+) => {{ - let out: usize = [ - $( - $val.encoded_size()?, - )* - ].iter().sum(); - out + 0 + $( + + $val.encoded_size()? + )* }} } @@ -343,12 +341,13 @@ macro_rules! sum_encoded_size { /// ``` macro_rules! create_buffer { ($($val:expr),+) => {{ - let len: usize = [ - $( - $val.encoded_size()?, + let len = ( + 0 + $( + + $val.encoded_size()? - )* - ].iter().sum(); + )* + ); vec![0; len] }} } From 8234211c3b5b653414ecd87f48a1bf3e6094dcba Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 13:28:50 -0400 Subject: [PATCH 48/56] Add fn for creating Box<[u8]> buffer --- src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6f59a85..855e3ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -345,7 +345,6 @@ macro_rules! create_buffer { 0 $( + $val.encoded_size()? - )* ); vec![0; len] @@ -441,6 +440,12 @@ macro_rules! map_first { (mapped, two) }}; } + +/// Returns a zerod `Box<[u8]>` where the slice is of length `encoded_size`. +pub fn fixed_buffer_from_encoded_size(encoded_size: usize) -> Box<[u8]> { + vec![0; encoded_size].into_boxed_slice() +} + /// Split a slice in two at `mid`. Returns encoding error when `mid` is out of bounds. pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), EncodingError> { buffer.split_at_checked(mid).ok_or_else(|| { From de0a24c0092eeea52b46ae1b7a7754e241db12e8 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 13:29:05 -0400 Subject: [PATCH 49/56] Fix benches to work with stuff --- benches/simple.rs | 48 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/benches/simple.rs b/benches/simple.rs index 53c00f9..a51414e 100644 --- a/benches/simple.rs +++ b/benches/simple.rs @@ -1,30 +1,27 @@ use std::time::Instant; -use compact_encoding::{CompactEncoding, State}; +use compact_encoding::{ + fixed_buffer_from_encoded_size, map_decode, map_encode, sum_encoded_size, CompactEncoding, + EncodingError, +}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; const U32_VALUE: u32 = 0xF0E1D2C3; const STR_VALUE: &str = "foo"; const U64_VALUE: u64 = u64::MAX; -fn preencode() -> State { - let mut enc_state = State::new(); - enc_state.preencode(&U32_VALUE).unwrap(); - enc_state.preencode_str(STR_VALUE).unwrap(); - enc_state.preencode(&U64_VALUE).unwrap(); - enc_state +fn preencode() -> Result { + Ok(sum_encoded_size!(U32_VALUE, STR_VALUE, U64_VALUE)) } -fn encode(enc_state: &mut State, buffer: &mut [u8]) { - enc_state.encode(&U32_VALUE, buffer).unwrap(); - enc_state.encode_str(STR_VALUE, buffer).unwrap(); - enc_state.encode(&U64_VALUE, buffer).unwrap(); +fn encode(buffer: &mut Box<[u8]>) -> Result<(), EncodingError> { + let _ = map_encode!(buffer, U32_VALUE, STR_VALUE, U64_VALUE); + Ok(()) } -fn decode(dec_state: &mut State, buffer: &[u8]) { - let _: u32 = dec_state.decode(buffer).unwrap(); - let _: String = dec_state.decode(buffer).unwrap(); - let _: u64 = dec_state.decode(buffer).unwrap(); +fn decode(buffer: &[u8]) -> Result<(), EncodingError> { + map_decode!(buffer, [u32, String, u64]); + Ok(()) } fn preencode_generic_simple(c: &mut Criterion) { @@ -36,10 +33,10 @@ fn preencode_generic_simple(c: &mut Criterion) { fn create_buffer_generic_simple(c: &mut Criterion) { c.bench_function("create buffer generic simple", |b| { b.iter_custom(|iters| { - let enc_state = preencode(); + let encoded_size = preencode().unwrap(); let start = Instant::now(); for _ in 0..iters { - black_box(enc_state.create_buffer()); + black_box(fixed_buffer_from_encoded_size(encoded_size)); } start.elapsed() }); @@ -50,13 +47,12 @@ fn create_buffer_generic_simple(c: &mut Criterion) { fn encode_generic_simple(c: &mut Criterion) { c.bench_function("encode generic simple", |b| { b.iter_custom(|iters| { - let enc_state = preencode(); - let buffer = enc_state.create_buffer(); + let encoded_size = preencode().unwrap(); + let buffer = fixed_buffer_from_encoded_size(encoded_size); let start = Instant::now(); for _ in 0..iters { - let mut enc_state = enc_state.clone(); let mut buffer = buffer.clone(); - black_box(encode(&mut enc_state, &mut buffer)); + black_box(encode(&mut buffer).unwrap()); } start.elapsed() }); @@ -67,15 +63,13 @@ fn encode_generic_simple(c: &mut Criterion) { fn decode_generic_simple(c: &mut Criterion) { c.bench_function("decode generic simple", |b| { b.iter_custom(|iters| { - let mut enc_state = preencode(); - let mut buffer = enc_state.create_buffer(); - encode(&mut enc_state, &mut buffer); - let dec_state = State::from_buffer(&buffer); + let encoded_size = preencode().unwrap(); + let mut buffer = fixed_buffer_from_encoded_size(encoded_size); + encode(&mut buffer).unwrap(); let start = Instant::now(); for _ in 0..iters { - let mut dec_state = dec_state.clone(); let buffer = buffer.clone(); - black_box(decode(&mut dec_state, &buffer)); + black_box(decode(&buffer).unwrap()); } start.elapsed() }); From 4ec35b468a1a591fb9a1a21f437f449e9221d887 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 30 Apr 2025 13:31:48 -0400 Subject: [PATCH 50/56] format doc comments code --- src/fixedwidth.rs | 6 +++--- src/lib.rs | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/fixedwidth.rs b/src/fixedwidth.rs index 4bdb638..1e309d7 100644 --- a/src/fixedwidth.rs +++ b/src/fixedwidth.rs @@ -26,13 +26,13 @@ //! //! ``` //! # use compact_encoding::EncodingError; -//! use compact_encoding::{map_decode, FixedWidthUint, FixedWidthU32}; +//! use compact_encoding::{map_decode, FixedWidthU32, FixedWidthUint}; //! let buff = vec![42, 0, 0, 0]; // 42_u32 fixed width encoded //! //! let ((decoded,), _) = map_decode!(&buff, [FixedWidthUint]); //! assert_eq!(decoded, 42); // NOT! FixedWidthUint(42_u32) -//! // or using the alias -//!assert_eq!(map_decode!(&buff, [FixedWidthU32]).0.0, 42); +//! +//! assert_eq!(map_decode!(&buff, [FixedWidthU32]).0 .0, 42); // or using the alias //! # Ok::<(), Box>(()) //! ``` diff --git a/src/lib.rs b/src/lib.rs index 855e3ed..1638c28 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,8 +61,7 @@ //! You can implement [`CompactEncoding`]` for your own structs like below: //! ``` //! use compact_encoding::{ -//! map_decode, map_encode, sum_encoded_size, to_encoded_bytes, -//! CompactEncoding, EncodingError, +//! map_decode, map_encode, sum_encoded_size, to_encoded_bytes, CompactEncoding, EncodingError, //! }; //! //! #[derive(Debug, PartialEq)] @@ -314,7 +313,7 @@ pub trait BoxArrayEncodable: CompactEncoding { /// ``` /// # use crate::compact_encoding::{sum_encoded_size, CompactEncoding, EncodingError}; /// let bar = 42u64; -/// let result = (|| { Ok::(sum_encoded_size!(bar)) })().unwrap(); +/// let result = (|| Ok::(sum_encoded_size!(bar)))().unwrap(); /// assert_eq!(result, 1); /// ``` macro_rules! sum_encoded_size { From ed621f4fda4758052fb6c64ba3b221d74b03f81a Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2025 15:32:16 -0400 Subject: [PATCH 51/56] BoxArrayEnc -> BoxedSliceEnc --- src/lib.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1638c28..6c6d5fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -265,17 +265,17 @@ pub trait VecEncodable: CompactEncoding { } } -// NB: we DO want &Box<..> because we want the trait to work for boxed things +// NB: we DO want &Box<..> because we want the trait to work for boxed things, hence clippy::allow #[allow(clippy::borrowed_box)] -/// Define this trait for `T` to get `Box<[T]>: CompactEncoding` -pub trait BoxArrayEncodable: CompactEncoding { +/// Define this trait for `T` to get `impl Box<[T]> for CompactEncoding` +pub trait BoxedSliceEncodable: CompactEncoding { /// The encoded size in bytes - fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result + fn boxed_slice_encoded_size(boxed: &Box<[Self]>) -> Result where Self: Sized; /// Encode `Box<[Self]>` to the buffer and return the remainder of the buffer - fn box_encode<'a>( + fn boxed_slice_encode<'a>( vec: &Box<[Self]>, buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> @@ -286,7 +286,7 @@ pub trait BoxArrayEncodable: CompactEncoding { } /// Decode [`Box<[Self]>`] from buffer - fn box_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + fn boxed_slice_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> where Self: Sized, { @@ -1046,15 +1046,15 @@ impl VecEncodable for [u8; N] { } } -impl BoxArrayEncodable for u8 { - fn boxed_array_encoded_size(boxed: &Box<[Self]>) -> Result +impl BoxedSliceEncodable for u8 { + fn boxed_slice_encoded_size(boxed: &Box<[Self]>) -> Result where Self: Sized, { Ok(encoded_size_usize(boxed.len()) + boxed.len()) } - fn box_encode<'a>( + fn boxed_slice_encode<'a>( boxed: &Box<[Self]>, buffer: &'a mut [u8], ) -> Result<&'a mut [u8], EncodingError> @@ -1065,7 +1065,7 @@ impl BoxArrayEncodable for u8 { write_slice(boxed, rest) } - fn box_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> + fn boxed_slice_decode(buffer: &[u8]) -> Result<(Box<[Self]>, &[u8]), EncodingError> where Self: Sized, { @@ -1075,20 +1075,20 @@ impl BoxArrayEncodable for u8 { } } -impl CompactEncoding for Box<[T]> { +impl CompactEncoding for Box<[T]> { fn encoded_size(&self) -> Result { - T::boxed_array_encoded_size(self) + T::boxed_slice_encoded_size(self) } fn encode<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a mut [u8], EncodingError> { - ::box_encode(self, buffer) + ::boxed_slice_encode(self, buffer) } fn decode(buffer: &[u8]) -> Result<(Self, &[u8]), EncodingError> where Self: Sized, { - ::box_decode(buffer) + ::boxed_slice_decode(buffer) } } From 329bf32a4ed7a316e1e37ec0925e1b0a43d79344 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2025 15:43:32 -0400 Subject: [PATCH 52/56] Use Box<[u8]> for buffer instead of Vec --- src/lib.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6c6d5fc..cf23b77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -202,8 +202,8 @@ pub trait CompactEncoding { /// foo.encode(&mut buff)?; /// # Ok::<(), Box>(()) /// ``` - fn to_encoded_bytes(&self) -> Result, EncodingError> { - let mut buff = vec![0; self.encoded_size()?]; + fn to_encoded_bytes(&self) -> Result, EncodingError> { + let mut buff = self.create_buffer()?; self.encode(&mut buff)?; Ok(buff) } @@ -216,8 +216,8 @@ pub trait CompactEncoding { /// vec![0; foo.encoded_size()?]; /// # Ok::<(), Box>(()) /// ``` - fn create_buffer(&self) -> Result, EncodingError> { - Ok(vec![0; self.encoded_size()?]) + fn create_buffer(&self) -> Result, EncodingError> { + Ok(vec![0; self.encoded_size()?].into_boxed_slice()) } /// Like [`CompactEncoding::encode`] but also return the number of bytes encoded. @@ -440,11 +440,6 @@ macro_rules! map_first { }}; } -/// Returns a zerod `Box<[u8]>` where the slice is of length `encoded_size`. -pub fn fixed_buffer_from_encoded_size(encoded_size: usize) -> Box<[u8]> { - vec![0; encoded_size].into_boxed_slice() -} - /// Split a slice in two at `mid`. Returns encoding error when `mid` is out of bounds. pub fn get_slices_checked(buffer: &[u8], mid: usize) -> Result<(&[u8], &[u8]), EncodingError> { buffer.split_at_checked(mid).ok_or_else(|| { From 70471c5cc2c2ba318607888fc578b6d779501adf Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2025 16:10:33 -0400 Subject: [PATCH 53/56] better doc comments --- benches/simple.rs | 11 +++++++---- src/error.rs | 2 +- src/fixedwidth.rs | 23 ++++++++++------------- src/lib.rs | 38 ++++++++++++++++++++------------------ 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/benches/simple.rs b/benches/simple.rs index a51414e..6072a59 100644 --- a/benches/simple.rs +++ b/benches/simple.rs @@ -1,8 +1,7 @@ use std::time::Instant; use compact_encoding::{ - fixed_buffer_from_encoded_size, map_decode, map_encode, sum_encoded_size, CompactEncoding, - EncodingError, + create_buffer, map_decode, map_encode, sum_encoded_size, CompactEncoding, EncodingError, }; use criterion::{black_box, criterion_group, criterion_main, Criterion}; @@ -24,6 +23,10 @@ fn decode(buffer: &[u8]) -> Result<(), EncodingError> { Ok(()) } +fn create_buffer(encoded_size: usize) -> Box<[u8]> { + vec![0; encoded_size].into_boxed_slice() +} + fn preencode_generic_simple(c: &mut Criterion) { c.bench_function("preencode generic simple", |b| { b.iter(preencode); @@ -36,7 +39,7 @@ fn create_buffer_generic_simple(c: &mut Criterion) { let encoded_size = preencode().unwrap(); let start = Instant::now(); for _ in 0..iters { - black_box(fixed_buffer_from_encoded_size(encoded_size)); + black_box(create_buffer(encoded_size)); } start.elapsed() }); @@ -48,7 +51,7 @@ fn encode_generic_simple(c: &mut Criterion) { c.bench_function("encode generic simple", |b| { b.iter_custom(|iters| { let encoded_size = preencode().unwrap(); - let buffer = fixed_buffer_from_encoded_size(encoded_size); + let buffer = create_buffer(encoded_size); let start = Instant::now(); for _ in 0..iters { let mut buffer = buffer.clone(); diff --git a/src/error.rs b/src/error.rs index 28f3ee7..04b8f4a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,7 +10,7 @@ pub enum EncodingErrorKind { Overflow, /// Buffer contained invalid data during decoding. InvalidData, - /// Some external error occurred causing a [`CompactEncoding`] method to fail. + /// Some external error occurred causing a [`crate::CompactEncoding`] method to fail. External, } diff --git a/src/fixedwidth.rs b/src/fixedwidth.rs index 1e309d7..4ed4819 100644 --- a/src/fixedwidth.rs +++ b/src/fixedwidth.rs @@ -1,20 +1,17 @@ //! Allow encoding of unsigned ints in a fixed width way, instead of the default variable width. //! -//! # Why? -//! -//! By default, unsigned integrers are variable width encoded with [`CompactEncoding`]. -//! However we sometimes want them fixed width encoded. -//! The [`FixedWidthEncoding`] lets us do this. To fixed width encode an unsigned integrer simply -//! call [`FixedWidthEncoding::as_fixed_width`] on it. Like this: -//! +//! Why? Because the default [`CompactEncoding`] implementation for unsigned integers uses a +//! variable width encoding. However sometimes want them encoded with a fixed width, +//! [`FixedWidthEncoding`] lets us do this. To fixed width encode an unsigned integrer simply call +//! [`FixedWidthEncoding::as_fixed_width`] on it. Like this: //! ``` //! # use compact_encoding::EncodingError; //! use compact_encoding::{to_encoded_bytes, FixedWidthEncoding}; //! let buff = to_encoded_bytes!(42u32.as_fixed_width()); -//! assert_eq!(buff, vec![42, 0, 0, 0]); +//! assert_eq!(buff, [42, 0, 0, 0].into()); //! // vs variable width //! let buff = to_encoded_bytes!(42u32); -//! assert_eq!(buff, vec![42]); +//! assert_eq!(buff, [42].into()); //! # Ok::<(), Box>(()) //! ``` //! @@ -138,8 +135,8 @@ mod test { let x = 42u32; let fixed_buff = to_encoded_bytes!(x.as_fixed_width()); let var_buff = to_encoded_bytes!(x); - assert_eq!(fixed_buff, vec![42, 0, 0, 0]); - assert_eq!(var_buff, vec![42]); + assert_eq!(fixed_buff, [42_u8, 0, 0, 0].into()); + assert_eq!(var_buff, [42_u8].into()); let ((fixed_dec,), rest) = map_decode!(&fixed_buff, [FixedWidthU32]); assert!(rest.is_empty()); @@ -156,8 +153,8 @@ mod test { let x = 42u64; let fixed_buff = to_encoded_bytes!(x.as_fixed_width()); let var_buff = to_encoded_bytes!(x); - assert_eq!(fixed_buff, vec![42, 0, 0, 0, 0, 0, 0, 0]); - assert_eq!(var_buff, vec![42]); + assert_eq!(fixed_buff, [42, 0, 0, 0, 0, 0, 0, 0].into()); + assert_eq!(var_buff, [42].into()); let ((fixed_dec,), rest) = map_decode!(&fixed_buff, [FixedWidthU64]); assert!(rest.is_empty()); diff --git a/src/lib.rs b/src/lib.rs index cf23b77..fbbb649 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,11 +4,12 @@ //! # Series of compact encoding schemes for building small and fast parsers and serializers //! //! Binary compatible with the -//! [original Javascript compact-encoding library](https://github.com/compact-encoding/compact-encoding/). +//! [original JavaScript compact-encoding library](https://github.com/compact-encoding/compact-encoding/). //! //! ## Usage //! -//! ### Quick start +//! The simplest way to encoded and decode a some data is using the [`to_encoded_bytes`] and +//! [`map_decode`] macros: //! ``` //! use compact_encoding::{map_decode, to_encoded_bytes}; //! @@ -23,32 +24,33 @@ //! assert_eq!(word_dec, word); //! # Ok::<(), Box>(()) //! ``` -//! ### Manually implement encoding and decoding +//! ### Manual encoding and decoding //! -//! This example shows how to manually calculate the needed buffer size, create the buffer, encode -//! data, and decode it. Using only the types which include a default [`CompactEncoding`] -//! implementation. A more ergonomic pattern is demonstrated in other examples +//! When more fine-grained control of encoding and decoding is needed you manually do each step of +//! encoding and decoding like in the following example, where we want to use a fixed width +//! encoding for `number` (instead of the default variable width encoding). It shows how to +//! manually calculate the needed buffer size, create the buffer, encode data, and decode it. //! ``` -//! use compact_encoding::CompactEncoding; +//! use compact_encoding::{CompactEncoding, FixedWidthEncoding, FixedWidthU32}; //! //! let number = 41_u32; //! let word = "hi"; //! -//! // Use `encoded_size` to figure out how big a buffer should be. -//! let size = number.encoded_size()? + word.encoded_size()?; +//! // Use `encoded_size` to figure out how big a buffer should be +//! let size = number.as_fixed_width().encoded_size()? + word.encoded_size()?; //! //! // Create a buffer with the calculated size //! let mut buffer = vec![0; size]; -//! assert_eq!(buffer.len(), 1 + 1 + 2); +//! assert_eq!(buffer.len(), 4 + 1 + 2); //! //! // Then actually encode the values -//! let mut remaining_buffer = number.encode(&mut buffer)?; +//! let mut remaining_buffer = number.as_fixed_width().encode(&mut buffer)?; //! remaining_buffer = word.encode(remaining_buffer)?; //! assert!(remaining_buffer.is_empty()); -//! assert_eq!(buffer.to_vec(), vec![41_u8, 2_u8, b'h', b'i']); +//! assert_eq!(buffer.to_vec(), vec![41_u8, 0, 0, 0, 2_u8, b'h', b'i']); //! //! // `buffer` now contains all the encoded data, and we can decode from it -//! let (number_dec, remaining_buffer) = u32::decode(&buffer)?; +//! let (number_dec, remaining_buffer) = FixedWidthU32::decode(&buffer)?; //! let (word_dec, remaining_buffer) = String::decode(remaining_buffer)?; //! assert!(remaining_buffer.is_empty()); //! assert_eq!(number_dec, number); @@ -56,9 +58,9 @@ //! # Ok::<(), Box>(()) //! ``` //! -//! ### Implementing CompactEncoding for new types +//! ### Implementing CompactEncoding for custom types //! -//! You can implement [`CompactEncoding`]` for your own structs like below: +//! Here we demonstrate how to implement [`CompactEncoding`] for a custom struct. //! ``` //! use compact_encoding::{ //! map_decode, map_encode, sum_encoded_size, to_encoded_bytes, CompactEncoding, EncodingError, @@ -346,7 +348,7 @@ macro_rules! create_buffer { + $val.encoded_size()? )* ); - vec![0; len] + vec![0; len].into_boxed_slice() }} } @@ -360,7 +362,7 @@ macro_rules! create_buffer { /// let mut buff = create_buffer!(num, word); /// let result = map_encode!(&mut buff, num, word); /// assert!(result.is_empty()); -/// assert_eq!(&buff, &[42, 2, b'y', b'o']); +/// assert_eq!(&*buff, &[42, 2, b'y', b'o']); /// # Ok::<(), Box>(()) /// ``` macro_rules! map_encode { @@ -384,7 +386,7 @@ macro_rules! map_encode { /// ``` /// # use crate::compact_encoding::to_encoded_bytes; /// let result = to_encoded_bytes!(42u64, "yo"); -/// assert_eq!(&result, &[42, 2, 121, 111]); +/// assert_eq!(&*result, &[42, 2, 121, 111]); /// # Ok::<(), Box>(()) /// ``` macro_rules! to_encoded_bytes { From 1289a939f728d66001944dea580e929a41a038a7 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 17:09:50 -0400 Subject: [PATCH 54/56] fix benches --- benches/simple.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/benches/simple.rs b/benches/simple.rs index 6072a59..59bb719 100644 --- a/benches/simple.rs +++ b/benches/simple.rs @@ -1,8 +1,6 @@ use std::time::Instant; -use compact_encoding::{ - create_buffer, map_decode, map_encode, sum_encoded_size, CompactEncoding, EncodingError, -}; +use compact_encoding::{map_decode, map_encode, sum_encoded_size, CompactEncoding, EncodingError}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; const U32_VALUE: u32 = 0xF0E1D2C3; @@ -13,7 +11,7 @@ fn preencode() -> Result { Ok(sum_encoded_size!(U32_VALUE, STR_VALUE, U64_VALUE)) } -fn encode(buffer: &mut Box<[u8]>) -> Result<(), EncodingError> { +fn encode(buffer: &mut [u8]) -> Result<(), EncodingError> { let _ = map_encode!(buffer, U32_VALUE, STR_VALUE, U64_VALUE); Ok(()) } @@ -67,7 +65,7 @@ fn decode_generic_simple(c: &mut Criterion) { c.bench_function("decode generic simple", |b| { b.iter_custom(|iters| { let encoded_size = preencode().unwrap(); - let mut buffer = fixed_buffer_from_encoded_size(encoded_size); + let mut buffer = vec![0_u8; encoded_size]; encode(&mut buffer).unwrap(); let start = Instant::now(); for _ in 0..iters { From a7f67ac4093d6ad92fd0c29ba4e1b2376d335fde Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 17:14:17 -0400 Subject: [PATCH 55/56] more doc tests --- src/lib.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fbbb649..34e3ebc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -180,7 +180,7 @@ const U16_SIZE: usize = 2; const U32_SIZE: usize = 4; const U64_SIZE: usize = 8; -/// Implement for a type to get encoding and decoding. +/// A trait for building small and fast parsers and serializers. pub trait CompactEncoding { /// The size in bytes required to encode `self`. fn encoded_size(&self) -> Result; @@ -477,7 +477,7 @@ pub fn as_array(buffer: &[u8]) -> Result<&[u8; N], EncodingError Ok(buffer.split_first_chunk::().expect("checked above").0) } -/// Get a slice as an array of size `N`. Errors when `slice.len() != N`. +/// Get a slice as a mutable array of size `N`. Errors when `slice.len() != N`. pub fn as_array_mut(buffer: &mut [u8]) -> Result<&mut [u8; N], EncodingError> { let blen = buffer.len(); if blen != N { @@ -621,7 +621,16 @@ pub fn decode_usize(buffer: &[u8]) -> Result<(usize, &[u8]), EncodingError> { }) } -/// Encoded a fixed sized array to a buffer +/// Encoded a fixed sized array to a buffer, return the remainder of the buffer. +/// Errors when `value.len() > buffer.len()`; +/// ``` +/// # use compact_encoding::encode_bytes_fixed; +/// let mut buffer = vec![0; 3]; +/// let rest = encode_bytes_fixed(&[4, 2], &mut buffer)?; +/// assert_eq!(rest, &[0]); +/// assert_eq!(buffer, &[4, 2, 0]); +/// # Ok::<(), Box>(()) +/// ``` pub fn encode_bytes_fixed<'a, const N: usize>( value: &[u8; N], buffer: &'a mut [u8], @@ -629,7 +638,16 @@ pub fn encode_bytes_fixed<'a, const N: usize>( write_array(value, buffer) } -/// Encoded a fixed sized array to a buffer +/// Decode a fixed sized array from a buffer. Return the array and the remainder of the buffer. +/// Errors when `buffer.len() < N`; +/// ``` +/// # use compact_encoding::decode_bytes_fixed; +/// let mut buffer = vec![1, 2, 3]; +/// let (arr, rest) = decode_bytes_fixed::<2>(&mut buffer)?; +/// assert_eq!(arr, [1, 2]); +/// assert_eq!(rest, &[3]); +/// # Ok::<(), Box>(()) +/// ``` pub fn decode_bytes_fixed( buffer: &[u8], ) -> Result<([u8; N], &[u8]), EncodingError> { From d1d597f81d68304ab6e5c8532ae32565f78dc7b9 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2025 18:01:47 -0400 Subject: [PATCH 56/56] don't re-export error module --- src/lib.rs | 2 +- tests/encodable.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 34e3ebc..e57eda6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -159,7 +159,7 @@ //! assert_eq!(bar_dec, bar); //! # Ok::<(), Box>(()) //! ``` -pub mod error; +mod error; mod fixedwidth; pub use fixedwidth::{FixedWidthEncoding, FixedWidthU32, FixedWidthU64, FixedWidthUint}; use std::{ diff --git a/tests/encodable.rs b/tests/encodable.rs index ab739bf..caefa88 100644 --- a/tests/encodable.rs +++ b/tests/encodable.rs @@ -1,6 +1,4 @@ -use compact_encoding::{ - create_buffer, error::EncodingError, map_decode, map_encode, CompactEncoding, -}; +use compact_encoding::{create_buffer, map_decode, map_encode, CompactEncoding, EncodingError}; // The max value for 1 byte length is 252 const MAX_ONE_BYTE_UINT: u8 = 252;