Skip to content

blobby: replace iterators with const fn parsing #1187

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions blobby/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ categories = ["no-std"]
edition = "2024"
rust-version = "1.85"
readme = "README.md"

[features]
alloc = []
68 changes: 43 additions & 25 deletions blobby/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,47 @@ Iterators over a simple binary blob storage.

## Examples
```
let buf = b"\x02\x05hello\x06world!\x01\x02 \x00\x03\x06:::\x03\x01\x00";
let mut v = blobby::BlobIterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok(&b"hello"[..])));
assert_eq!(v.next(), Some(Ok(&b" "[..])));
assert_eq!(v.next(), Some(Ok(&b""[..])));
assert_eq!(v.next(), Some(Ok(&b"world!"[..])));
assert_eq!(v.next(), Some(Ok(&b":::"[..])));
assert_eq!(v.next(), Some(Ok(&b"world!"[..])));
assert_eq!(v.next(), Some(Ok(&b"hello"[..])));
assert_eq!(v.next(), Some(Ok(&b""[..])));
assert_eq!(v.next(), None);

let mut v = blobby::Blob2Iterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b" "])));
assert_eq!(v.next(), Some(Ok([&b""[..], b"world!"])));
assert_eq!(v.next(), Some(Ok([&b":::"[..], b"world!"])));
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b""])));
assert_eq!(v.next(), None);

let mut v = blobby::Blob4Iterator::new(buf).unwrap();
assert_eq!(v.next(), Some(Ok([&b"hello"[..], b" ", b"", b"world!"])));
assert_eq!(v.next(), Some(Ok([&b":::"[..], b"world!", b"hello", b""])));
assert_eq!(v.next(), None);
// We recommend to save blobby data into separate files and
// use the `include_bytes!` macro
static BLOBBY_DATA: &[u8] = b"\x02\x05hello\x06world!\x01\x02 \x00\x03\x06:::\x03\x01\x00";

static SLICE: &[&[u8]] = blobby::parse_into_slice!(BLOBBY_DATA);

assert_eq!(SLICE[0], b"hello".as_slice());
assert_eq!(SLICE[1], b" ".as_slice());
assert_eq!(SLICE[2], b"".as_slice());
assert_eq!(SLICE[3], b"world!".as_slice());
assert_eq!(SLICE[4], b":::".as_slice());
assert_eq!(SLICE[5], b"world!".as_slice());
assert_eq!(SLICE[6], b"hello".as_slice());
assert_eq!(SLICE[7], b"".as_slice());
assert_eq!(SLICE.len(), 8);

blobby::parse_into_structs!(
BLOBBY_DATA;
#[define_struct]
static ITEMS: &[Item { a, b, c, d }];
);

assert_eq!(
ITEMS[0],
Item {
a: b"hello",
b: b" ",
c: b"",
d: b"world!",
},
);
assert_eq!(
ITEMS[1],
Item {
a: b":::",
b: b"world!",
c: b"hello",
d: b"",
},
);
assert_eq!(ITEMS.len(), 2);
```

## Encoding and decoding
Expand Down Expand Up @@ -76,14 +94,14 @@ with `\n`).

This file can be converted to the Blobby format by running the following command:
```sh
cargo run --releae --bin encode -- /path/to/input.txt /path/to/output.blb
cargo run --release --features alloc --bin encode -- /path/to/input.txt /path/to/output.blb
```

This will create a file which can be read using `blobby::Blob2Iterator`.

To see contents of an existing Blobby file you can use the following command:
```sh
cargo run --releae --bin decode -- /path/to/input.blb /path/to/output.txt
cargo run --release --features alloc --bin decode -- /path/to/input.blb /path/to/output.txt
```
The output file will contain a sequence of hex-encoded byte strings stored
in the input file.
Expand Down
56 changes: 28 additions & 28 deletions blobby/src/bin/decode.rs
Original file line number Diff line number Diff line change
@@ -1,41 +1,41 @@
//! Encoding utility
use blobby::BlobIterator;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, error::Error, fs::File};

fn encode_hex(data: &[u8]) -> String {
let mut res = String::with_capacity(2 * data.len());
for &byte in data {
res.push_str(&format!("{byte:02X}"));
}
res
use std::error::Error;

#[cfg(not(feature = "alloc"))]
fn main() -> Result<(), Box<dyn Error>> {
Err("The decode binary should be compiled with enabled `alloc` feature!".into())
}

fn decode<R: BufRead, W: Write>(mut reader: R, mut writer: W) -> io::Result<usize> {
let mut data = Vec::new();
reader.read_to_end(&mut data)?;
let res = BlobIterator::new(&data)
.map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid blobby data: {e:?}"),
)
})?
.collect::<Vec<_>>();
for blob in res.iter() {
let blob = blob.map_err(|e| {
#[cfg(feature = "alloc")]
fn main() -> Result<(), Box<dyn Error>> {
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, fs::File};

fn encode_hex(data: &[u8]) -> String {
let mut res = String::with_capacity(2 * data.len());
for &byte in data {
res.push_str(&format!("{byte:02X}"));
}
res
}

fn decode<R: BufRead, W: Write>(mut reader: R, mut writer: W) -> io::Result<usize> {
let mut data = Vec::new();
reader.read_to_end(&mut data)?;
let res = blobby::parse_into_vec(&data).map_err(|e| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid blobby data: {e:?}"),
)
})?;
writer.write_all(encode_hex(blob).as_bytes())?;
writer.write_all(b"\n")?;
let len = res.len();
for blob in res {
writer.write_all(encode_hex(blob).as_bytes())?;
writer.write_all(b"\n")?;
}
Ok(len)
}
Ok(res.len())
}

fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().skip(1).collect();

if args.is_empty() {
Expand Down
84 changes: 46 additions & 38 deletions blobby/src/bin/encode.rs
Original file line number Diff line number Diff line change
@@ -1,49 +1,57 @@
//! Encoding utility
use blobby::encode_blobs;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, error::Error, fs::File};
use std::error::Error;

fn decode_hex_char(b: u8) -> io::Result<u8> {
let res = match b {
b'0'..=b'9' => b - b'0',
b'a'..=b'f' => b - b'a' + 10,
b'A'..=b'F' => b - b'A' + 10,
_ => {
let msg = "Invalid hex string: invalid byte {b}";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
};
Ok(res)
#[cfg(not(feature = "alloc"))]
fn main() -> Result<(), Box<dyn Error>> {
Err("The encode binary should be compiled with enabled `alloc` feature!".into())
}

fn decode_hex(data: &str) -> io::Result<Vec<u8>> {
if data.len() % 2 != 0 {
let msg = "Invalid hex string: length is not even";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
#[cfg(feature = "alloc")]
fn main() -> Result<(), Box<dyn Error>> {
use blobby::encode_blobs;
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::{env, fs::File};

fn decode_hex_char(b: u8) -> io::Result<u8> {
let res = match b {
b'0'..=b'9' => b - b'0',
b'a'..=b'f' => b - b'a' + 10,
b'A'..=b'F' => b - b'A' + 10,
_ => {
let msg = "Invalid hex string: invalid byte {b}";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
};
Ok(res)
}
data.as_bytes()
.chunks_exact(2)
.map(|chunk| {
let a = decode_hex_char(chunk[0])?;
let b = decode_hex_char(chunk[1])?;
Ok((a << 4) | b)
})
.collect()
}

fn encode(reader: impl BufRead, mut writer: impl Write) -> io::Result<usize> {
let mut blobs = Vec::new();
for line in reader.lines() {
let blob = decode_hex(&line?)?;
blobs.push(blob);
fn decode_hex(data: &str) -> io::Result<Vec<u8>> {
if data.len() % 2 != 0 {
let msg = "Invalid hex string: length is not even";
return Err(io::Error::new(io::ErrorKind::InvalidData, msg));
}
data.as_bytes()
.chunks_exact(2)
.map(|chunk| {
let a = decode_hex_char(chunk[0])?;
let b = decode_hex_char(chunk[1])?;
Ok((a << 4) | b)
})
.collect()
}

fn encode(reader: impl BufRead, mut writer: impl Write) -> io::Result<usize> {
let mut blobs = Vec::new();
for line in reader.lines() {
let blob = decode_hex(&line?)?;
blobs.push(blob);
}
let (data, idx_len) = encode_blobs(&blobs);
println!("Index len: {idx_len:?}");
writer.write_all(&data)?;
Ok(blobs.len())
}
let (data, idx_len) = encode_blobs(&blobs);
println!("Index len: {idx_len:?}");
writer.write_all(&data)?;
Ok(blobs.len())
}

fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().skip(1).collect();

if args.is_empty() {
Expand Down
Loading