Skip to content

Commit

Permalink
re-export FrameEncoder under encoding:: and FrameDecompressor/Streami…
Browse files Browse the repository at this point in the history
…ngDecompressor und decoding::
  • Loading branch information
KillingSpark committed Dec 20, 2024
1 parent 6f0374c commit 8de2364
Show file tree
Hide file tree
Showing 18 changed files with 82 additions and 84 deletions.
4 changes: 2 additions & 2 deletions Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Measuring with the 'time' utility the original zstd and my decoder both decoding
The easiest is to use the provided `compress`/`compress_to_vec` functions

```rust, no_run
use ruzstd::encoding::{compress, compress_to_vec, frame_compressor::CompressionLevel};
use ruzstd::encoding::{compress, compress_to_vec, CompressionLevel};
let data: &[u8] = todo!();
// Either
let mut compressed = Vec::new();
Expand All @@ -60,7 +60,7 @@ Additionally to the descriptions and the docs you can have a look at the zstd /
The easiest is to wrap the io::Read into a StreamingDecoder which itself implements io::Read. It will decode blocks as necessary to fulfill the read requests

```rust, no_run
use ruzstd::decoding::streaming_decoder::StreamingDecoder;
use ruzstd::decoding::StreamingDecoder;
use ruzstd::io::Read;
let mut source: &[u8] = todo!("Get a reader from a File or any other source");
Expand Down
2 changes: 1 addition & 1 deletion benches/decode_all.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use criterion::{criterion_group, criterion_main, Criterion};
use ruzstd::decoding::frame_decoder::FrameDecoder;
use ruzstd::decoding::FrameDecoder;

fn criterion_benchmark(c: &mut Criterion) {
let mut fr = FrameDecoder::new();
Expand Down
2 changes: 1 addition & 1 deletion fuzz/fuzz_targets/decode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ extern crate ruzstd;
use std::io::Read;

fuzz_target!(|data: &[u8]| {
if let Ok(mut decoder) = ruzstd::decoding::streaming_decoder::StreamingDecoder::new(data) {
if let Ok(mut decoder) = ruzstd::decoding::StreamingDecoder::new(data) {
let mut output = Vec::new();
_ = decoder.read_to_end(&mut output);
}
Expand Down
6 changes: 3 additions & 3 deletions fuzz/fuzz_targets/encode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
#[macro_use]
extern crate libfuzzer_sys;
extern crate ruzstd;
use ruzstd::encoding::frame_compressor::{CompressionLevel, FrameCompressor};
use ruzstd::encoding::{CompressionLevel, FrameCompressor};

fuzz_target!(|data: &[u8]| {
let mut output = Vec::new();
let mut compressor = FrameCompressor::new(data, &mut output, CompressionLevel::Uncompressed);
compressor.compress();

let mut decoded = Vec::with_capacity(data.len());
let mut decoder = ruzstd::decoding::frame_decoder::FrameDecoder::new();
let mut decoder = ruzstd::decoding::FrameDecoder::new();
decoder.decode_all_to_vec(&output, &mut decoded).unwrap();
assert_eq!(data, &decoded);

Expand All @@ -19,7 +19,7 @@ fuzz_target!(|data: &[u8]| {
compressor.compress();

let mut decoded = Vec::with_capacity(data.len());
let mut decoder = ruzstd::decoding::frame_decoder::FrameDecoder::new();
let mut decoder = ruzstd::decoding::FrameDecoder::new();
decoder.decode_all_to_vec(&output, &mut decoded).unwrap();
assert_eq!(data, &decoded);
});
14 changes: 7 additions & 7 deletions fuzz/fuzz_targets/interop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,21 @@ extern crate ruzstd;
use std::io::Read;

fn decode_ruzstd(data: &mut dyn std::io::Read) -> Vec<u8> {
let mut decoder = ruzstd::decoding::streaming_decoder::StreamingDecoder::new(data).unwrap();
let mut decoder = ruzstd::decoding::StreamingDecoder::new(data).unwrap();
let mut result: Vec<u8> = Vec::new();
decoder.read_to_end(&mut result).expect("Decoding failed");
result
}

fn decode_ruzstd_writer(mut data: impl Read) -> Vec<u8> {
let mut decoder = ruzstd::decoding::frame_decoder::FrameDecoder::new();
let mut decoder = ruzstd::decoding::FrameDecoder::new();
decoder.reset(&mut data).unwrap();
let mut result = vec![];
while !decoder.is_finished() || decoder.can_collect() > 0 {
decoder
.decode_blocks(
&mut data,
ruzstd::decoding::frame_decoder::BlockDecodingStrategy::UptoBytes(1024 * 1024),
ruzstd::decoding::BlockDecodingStrategy::UptoBytes(1024 * 1024),
)
.unwrap();
decoder.collect_to_writer(&mut result).unwrap();
Expand All @@ -35,10 +35,10 @@ fn encode_ruzstd_uncompressed(data: &mut dyn std::io::Read) -> Vec<u8> {
let mut input = Vec::new();
let mut output = Vec::new();
data.read_to_end(&mut input).unwrap();
let mut compressor = ruzstd::encoding::frame_compressor::FrameCompressor::new(
let mut compressor = ruzstd::encoding::FrameCompressor::new(
input.as_slice(),
&mut output,
ruzstd::encoding::frame_compressor::CompressionLevel::Uncompressed,
ruzstd::encoding::CompressionLevel::Uncompressed,
);
compressor.compress();
output
Expand All @@ -48,10 +48,10 @@ fn encode_ruzstd_compressed(data: &mut dyn std::io::Read) -> Vec<u8> {
let mut input = Vec::new();
let mut output = Vec::new();
data.read_to_end(&mut input).unwrap();
let mut compressor = ruzstd::encoding::frame_compressor::FrameCompressor::new(
let mut compressor = ruzstd::encoding::FrameCompressor::new(
input.as_slice(),
&mut output,
ruzstd::encoding::frame_compressor::CompressionLevel::Fastest,
ruzstd::encoding::CompressionLevel::Fastest,
);
compressor.compress();
output
Expand Down
10 changes: 4 additions & 6 deletions src/bin/zstd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ use std::time::Instant;

use ruzstd::decoding::errors::FrameDecoderError;
use ruzstd::decoding::errors::ReadFrameHeaderError;
use ruzstd::encoding::frame_compressor::CompressionLevel;
use ruzstd::encoding::frame_compressor::FrameCompressor;
use ruzstd::encoding::CompressionLevel;
use ruzstd::encoding::FrameCompressor;

struct StateTracker {
bytes_used: u64,
Expand Down Expand Up @@ -41,7 +41,7 @@ fn decompress(flags: &[String], file_paths: &[String]) {
return;
}

let mut frame_dec = ruzstd::decoding::frame_decoder::FrameDecoder::new();
let mut frame_dec = ruzstd::decoding::FrameDecoder::new();

for path in file_paths {
eprintln!("File: {}", path);
Expand Down Expand Up @@ -81,9 +81,7 @@ fn decompress(flags: &[String], file_paths: &[String]) {
frame_dec
.decode_blocks(
&mut f,
ruzstd::decoding::frame_decoder::BlockDecodingStrategy::UptoBytes(
batch_size,
),
ruzstd::decoding::BlockDecodingStrategy::UptoBytes(batch_size),
)
.unwrap();

Expand Down
3 changes: 1 addition & 2 deletions src/bin/zstd_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ fn main() {
let f = File::open(path).unwrap();
let mut buf_read = std::io::BufReader::new(f);

let mut decoder =
ruzstd::decoding::streaming_decoder::StreamingDecoder::new(&mut buf_read).unwrap();
let mut decoder = ruzstd::decoding::StreamingDecoder::new(&mut buf_read).unwrap();
let mut buf = [0u8; 1024 * 1024];
let mut stdout = std::io::stdout();
while !decoder.decoder.is_finished() || decoder.decoder.can_collect() > 0 {
Expand Down
6 changes: 3 additions & 3 deletions src/decoding/frame_decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ use core::convert::TryInto;
/// It reads bytes as needed from a provided source and can be read from to collect partial results.
///
/// If you want to just read the whole frame with an `io::Read` without having to deal with manually calling [FrameDecoder::decode_blocks]
/// you can use the provided [crate::decoding::streaming_decoder::StreamingDecoder] wich wraps this FrameDecoder.
/// you can use the provided [crate::decoding::StreamingDecoder] wich wraps this FrameDecoder.
///
/// Workflow is as follows:
/// ```
/// use ruzstd::decoding::frame_decoder::BlockDecodingStrategy;
/// use ruzstd::decoding::BlockDecodingStrategy;
///
/// # #[cfg(feature = "std")]
/// use std::io::{Read, Write};
Expand All @@ -36,7 +36,7 @@ use core::convert::TryInto;
///
/// fn decode_this(mut file: impl Read) {
/// //Create a new decoder
/// let mut frame_dec = ruzstd::decoding::frame_decoder::FrameDecoder::new();
/// let mut frame_dec = ruzstd::decoding::FrameDecoder::new();
/// let mut result = Vec::new();
///
/// // Use reset or init to make the decoder ready to decode the frame from the io::Read
Expand Down
7 changes: 5 additions & 2 deletions src/decoding/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
//! Structures and utilities used for decoding zstd formatted data
pub mod errors;
pub mod frame_decoder;
pub mod streaming_decoder;
mod frame_decoder;
mod streaming_decoder;

pub use frame_decoder::{BlockDecodingStrategy, FrameDecoder};
pub use streaming_decoder::StreamingDecoder;

pub(crate) mod bit_reader;
pub(crate) mod bit_reader_reverse;
Expand Down
4 changes: 2 additions & 2 deletions src/decoding/streaming_decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use core::borrow::BorrowMut;

use crate::decoding::errors::FrameDecoderError;
use crate::decoding::frame_decoder::{BlockDecodingStrategy, FrameDecoder};
use crate::decoding::{BlockDecodingStrategy, FrameDecoder};
use crate::io::{Error, ErrorKind, Read};

/// High level Zstandard frame decoder that can be used to decompress a given Zstandard frame.
Expand Down Expand Up @@ -31,7 +31,7 @@ use crate::io::{Error, ErrorKind, Read};
/// {
/// use std::fs::File;
/// use std::io::Read;
/// use ruzstd::decoding::streaming_decoder::StreamingDecoder;
/// use ruzstd::decoding::StreamingDecoder;
///
/// // Read a Zstandard archive from the filesystem then decompress it into a vec.
/// let mut f: File = todo!("Read a .zstd archive from somewhere");
Expand Down
13 changes: 5 additions & 8 deletions src/encoding/frame_compressor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ pub enum CompressionLevel {
///
/// # Examples
/// ```
/// use ruzstd::encoding::frame_compressor::{FrameCompressor, CompressionLevel};
/// use ruzstd::encoding::{FrameCompressor, CompressionLevel};
/// let mock_data: &[_] = &[0x1, 0x2, 0x3, 0x4];
/// let mut output = std::vec::Vec::new();
/// // Initialize a compressor.
Expand Down Expand Up @@ -190,7 +190,7 @@ mod tests {
use alloc::vec;

use super::FrameCompressor;
use crate::decoding::{frame::MAGIC_NUM, frame_decoder::FrameDecoder};
use crate::decoding::{frame::MAGIC_NUM, FrameDecoder};
use alloc::vec::Vec;

#[test]
Expand Down Expand Up @@ -286,24 +286,21 @@ mod tests {
fn fuzz_targets() {
use std::io::Read;
fn decode_ruzstd(data: &mut dyn std::io::Read) -> Vec<u8> {
let mut decoder =
crate::decoding::streaming_decoder::StreamingDecoder::new(data).unwrap();
let mut decoder = crate::decoding::StreamingDecoder::new(data).unwrap();
let mut result: Vec<u8> = Vec::new();
decoder.read_to_end(&mut result).expect("Decoding failed");
result
}

fn decode_ruzstd_writer(mut data: impl Read) -> Vec<u8> {
let mut decoder = crate::decoding::frame_decoder::FrameDecoder::new();
let mut decoder = crate::decoding::FrameDecoder::new();
decoder.reset(&mut data).unwrap();
let mut result = vec![];
while !decoder.is_finished() || decoder.can_collect() > 0 {
decoder
.decode_blocks(
&mut data,
crate::decoding::frame_decoder::BlockDecodingStrategy::UptoBytes(
1024 * 1024,
),
crate::decoding::BlockDecodingStrategy::UptoBytes(1024 * 1024),
)
.unwrap();
decoder.collect_to_writer(&mut result).unwrap();
Expand Down
8 changes: 4 additions & 4 deletions src/encoding/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ pub(crate) mod frame_header;
pub(crate) mod match_generator;
pub(crate) mod util;

pub mod frame_compressor;
mod frame_compressor;
pub use frame_compressor::{CompressionLevel, FrameCompressor};

use crate::io::{Read, Write};
use alloc::vec::Vec;
use frame_compressor::{CompressionLevel, FrameCompressor};
use match_generator::Sequence;

/// Convenience function to compress some source into a target without reusing any resources of the compressor
/// ```rust
/// use ruzstd::encoding::{compress, frame_compressor::CompressionLevel};
/// use ruzstd::encoding::{compress, CompressionLevel};
/// let data: &[u8] = &[0,0,0,0,0,0,0,0,0,0,0,0];
/// let mut target = Vec::new();
/// compress(data, &mut target, CompressionLevel::Fastest);
Expand All @@ -28,7 +28,7 @@ pub fn compress<R: Read, W: Write>(source: R, target: W, level: CompressionLevel

/// Convenience function to compress some source into a target without reusing any resources of the compressor into a Vec
/// ```rust
/// use ruzstd::encoding::{compress_to_vec, frame_compressor::CompressionLevel};
/// use ruzstd::encoding::{compress_to_vec, CompressionLevel};
/// let data: &[u8] = &[0,0,0,0,0,0,0,0,0,0,0,0];
/// let compressed = compress_to_vec(data, CompressionLevel::Fastest);
/// ```
Expand Down
6 changes: 3 additions & 3 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
//!
//! ## Decompression
//! The [decoding] module contains the code for decompression.
//! Decompression can be achieved by using the [`decoding::streaming_decoder::StreamingDecoder`]
//! or the more low-level [`decoding::frame_decoder::FrameDecoder`]
//! Decompression can be achieved by using the [`decoding::StreamingDecoder`]
//! or the more low-level [`decoding::FrameDecoder`]
//!
//! ## Compression
//! The [encoding] module contains the code for compression.
//! Decompression can be achieved by using the [`encoding::compress`]/[`encoding::compress_to_vec`]
//! functions or the [`encoding::frame_compressor::FrameCompressor`]
//! functions or the [`encoding::FrameCompressor`]
//!
#![doc = include_str!("../Readme.md")]
#![no_std]
Expand Down
7 changes: 4 additions & 3 deletions src/tests/decode_corpus.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#[test]
fn test_decode_corpus_files() {
extern crate std;
use crate::decoding::frame_decoder;
use crate::decoding::BlockDecodingStrategy;
use crate::decoding::FrameDecoder;
use alloc::borrow::ToOwned;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
Expand Down Expand Up @@ -31,7 +32,7 @@ fn test_decode_corpus_files() {
Ok(entry) => entry.path().to_str().unwrap().to_owned(),
});

let mut frame_dec = frame_decoder::FrameDecoder::new();
let mut frame_dec = FrameDecoder::new();

for file in files {
let f = file.unwrap();
Expand All @@ -51,7 +52,7 @@ fn test_decode_corpus_files() {
let start_time = std::time::Instant::now();
/////DECODING
frame_dec
.decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
.decode_blocks(&mut content, BlockDecodingStrategy::All)
.unwrap();
let result = frame_dec.collect().unwrap();
let end_time = start_time.elapsed();
Expand Down
7 changes: 4 additions & 3 deletions src/tests/dict_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ fn test_dict_parsing() {
#[test]
fn test_dict_decoding() {
extern crate std;
use crate::decoding::frame_decoder;
use crate::decoding::BlockDecodingStrategy;
use crate::decoding::FrameDecoder;
use alloc::borrow::ToOwned;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
Expand All @@ -104,7 +105,7 @@ fn test_dict_decoding() {
Ok(entry) => entry.path().to_str().unwrap().to_owned(),
});

let mut frame_dec = frame_decoder::FrameDecoder::new();
let mut frame_dec = FrameDecoder::new();
let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap();
frame_dec.add_dict(dict).unwrap();

Expand All @@ -126,7 +127,7 @@ fn test_dict_decoding() {
let start_time = std::time::Instant::now();
/////DECODING
frame_dec
.decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
.decode_blocks(&mut content, BlockDecodingStrategy::All)
.unwrap();
let result = frame_dec.collect().unwrap();
let end_time = start_time.elapsed();
Expand Down
Loading

0 comments on commit 8de2364

Please sign in to comment.