lz4/block/
mod.rs

1//! This module provides access to the block mode functions of the lz4 C library.
2//! It somehow resembles the [Python-lz4](http://python-lz4.readthedocs.io/en/stable/lz4.block.html) api,
3//! but using Rust's Option type, the function parameters have been a little simplified.
4//! As does python-lz4, this module supports prepending the compressed buffer with a u32 value
5//! representing the size of the original, uncompressed data.
6//!
7//! # Examples
8//! ```
9//!
10//! use lz4::block::{compress,decompress};
11//!
12//! let v = vec![0u8; 1024];
13//!
14//! let comp_with_prefix = compress(&v, None, true).unwrap();
15//! let comp_wo_prefix = compress(&v, None, false).unwrap();
16//!
17//! assert_eq!(v, decompress(&comp_with_prefix, None).unwrap());
18//! assert_eq!(v, decompress(&comp_wo_prefix, Some(1024)).unwrap());
19//! ```
20
21use super::c_char;
22use super::liblz4::*;
23use std::io::{Error, ErrorKind, Result};
24
25/// Represents the compression mode do be used.
26#[derive(Clone, Copy, Debug)]
27pub enum CompressionMode {
28    /// High compression with compression parameter
29    HIGHCOMPRESSION(i32),
30    /// Fast compression with acceleration paramet
31    FAST(i32),
32    /// Default compression
33    DEFAULT,
34}
35
36impl Default for CompressionMode {
37    fn default() -> Self {
38        CompressionMode::DEFAULT
39    }
40}
41
42/// Returns the size of the buffer that is guaranteed to hold the result of
43/// compressing `uncompressed_size` bytes of in data. Returns std::io::Error
44/// with ErrorKind::InvalidInput if input data is too long to be compressed by lz4.
45pub fn compress_bound(uncompressed_size: usize) -> Result<usize> {
46    // 0 iff src too large
47    let compress_bound: i32 = unsafe { LZ4_compressBound(uncompressed_size as i32) };
48
49    if uncompressed_size > (i32::max_value() as usize) || compress_bound <= 0 {
50        return Err(Error::new(
51            ErrorKind::InvalidInput,
52            "Compression input too long.",
53        ));
54    }
55
56    Ok(compress_bound as usize)
57}
58
59/// Compresses the full src buffer using the specified CompressionMode, where None and Some(Default)
60/// are treated equally. If prepend_size is set, the source length will be prepended to the output
61/// buffer.
62///
63///
64/// # Errors
65/// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long.
66/// Returns std::io::Error with ErrorKind::Other if the compression failed inside the C library. If
67/// this happens, the C api was not able to provide more information about the cause.
68pub fn compress(src: &[u8], mode: Option<CompressionMode>, prepend_size: bool) -> Result<Vec<u8>> {
69    // 0 iff src too large
70    let compress_bound: i32 = unsafe { LZ4_compressBound(src.len() as i32) };
71
72    if src.len() > (i32::max_value() as usize) || compress_bound <= 0 {
73        return Err(Error::new(
74            ErrorKind::InvalidInput,
75            "Compression input too long.",
76        ));
77    }
78
79    let mut compressed: Vec<u8> = vec![
80        0;
81        (if prepend_size {
82            compress_bound + 4
83        } else {
84            compress_bound
85        }) as usize
86    ];
87
88    let dec_size = compress_to_buffer(src, mode, prepend_size, &mut compressed)?;
89    compressed.truncate(dec_size as usize);
90    Ok(compressed)
91}
92
93/// Compresses the full `src` buffer using the specified CompressionMode, where None and Some(Default)
94/// are treated equally, writing compressed bytes to `buffer`.
95///
96/// # Errors
97/// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long.
98/// The buffer cannot be larger than `i32::MAX`.
99/// Returns std::io::Error with ErrorKind::Other if the compression data does not fit in `buffer`.
100pub fn compress_to_buffer(
101    src: &[u8],
102    mode: Option<CompressionMode>,
103    prepend_size: bool,
104    buffer: &mut [u8],
105) -> Result<usize> {
106    // check that src isn't too big for lz4
107    let max_len: i32 = unsafe { LZ4_compressBound(src.len() as i32) };
108
109    if src.len() > (i32::max_value() as usize) || max_len <= 0 {
110        return Err(Error::new(
111            ErrorKind::InvalidInput,
112            "Compression input too long.",
113        ));
114    }
115
116    let dec_size;
117    {
118        let dst_buf = if prepend_size {
119            let size = src.len() as u32;
120            buffer[0] = size as u8;
121            buffer[1] = (size >> 8) as u8;
122            buffer[2] = (size >> 16) as u8;
123            buffer[3] = (size >> 24) as u8;
124            &mut buffer[4..]
125        } else {
126            buffer
127        };
128
129        let buf_len = dst_buf.len() as i32;
130
131        dec_size = match mode {
132            Some(CompressionMode::HIGHCOMPRESSION(level)) => unsafe {
133                LZ4_compress_HC(
134                    src.as_ptr() as *const c_char,
135                    dst_buf.as_mut_ptr() as *mut c_char,
136                    src.len() as i32,
137                    buf_len,
138                    level,
139                )
140            },
141            Some(CompressionMode::FAST(accel)) => unsafe {
142                LZ4_compress_fast(
143                    src.as_ptr() as *const c_char,
144                    dst_buf.as_mut_ptr() as *mut c_char,
145                    src.len() as i32,
146                    buf_len,
147                    accel,
148                )
149            },
150            _ => unsafe {
151                LZ4_compress_default(
152                    src.as_ptr() as *const c_char,
153                    dst_buf.as_mut_ptr() as *mut c_char,
154                    src.len() as i32,
155                    buf_len,
156                )
157            },
158        };
159    }
160    if dec_size <= 0 {
161        return Err(Error::new(ErrorKind::Other, "Compression failed"));
162    }
163
164    let written_size = if prepend_size { dec_size + 4 } else { dec_size };
165
166    Ok(written_size as usize)
167}
168
169fn get_decompressed_size(src: &[u8], uncompressed_size: Option<i32>) -> Result<usize> {
170    let size;
171
172    if let Some(s) = uncompressed_size {
173        size = s;
174    } else {
175        if src.len() < 4 {
176            return Err(Error::new(
177                ErrorKind::InvalidInput,
178                "Source buffer must at least contain size prefix.",
179            ));
180        }
181        size =
182            (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24;
183    }
184
185    if size < 0 {
186        return Err(Error::new(
187            ErrorKind::InvalidInput,
188            if uncompressed_size.is_some() {
189                "Size parameter must not be negative."
190            } else {
191                "Parsed size prefix in buffer must not be negative."
192            },
193        ));
194    }
195
196    if unsafe { LZ4_compressBound(size) } <= 0 {
197        return Err(Error::new(
198            ErrorKind::InvalidInput,
199            "Given size parameter is too big",
200        ));
201    }
202
203    Ok(size as usize)
204}
205
206/// Decompresses the src buffer. If uncompressed_size is None, the source length will be read from
207/// the start of the input buffer.
208///
209///
210/// # Errors
211/// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too short, the
212/// provided (or parsed) uncompressed_size is to large or negative.
213/// Returns std::io::Error with ErrorKind::InvalidData if the decompression failed inside the C
214/// library. This is most likely due to malformed input.
215///
216pub fn decompress(src: &[u8], uncompressed_size: Option<i32>) -> Result<Vec<u8>> {
217    let size = get_decompressed_size(src, uncompressed_size)?;
218
219    let mut buffer = vec![0u8; size];
220
221    let sz = decompress_to_buffer(src, uncompressed_size, &mut buffer)?;
222    buffer.truncate(sz);
223    Ok(buffer)
224}
225
226pub fn decompress_to_buffer(
227    mut src: &[u8],
228    uncompressed_size: Option<i32>,
229    buffer: &mut [u8],
230) -> Result<usize> {
231    let size;
232
233    if let Some(s) = uncompressed_size {
234        size = s;
235    } else {
236        if src.len() < 4 {
237            return Err(Error::new(
238                ErrorKind::InvalidInput,
239                "Source buffer must at least contain size prefix.",
240            ));
241        }
242        size =
243            (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24;
244
245        src = &src[4..];
246    }
247
248    if size < 0 {
249        return Err(Error::new(
250            ErrorKind::InvalidInput,
251            if uncompressed_size.is_some() {
252                "Size parameter must not be negative."
253            } else {
254                "Parsed size prefix in buffer must not be negative."
255            },
256        ));
257    }
258
259    if unsafe { LZ4_compressBound(size) } <= 0 {
260        return Err(Error::new(
261            ErrorKind::InvalidInput,
262            "Given size parameter is too big",
263        ));
264    }
265
266    if size as usize > buffer.len() {
267        return Err(Error::new(
268            ErrorKind::InvalidInput,
269            "buffer isn't large enough to hold decompressed data",
270        ));
271    }
272
273    let dec_bytes = unsafe {
274        LZ4_decompress_safe(
275            src.as_ptr() as *const c_char,
276            buffer.as_mut_ptr() as *mut c_char,
277            src.len() as i32,
278            size,
279        )
280    };
281
282    if dec_bytes < 0 {
283        return Err(Error::new(
284            ErrorKind::InvalidData,
285            "Decompression failed. Input invalid or too long?",
286        ));
287    }
288
289    Ok(dec_bytes as usize)
290}
291
292#[cfg(test)]
293mod test {
294    use crate::block::{compress, decompress, decompress_to_buffer, CompressionMode};
295
296    use super::compress_to_buffer;
297
298    /// This test will fail unless the buffer created by decompress is correctly truncated
299    #[test]
300    fn decompress_truncate_test() {
301        let src = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111".as_bytes();
302        let rs_compressed = compress(src, None, false).unwrap();
303        let rs_compressed_rs_uncompressed =
304            decompress(&rs_compressed, Some((src.len() as i32) * 256)).unwrap();
305
306        // compare the uncompressed result from rust
307        assert_eq!(rs_compressed_rs_uncompressed, src,);
308    }
309
310    #[test]
311    fn test_compression_without_prefix() {
312        let size = 65536;
313        let mut to_compress = Vec::with_capacity(size);
314        for i in 0..size {
315            to_compress.push(i as u8);
316        }
317        let mut v: Vec<Vec<u8>> = vec![];
318        for i in 1..100 {
319            v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), false).unwrap());
320        }
321
322        // 12 is max high compression parameter
323        for i in 1..12 {
324            v.push(
325                compress(
326                    &to_compress,
327                    Some(CompressionMode::HIGHCOMPRESSION(i)),
328                    false,
329                )
330                .unwrap(),
331            );
332        }
333
334        v.push(compress(&to_compress, None, false).unwrap());
335
336        for val in v {
337            assert_eq!(
338                decompress(&val, Some(to_compress.len() as i32)).unwrap(),
339                to_compress
340            );
341        }
342    }
343
344    #[test]
345    fn test_compression_with_prefix() {
346        let size = 65536;
347        let mut to_compress = Vec::with_capacity(size);
348        for i in 0..size {
349            to_compress.push(i as u8);
350        }
351        let mut v: Vec<Vec<u8>> = vec![];
352        for i in 1..100 {
353            v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), true).unwrap());
354        }
355
356        // 12 is max high compression parameter
357        for i in 1..12 {
358            v.push(
359                compress(
360                    &to_compress,
361                    Some(CompressionMode::HIGHCOMPRESSION(i)),
362                    true,
363                )
364                .unwrap(),
365            );
366        }
367
368        v.push(compress(&to_compress, None, true).unwrap());
369
370        for val in v {
371            assert_eq!(decompress(&val, None).unwrap(), to_compress);
372        }
373    }
374
375    #[test]
376    fn test_compress_to_buffer() {
377        let data = b"qfn3489fqASFqvegrwe$%344thI,..kmTMN3 g{P}wefwf2fv2443ef3443RT[]rete$7-80956GRWbefvw@fVGrwGB24tggrm%&*I@!";
378
379        // test what happens when not enough space is available.
380        let mut small_buf = vec![0; 5];
381        let r = compress_to_buffer(&data[..], None, true, &mut small_buf);
382        assert!(r.is_err());
383
384        let mut big_buf = vec![0; 1000];
385        let r = compress_to_buffer(&data[..], None, true, &mut big_buf).unwrap();
386        assert_eq!(big_buf[r], 0);
387
388        let mut dec_buf = vec![0u8; data.len() + 1];
389        let dec_bytes = decompress_to_buffer(&big_buf[..r], None, &mut dec_buf).unwrap();
390        assert_eq!(&dec_buf[..dec_bytes], &data[..]);
391    }
392    #[test]
393    fn test_decompression_with_prefix() {
394        let compressed: [u8; 250] = [
395            0, 188, 0, 0, 255, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 116, 101, 115,
396            116, 32, 115, 116, 114, 105, 110, 103, 32, 99, 111, 109, 112, 114, 101, 115, 115, 101,
397            100, 32, 98, 121, 32, 112, 121, 116, 104, 111, 110, 45, 108, 122, 52, 32, 47, 0, 255,
398            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
399            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
400            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
401            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
402            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
403            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
404            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
405            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
406            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
407            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
408            255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
409            117, 80, 45, 108, 122, 52, 32,
410        ];
411
412        let mut reference: String = String::new();
413        for _ in 0..1024 {
414            reference += "this is a test string compressed by python-lz4 ";
415        }
416
417        assert_eq!(decompress(&compressed, None).unwrap(), reference.as_bytes())
418    }
419
420    #[test]
421    fn test_empty_compress() {
422        use crate::block::{compress, decompress};
423        let v = vec![0u8; 0];
424        let comp_with_prefix = compress(&v, None, true).unwrap();
425        dbg!(&comp_with_prefix);
426        assert_eq!(v, decompress(&comp_with_prefix, None).unwrap());
427    }
428}
lz4/block/mod.rs

lz4/block/
mod.rs
