Migrate all compression code to Rust

native/src/external/Android.mk

@@ -218,27 +218,6 @@ LOCAL_SRC_FILES := \
     lsplt/lsplt/src/main/jni/lsplt.cc
 include $(BUILD_STATIC_LIBRARY)
 
-# libzopfli.a
-include $(CLEAR_VARS)
-LOCAL_MODULE:= libzopfli
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/zopfli/src
-LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_INCLUDES)
-LOCAL_CFLAGS := -Wall -Werror -Wno-unused -Wno-unused-parameter
-LOCAL_SRC_FILES := \
-    zopfli/src/zopfli/blocksplitter.c \
-    zopfli/src/zopfli/cache.c \
-    zopfli/src/zopfli/deflate.c \
-    zopfli/src/zopfli/gzip_container.c \
-    zopfli/src/zopfli/hash.c \
-    zopfli/src/zopfli/katajainen.c \
-    zopfli/src/zopfli/lz77.c \
-    zopfli/src/zopfli/squeeze.c \
-    zopfli/src/zopfli/tree.c \
-    zopfli/src/zopfli/util.c \
-    zopfli/src/zopfli/zlib_container.c \
-    zopfli/src/zopfli/zopfli_lib.c
-include $(BUILD_STATIC_LIBRARY)
-
 CWD := $(LOCAL_PATH)
 include $(CWD)/system_properties/Android.mk
 include $(CWD)/libcxx/Android.mk

native/src/external/lz4-sys/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "lz4-sys"
+license = "MIT"
+version = "1.11.1+lz4-1.10.0"
+authors = [ "Jens Heyens <jens.heyens@ewetel.net>", "Artem V. Navrotskiy <bozaro@buzzsoft.ru>", "Patrick Marks <pmarks@gmail.com>"]
+description = "Rust LZ4 sys package."
+repository = "https://github.com/10xGenomics/lz4-rs"
+
+[dependencies]
+libc = "0.2"

native/src/external/lz4-sys/src/lib.rs

@@ -0,0 +1,480 @@
+#![allow(unexpected_cfgs)]
+#![no_std]
+extern crate libc;
+
+#[cfg(not(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+)))]
+pub use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t};
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+extern crate alloc;
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+mod wasm_shim;
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+extern crate std;
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+pub use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void};
+
+#[cfg(all(
+    target_arch = "wasm32",
+    not(any(target_env = "wasi", target_os = "wasi"))
+))]
+#[allow(non_camel_case_types)]
+pub type size_t = usize;
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct LZ4FCompressionContext(pub *mut c_void);
+unsafe impl Send for LZ4FCompressionContext {}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct LZ4FDecompressionContext(pub *mut c_void);
+unsafe impl Send for LZ4FDecompressionContext {}
+
+pub type LZ4FErrorCode = size_t;
+
+#[derive(Clone, Debug)]
+#[repr(u32)]
+pub enum BlockSize {
+    Default = 0, // Default - 64KB
+    Max64KB = 4,
+    Max256KB = 5,
+    Max1MB = 6,
+    Max4MB = 7,
+}
+
+impl BlockSize {
+    pub fn get_size(&self) -> usize {
+        match self {
+            &BlockSize::Default | &BlockSize::Max64KB => 64 * 1024,
+            &BlockSize::Max256KB => 256 * 1024,
+            &BlockSize::Max1MB => 1 * 1024 * 1024,
+            &BlockSize::Max4MB => 4 * 1024 * 1024,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+#[repr(u32)]
+pub enum BlockMode {
+    Linked = 0,
+    Independent,
+}
+
+#[derive(Clone, Debug)]
+#[repr(u32)]
+pub enum ContentChecksum {
+    NoChecksum = 0,
+    ChecksumEnabled,
+}
+
+#[derive(Clone, Debug)]
+#[repr(u32)]
+pub enum FrameType {
+    Frame = 0,
+    SkippableFrame,
+}
+
+#[derive(Clone, Debug)]
+#[repr(u32)]
+pub enum BlockChecksum {
+    NoBlockChecksum = 0,
+    BlockChecksumEnabled,
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4FFrameInfo {
+    pub block_size_id: BlockSize,
+    pub block_mode: BlockMode,
+    pub content_checksum_flag: ContentChecksum,
+    pub frame_type: FrameType,
+    pub content_size: c_ulonglong,
+    pub dict_id: c_uint,
+    pub block_checksum_flag: BlockChecksum,
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4FPreferences {
+    pub frame_info: LZ4FFrameInfo,
+    pub compression_level: c_uint, // 0 == default (fast mode); values above 16 count as 16
+    pub auto_flush: c_uint,        // 1 == always flush : reduce need for tmp buffer
+    pub favor_dec_speed: c_uint,   // 1 == favor decompression speed over ratio, requires level 10+
+    pub reserved: [c_uint; 3],
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4FCompressOptions {
+    pub stable_src: c_uint, /* 1 == src content will remain available on future calls
+                             * to LZ4F_compress(); avoid saving src content within tmp
+                             * buffer as future dictionary */
+    pub reserved: [c_uint; 3],
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4FDecompressOptions {
+    pub stable_dst: c_uint, /* guarantee that decompressed data will still be there on next
+                             * function calls (avoid storage into tmp buffers) */
+    pub reserved: [c_uint; 3],
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4StreamEncode(c_void);
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct LZ4StreamDecode(c_void);
+
+pub const LZ4F_VERSION: c_uint = 100;
+
+extern "C" {
+
+    // int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+    #[allow(non_snake_case)]
+    pub fn LZ4_compress_default(
+        source: *const c_char,
+        dest: *mut c_char,
+        sourceSize: c_int,
+        maxDestSize: c_int,
+    ) -> c_int;
+
+    // int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+    #[allow(non_snake_case)]
+    pub fn LZ4_compress_fast(
+        source: *const c_char,
+        dest: *mut c_char,
+        sourceSize: c_int,
+        maxDestSize: c_int,
+        acceleration: c_int,
+    ) -> c_int;
+
+    // int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
+    #[allow(non_snake_case)]
+    pub fn LZ4_compress_HC(
+        src: *const c_char,
+        dst: *mut c_char,
+        srcSize: c_int,
+        dstCapacity: c_int,
+        compressionLevel: c_int,
+    ) -> c_int;
+
+    // int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+    #[allow(non_snake_case)]
+    pub fn LZ4_decompress_safe(
+        source: *const c_char,
+        dest: *mut c_char,
+        compressedSize: c_int,
+        maxDecompressedSize: c_int,
+    ) -> c_int;
+
+    // unsigned    LZ4F_isError(LZ4F_errorCode_t code);
+    pub fn LZ4F_isError(code: size_t) -> c_uint;
+
+    // const char* LZ4F_getErrorName(LZ4F_errorCode_t code);
+    pub fn LZ4F_getErrorName(code: size_t) -> *const c_char;
+
+    // LZ4F_createCompressionContext() :
+    // The first thing to do is to create a compressionContext object, which will be used in all
+    // compression operations.
+    // This is achieved using LZ4F_createCompressionContext(), which takes as argument a version
+    // and an LZ4F_preferences_t structure.
+    // The version provided MUST be LZ4F_VERSION. It is intended to track potential version
+    // differences between different binaries.
+    // The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object.
+    // If the result LZ4F_errorCode_t is not zero, there was an error during context creation.
+    // Object can release its memory using LZ4F_freeCompressionContext();
+    //
+    // LZ4F_errorCode_t LZ4F_createCompressionContext(
+    //                                   LZ4F_compressionContext_t* LZ4F_compressionContextPtr,
+    //                                   unsigned version);
+    pub fn LZ4F_createCompressionContext(
+        ctx: &mut LZ4FCompressionContext,
+        version: c_uint,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_errorCode_t LZ4F_freeCompressionContext(
+    //                                  LZ4F_compressionContext_t LZ4F_compressionContext);
+    pub fn LZ4F_freeCompressionContext(ctx: LZ4FCompressionContext) -> LZ4FErrorCode;
+
+    // LZ4F_compressBegin() :
+    // will write the frame header into dstBuffer.
+    // dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header
+    // size is 19 bytes.
+    // The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all
+    // preferences will then be set to default.
+    // The result of the function is the number of bytes written into dstBuffer for the header
+    // or an error code (can be tested using LZ4F_isError())
+    //
+    // size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext,
+    //                           void* dstBuffer,
+    //                           size_t dstMaxSize,
+    //                           const LZ4F_preferences_t* preferencesPtr);
+    pub fn LZ4F_compressBegin(
+        ctx: LZ4FCompressionContext,
+        dstBuffer: *mut u8,
+        dstMaxSize: size_t,
+        preferencesPtr: *const LZ4FPreferences,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_compressBound() :
+    // Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
+    // preferencesPtr is optional : you can provide NULL as argument, all preferences will then
+    // be set to default.
+    // Note that different preferences will produce in different results.
+    //
+    // size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
+    pub fn LZ4F_compressBound(
+        srcSize: size_t,
+        preferencesPtr: *const LZ4FPreferences,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_compressUpdate()
+    // LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+    // The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure
+    // compression completion even in worst case.
+    // If this condition is not respected, LZ4F_compress() will fail (result is an errorCode)
+    // You can get the minimum value of dstMaxSize by using LZ4F_compressBound()
+    // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+    // The result of the function is the number of bytes written into dstBuffer : it can be zero,
+    // meaning input data was just buffered.
+    // The function outputs an error code if it fails (can be tested using LZ4F_isError())
+    //
+    // size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext,
+    //                            void* dstBuffer,
+    //                            size_t dstMaxSize,
+    //                            const void* srcBuffer,
+    //                            size_t srcSize,
+    //                            const LZ4F_compressOptions_t* compressOptionsPtr);
+    pub fn LZ4F_compressUpdate(
+        ctx: LZ4FCompressionContext,
+        dstBuffer: *mut u8,
+        dstMaxSize: size_t,
+        srcBuffer: *const u8,
+        srcSize: size_t,
+        compressOptionsPtr: *const LZ4FCompressOptions,
+    ) -> size_t;
+
+    // LZ4F_flush()
+    // Should you need to create compressed data immediately, without waiting for a block
+    // to be be filled, you can call LZ4_flush(), which will immediately compress any remaining
+    // data buffered within compressionContext.
+    // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+    // The result of the function is the number of bytes written into dstBuffer
+    // (it can be zero, this means there was no data left within compressionContext)
+    // The function outputs an error code if it fails (can be tested using LZ4F_isError())
+    //
+    // size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext,
+    //                   void* dstBuffer,
+    //                   size_t dstMaxSize,
+    //                   const LZ4F_compressOptions_t* compressOptionsPtr);
+    pub fn LZ4F_flush(
+        ctx: LZ4FCompressionContext,
+        dstBuffer: *mut u8,
+        dstMaxSize: size_t,
+        compressOptionsPtr: *const LZ4FCompressOptions,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_compressEnd()
+    // When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+    // It will flush whatever data remained within compressionContext (like LZ4_flush())
+    // but also properly finalize the frame, with an endMark and a checksum.
+    // The result of the function is the number of bytes written into dstBuffer
+    // (necessarily >= 4 (endMark size))
+    // The function outputs an error code if it fails (can be tested using LZ4F_isError())
+    // The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+    // compressionContext can then be used again, starting with LZ4F_compressBegin().
+    //
+    // size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext,
+    //                         void* dstBuffer,
+    //                         size_t dstMaxSize,
+    //                         const LZ4F_compressOptions_t* compressOptionsPtr);
+    pub fn LZ4F_compressEnd(
+        ctx: LZ4FCompressionContext,
+        dstBuffer: *mut u8,
+        dstMaxSize: size_t,
+        compressOptionsPtr: *const LZ4FCompressOptions,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_createDecompressionContext() :
+    // The first thing to do is to create a decompressionContext object, which will be used
+    // in all decompression operations.
+    // This is achieved using LZ4F_createDecompressionContext().
+    // The version provided MUST be LZ4F_VERSION. It is intended to track potential version
+    // differences between different binaries.
+    // The function will provide a pointer to a fully allocated and initialized
+    // LZ4F_decompressionContext_t object.
+    // If the result LZ4F_errorCode_t is not OK_NoError, there was an error during
+    // context creation.
+    // Object can release its memory using LZ4F_freeDecompressionContext();
+    //
+    // LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* ctxPtr,
+    //                                                  unsigned version);
+    pub fn LZ4F_createDecompressionContext(
+        ctx: &mut LZ4FDecompressionContext,
+        version: c_uint,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t ctx);
+    pub fn LZ4F_freeDecompressionContext(ctx: LZ4FDecompressionContext) -> LZ4FErrorCode;
+
+    // LZ4F_getFrameInfo()
+    // This function decodes frame header information, such as blockSize.
+    // It is optional : you could start by calling directly LZ4F_decompress() instead.
+    // The objective is to extract header information without starting decompression, typically
+    // for allocation purposes.
+    // LZ4F_getFrameInfo() can also be used *after* starting decompression, on a
+    // valid LZ4F_decompressionContext_t.
+    // The number of bytes read from srcBuffer will be provided within *srcSizePtr
+    // (necessarily <= original value).
+    // You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
+    // The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for
+    // next call, or an error code which can be tested using LZ4F_isError().
+    //
+    // size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx,
+    // 					LZ4F_frameInfo_t* frameInfoPtr,
+    // 					const void* srcBuffer, size_t* srcSizePtr);
+    pub fn LZ4F_getFrameInfo(
+        ctx: LZ4FDecompressionContext,
+        frameInfoPtr: &mut LZ4FFrameInfo,
+        srcBuffer: *const u8,
+        srcSizePtr: &mut size_t,
+    ) -> LZ4FErrorCode;
+
+    // LZ4F_decompress()
+    // Call this function repetitively to regenerate data compressed within srcBuffer.
+    // The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of
+    // maximum size *dstSizePtr.
+    //
+    // The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr
+    // (necessarily <= original value).
+    //
+    // The number of bytes read from srcBuffer will be provided within *srcSizePtr
+    // (necessarily <= original value).
+    // If number of bytes read is < number of bytes provided, then decompression operation
+    // is not completed. It typically happens when dstBuffer is not large enough to contain
+    // all decoded data.
+    // LZ4F_decompress() must be called again, starting from where it stopped
+    // (srcBuffer + *srcSizePtr)
+    // The function will check this condition, and refuse to continue if it is not respected.
+    //
+    // dstBuffer is supposed to be flushed between each call to the function, since its content
+    // will be overwritten.
+    // dst arguments can be changed at will with each consecutive call to the function.
+    //
+    // The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for
+    // next call.
+    // Schematically, it's the size of the current (or remaining) compressed block + header of
+    // next block.
+    // Respecting the hint provides some boost to performance, since it does skip intermediate
+    // buffers.
+    // This is just a hint, you can always provide any srcSize you want.
+    // When a frame is fully decoded, the function result will be 0. (no more data expected)
+    // If decompression failed, function result is an error code, which can be tested
+    // using LZ4F_isError().
+    //
+    // size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx,
+    //                        void* dstBuffer, size_t* dstSizePtr,
+    //                        const void* srcBuffer, size_t* srcSizePtr,
+    //                        const LZ4F_decompressOptions_t* optionsPtr);
+    pub fn LZ4F_decompress(
+        ctx: LZ4FDecompressionContext,
+        dstBuffer: *mut u8,
+        dstSizePtr: &mut size_t,
+        srcBuffer: *const u8,
+        srcSizePtr: &mut size_t,
+        optionsPtr: *const LZ4FDecompressOptions,
+    ) -> LZ4FErrorCode;
+
+    // int LZ4_versionNumber(void)
+    pub fn LZ4_versionNumber() -> c_int;
+
+    // int LZ4_compressBound(int isize)
+    pub fn LZ4_compressBound(size: c_int) -> c_int;
+
+    // LZ4_stream_t* LZ4_createStream(void)
+    pub fn LZ4_createStream() -> *mut LZ4StreamEncode;
+
+    // int LZ4_compress_continue(LZ4_stream_t* LZ4_streamPtr,
+    //                           const char* source,
+    //                           char* dest,
+    //                           int inputSize)
+    pub fn LZ4_compress_continue(
+        LZ4_stream: *mut LZ4StreamEncode,
+        source: *const u8,
+        dest: *mut u8,
+        input_size: c_int,
+    ) -> c_int;
+
+    // int LZ4_freeStream(LZ4_stream_t* LZ4_streamPtr)
+    pub fn LZ4_freeStream(LZ4_stream: *mut LZ4StreamEncode) -> c_int;
+
+    // int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode,
+    //                          const char* dictionary,
+    //                          int dictSize)
+    pub fn LZ4_setStreamDecode(
+        LZ4_stream: *mut LZ4StreamDecode,
+        dictionary: *const u8,
+        dict_size: c_int,
+    ) -> c_int;
+
+    // LZ4_streamDecode_t* LZ4_createStreamDecode(void)
+    pub fn LZ4_createStreamDecode() -> *mut LZ4StreamDecode;
+
+    // int LZ4_decompress_safe_continue(LZ4_streamDecode_t* LZ4_streamDecode,
+    //                                  const char* source,
+    //                                  char* dest,
+    //                                  int compressedSize,
+    //                                  int maxDecompressedSize)
+    pub fn LZ4_decompress_safe_continue(
+        LZ4_stream: *mut LZ4StreamDecode,
+        source: *const u8,
+        dest: *mut u8,
+        compressed_size: c_int,
+        max_decompressed_size: c_int,
+    ) -> c_int;
+
+    // int LZ4_freeStreamDecode(LZ4_streamDecode_t* LZ4_stream)
+    pub fn LZ4_freeStreamDecode(LZ4_stream: *mut LZ4StreamDecode) -> c_int;
+
+    // LZ4F_resetDecompressionContext()
+    // In case of an error, the context is left in "undefined" state.
+    // In which case, it's necessary to reset it, before re-using it.
+    // This method can also be used to abruptly stop any unfinished decompression,
+    // and start a new one using same context resources.
+    pub fn LZ4F_resetDecompressionContext(ctx: LZ4FDecompressionContext);
+
+}
+
+#[test]
+fn test_version_number() {
+    unsafe {
+        LZ4_versionNumber();
+    }
+}
+
+#[test]
+fn test_frame_info_size() {
+    assert_eq!(core::mem::size_of::<LZ4FFrameInfo>(), 32);
+}

native/src/external/lz4-sys/src/wasm_shim.rs

@@ -0,0 +1,123 @@
+//! A shim for the libc functions used in lz4-rs that are not available when building for wasm
+//! targets. Adapted from the shim present in the [zstd](https://github.com/gyscos/zstd-rs) crate.
+//! zstd-rs license here:
+//! The MIT License (MIT)
+//! Copyright (c) 2016 Alexandre Bury
+//!
+//! Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+//! and associated documentation files (the "Software"), to deal in the Software without
+//! restriction, including without limitation the rights to use, copy, modify, merge, publish,
+//! distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+//! Software is furnished to do so, subject to the following conditions:
+//!
+//! The above copyright notice and this permission notice shall be included in all copies or
+//! substantial portions of the Software.
+//!
+//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+//! BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+//! NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+//! DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+use alloc::alloc::{alloc, alloc_zeroed, dealloc, Layout};
+use core::ffi::{c_int, c_void};
+
+const USIZE_ALIGN: usize = core::mem::align_of::<usize>();
+const USIZE_SIZE: usize = core::mem::size_of::<usize>();
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_malloc(size: usize) -> *mut c_void {
+    wasm_shim_alloc::<false>(size)
+}
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_memcmp(
+    str1: *const c_void,
+    str2: *const c_void,
+    n: usize,
+) -> i32 {
+    // Safety: function contracts requires str1 and str2 at least `n`-long.
+    unsafe {
+        let str1: &[u8] = core::slice::from_raw_parts(str1 as *const u8, n);
+        let str2: &[u8] = core::slice::from_raw_parts(str2 as *const u8, n);
+        match str1.cmp(str2) {
+            core::cmp::Ordering::Less => -1,
+            core::cmp::Ordering::Equal => 0,
+            core::cmp::Ordering::Greater => 1,
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn rust_lz4_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void {
+    // note: calloc expects the allocation to be zeroed
+    wasm_shim_alloc::<true>(nmemb * size)
+}
+
+#[inline]
+fn wasm_shim_alloc<const ZEROED: bool>(size: usize) -> *mut c_void {
+    // in order to recover the size upon free, we store the size below the allocation
+    // special alignment is never requested via the malloc API,
+    // so it's not stored, and usize-alignment is used
+    // memory layout: [size] [allocation]
+
+    let full_alloc_size = size + USIZE_SIZE;
+
+    unsafe {
+        let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);
+
+        let ptr = if ZEROED {
+            alloc_zeroed(layout)
+        } else {
+            alloc(layout)
+        };
+
+        // SAFETY: ptr is usize-aligned and we've allocated sufficient memory
+        ptr.cast::<usize>().write(full_alloc_size);
+
+        ptr.add(USIZE_SIZE).cast()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_free(ptr: *mut c_void) {
+    // the layout for the allocation needs to be recovered for dealloc
+    // - the size must be recovered from directly below the allocation
+    // - the alignment will always by USIZE_ALIGN
+
+    let alloc_ptr = ptr.sub(USIZE_SIZE);
+    // SAFETY: the allocation routines must uphold having a valid usize below the provided pointer
+    let full_alloc_size = alloc_ptr.cast::<usize>().read();
+
+    let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN);
+    dealloc(alloc_ptr.cast(), layout);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memcpy(
+    dest: *mut c_void,
+    src: *const c_void,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memmove(
+    dest: *mut c_void,
+    src: *const c_void,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::copy(src as *const u8, dest as *mut u8, n);
+    dest
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rust_lz4_wasm_shim_memset(
+    dest: *mut c_void,
+    c: c_int,
+    n: usize,
+) -> *mut c_void {
+    core::ptr::write_bytes(dest as *mut u8, c as u8, n);
+    dest
+}

native/src/external/lzma-sys/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "lzma-sys"
+version = "0.1.20"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/alexcrichton/xz2-rs"
+homepage = "https://github.com/alexcrichton/xz2-rs"
+documentation = "https://docs.rs/lzma-sys"
+description = """
+Raw bindings to liblzma which contains an implementation of LZMA and xz stream
+encoding/decoding.
+
+High level Rust bindings are available in the `xz2` crate.
+"""
+categories = ["external-ffi-bindings"]
+edition = "2018"
+
+[dependencies]
+libc = "0.2.51"
+
+[features]
+static = []

native/src/external/lzma-sys/src/lib.rs

@@ -0,0 +1,359 @@
+#![allow(bad_style)]
+#![doc(html_root_url = "https://docs.rs/lzma-sys/0.1")]
+
+use libc::{c_char, c_uchar, c_void, size_t};
+use std::u64;
+
+#[cfg(target_env = "msvc")]
+#[doc(hidden)]
+pub type __enum_ty = libc::c_int;
+#[cfg(not(target_env = "msvc"))]
+#[doc(hidden)]
+pub type __enum_ty = libc::c_uint;
+pub type lzma_bool = c_uchar;
+pub type lzma_ret = __enum_ty;
+pub type lzma_action = __enum_ty;
+type lzma_reserved_enum = __enum_ty;
+pub type lzma_check = __enum_ty;
+pub type lzma_vli = u64;
+pub type lzma_mode = __enum_ty;
+pub type lzma_match_finder = __enum_ty;
+
+pub const LZMA_OK: lzma_ret = 0;
+pub const LZMA_STREAM_END: lzma_ret = 1;
+pub const LZMA_NO_CHECK: lzma_ret = 2;
+pub const LZMA_UNSUPPORTED_CHECK: lzma_ret = 3;
+pub const LZMA_GET_CHECK: lzma_ret = 4;
+pub const LZMA_MEM_ERROR: lzma_ret = 5;
+pub const LZMA_MEMLIMIT_ERROR: lzma_ret = 6;
+pub const LZMA_FORMAT_ERROR: lzma_ret = 7;
+pub const LZMA_OPTIONS_ERROR: lzma_ret = 8;
+pub const LZMA_DATA_ERROR: lzma_ret = 9;
+pub const LZMA_BUF_ERROR: lzma_ret = 10;
+pub const LZMA_PROG_ERROR: lzma_ret = 11;
+
+pub const LZMA_RUN: lzma_action = 0;
+pub const LZMA_SYNC_FLUSH: lzma_action = 1;
+pub const LZMA_FULL_FLUSH: lzma_action = 2;
+pub const LZMA_FULL_BARRIER: lzma_action = 4;
+pub const LZMA_FINISH: lzma_action = 3;
+
+pub const LZMA_CHECK_NONE: lzma_check = 0;
+pub const LZMA_CHECK_CRC32: lzma_check = 1;
+pub const LZMA_CHECK_CRC64: lzma_check = 4;
+pub const LZMA_CHECK_SHA256: lzma_check = 10;
+
+pub const LZMA_MODE_FAST: lzma_mode = 1;
+pub const LZMA_MODE_NORMAL: lzma_mode = 2;
+
+pub const LZMA_MF_HC3: lzma_match_finder = 0x03;
+pub const LZMA_MF_HC4: lzma_match_finder = 0x04;
+pub const LZMA_MF_BT2: lzma_match_finder = 0x12;
+pub const LZMA_MF_BT3: lzma_match_finder = 0x13;
+pub const LZMA_MF_BT4: lzma_match_finder = 0x14;
+
+pub const LZMA_TELL_NO_CHECK: u32 = 0x01;
+pub const LZMA_TELL_UNSUPPORTED_CHECK: u32 = 0x02;
+pub const LZMA_TELL_ANY_CHECK: u32 = 0x04;
+pub const LZMA_IGNORE_CHECK: u32 = 0x10;
+pub const LZMA_CONCATENATED: u32 = 0x08;
+
+pub const LZMA_PRESET_DEFAULT: u32 = 6;
+pub const LZMA_PRESET_LEVEL_MASK: u32 = 0x1f;
+pub const LZMA_PRESET_EXTREME: u32 = 1 << 31;
+
+pub const LZMA_DICT_SIZE_MIN: u32 = 4096;
+pub const LZMA_DICT_SIZE_DEFAULT: u32 = 1 << 23;
+
+pub const LZMA_LCLP_MIN: u32 = 0;
+pub const LZMA_LCLP_MAX: u32 = 4;
+pub const LZMA_LC_DEFAULT: u32 = 3;
+
+pub const LZMA_LP_DEFAULT: u32 = 0;
+
+pub const LZMA_PB_MIN: u32 = 0;
+pub const LZMA_PB_MAX: u32 = 4;
+pub const LZMA_PB_DEFAULT: u32 = 2;
+
+pub const LZMA_BACKWARD_SIZE_MIN: lzma_vli = 4;
+pub const LZMA_BACKWARD_SIZE_MAX: lzma_vli = 1 << 34;
+
+pub const LZMA_VLI_MAX: lzma_vli = u64::MAX / 2;
+pub const LZMA_VLI_UNKNOWN: lzma_vli = u64::MAX;
+pub const LZMA_VLI_BYTES_MAX: usize = 9;
+
+pub const LZMA_FILTER_X86: lzma_vli = 0x04;
+pub const LZMA_FILTER_POWERPC: lzma_vli = 0x05;
+pub const LZMA_FILTER_IA64: lzma_vli = 0x06;
+pub const LZMA_FILTER_ARM: lzma_vli = 0x07;
+pub const LZMA_FILTER_ARMTHUMB: lzma_vli = 0x08;
+pub const LZMA_FILTER_SPARC: lzma_vli = 0x09;
+pub const LZMA_FILTER_LZMA1: lzma_vli = 0x4000000000000001;
+pub const LZMA_FILTER_LZMA2: lzma_vli = 0x21;
+
+#[repr(C)]
+pub struct lzma_allocator {
+    pub alloc: Option<extern "C" fn(*mut c_void, size_t, size_t) -> *mut c_void>,
+    pub free: Option<extern "C" fn(*mut c_void, *mut c_void)>,
+    pub opaque: *mut c_void,
+}
+
+pub enum lzma_internal {}
+
+#[repr(C)]
+pub struct lzma_stream {
+    pub next_in: *const u8,
+    pub avail_in: size_t,
+    pub total_in: u64,
+    pub next_out: *mut u8,
+    pub avail_out: size_t,
+    pub total_out: u64,
+    pub allocator: *const lzma_allocator,
+
+    internal: *mut lzma_internal,
+    reserved_ptr1: *mut c_void,
+    reserved_ptr2: *mut c_void,
+    reserved_ptr3: *mut c_void,
+    reserved_ptr4: *mut c_void,
+    reserved_int1: u64,
+    reserved_int2: u64,
+    reserved_int3: size_t,
+    reserved_int4: size_t,
+    reserved_enum1: lzma_reserved_enum,
+    reserved_enum2: lzma_reserved_enum,
+}
+
+#[repr(C)]
+pub struct lzma_filter {
+    pub id: lzma_vli,
+    pub options: *mut c_void,
+}
+
+#[repr(C)]
+pub struct lzma_mt {
+    pub flags: u32,
+    pub threads: u32,
+    pub block_size: u64,
+    pub timeout: u32,
+    pub preset: u32,
+    pub filters: *const lzma_filter,
+    pub check: lzma_check,
+
+    reserved_enum1: lzma_reserved_enum,
+    reserved_enum2: lzma_reserved_enum,
+    reserved_enum3: lzma_reserved_enum,
+    reserved_int1: u32,
+    reserved_int2: u32,
+    reserved_int3: u32,
+    reserved_int4: u32,
+    reserved_int5: u64,
+    reserved_int6: u64,
+    reserved_int7: u64,
+    reserved_int8: u64,
+    reserved_ptr1: *mut c_void,
+    reserved_ptr2: *mut c_void,
+    reserved_ptr3: *mut c_void,
+    reserved_ptr4: *mut c_void,
+}
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct lzma_options_lzma {
+    pub dict_size: u32,
+    pub preset_dict: *const u8,
+    pub preset_dict_size: u32,
+    pub lc: u32,
+    pub lp: u32,
+    pub pb: u32,
+    pub mode: lzma_mode,
+    pub nice_len: u32,
+    pub mf: lzma_match_finder,
+    pub depth: u32,
+
+    reserved_int1: u32,
+    reserved_int2: u32,
+    reserved_int3: u32,
+    reserved_int4: u32,
+    reserved_int5: u32,
+    reserved_int6: u32,
+    reserved_int7: u32,
+    reserved_int8: u32,
+    reserved_enum1: lzma_reserved_enum,
+    reserved_enum2: lzma_reserved_enum,
+    reserved_enum3: lzma_reserved_enum,
+    reserved_enum4: lzma_reserved_enum,
+    reserved_ptr1: *mut c_void,
+    reserved_ptr2: *mut c_void,
+}
+
+#[repr(C)]
+pub struct lzma_stream_flags {
+    pub version: u32,
+    pub backward_size: lzma_vli,
+    pub check: lzma_check,
+
+    reserved_enum1: lzma_reserved_enum,
+    reserved_enum2: lzma_reserved_enum,
+    reserved_enum3: lzma_reserved_enum,
+    reserved_enum4: lzma_reserved_enum,
+    reserved_bool1: lzma_bool,
+    reserved_bool2: lzma_bool,
+    reserved_bool3: lzma_bool,
+    reserved_bool4: lzma_bool,
+    reserved_bool5: lzma_bool,
+    reserved_bool6: lzma_bool,
+    reserved_bool7: lzma_bool,
+    reserved_bool8: lzma_bool,
+    reserved_int1: u32,
+    reserved_int2: u32,
+}
+
+#[repr(C)]
+pub struct lzma_options_bcj {
+    pub start_offset: u32,
+}
+
+extern "C" {
+    pub fn lzma_code(strm: *mut lzma_stream, action: lzma_action) -> lzma_ret;
+    pub fn lzma_end(strm: *mut lzma_stream);
+    pub fn lzma_get_progress(strm: *mut lzma_stream, progress_in: *mut u64, progress_out: *mut u64);
+    pub fn lzma_memusage(strm: *const lzma_stream) -> u64;
+    pub fn lzma_memlimit_get(strm: *const lzma_stream) -> u64;
+    pub fn lzma_memlimit_set(strm: *mut lzma_stream, memlimit: u64) -> lzma_ret;
+
+    pub fn lzma_easy_encoder_memusage(preset: u32) -> u64;
+    pub fn lzma_easy_decoder_memusage(preset: u32) -> u64;
+    pub fn lzma_easy_encoder(strm: *mut lzma_stream, preset: u32, check: lzma_check) -> lzma_ret;
+    pub fn lzma_easy_buffer_encode(
+        preset: u32,
+        check: lzma_check,
+        allocator: *const lzma_allocator,
+        input: *const u8,
+        in_size: size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+
+    pub fn lzma_stream_encoder(
+        strm: *mut lzma_stream,
+        filters: *const lzma_filter,
+        check: lzma_check,
+    ) -> lzma_ret;
+    pub fn lzma_stream_encoder_mt_memusage(options: *const lzma_mt) -> u64;
+    pub fn lzma_stream_encoder_mt(strm: *mut lzma_stream, options: *const lzma_mt) -> lzma_ret;
+
+    pub fn lzma_alone_encoder(
+        strm: *mut lzma_stream,
+        options: *const lzma_options_lzma,
+    ) -> lzma_ret;
+
+    pub fn lzma_stream_buffer_bound(uncompressed_size: size_t) -> size_t;
+    pub fn lzma_stream_buffer_encode(
+        filters: *mut lzma_filter,
+        check: lzma_check,
+        allocator: *const lzma_allocator,
+        input: *const u8,
+        in_size: size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+
+    pub fn lzma_stream_decoder(strm: *mut lzma_stream, memlimit: u64, flags: u32) -> lzma_ret;
+    pub fn lzma_auto_decoder(strm: *mut lzma_stream, memlimit: u64, flags: u32) -> lzma_ret;
+    pub fn lzma_alone_decoder(strm: *mut lzma_stream, memlimit: u64) -> lzma_ret;
+    pub fn lzma_stream_buffer_decode(
+        memlimit: *mut u64,
+        flags: u32,
+        allocator: *const lzma_allocator,
+        input: *const u8,
+        in_pos: *mut size_t,
+        in_size: size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+
+    pub fn lzma_check_is_supported(check: lzma_check) -> lzma_bool;
+    pub fn lzma_check_size(check: lzma_check) -> u32;
+
+    pub fn lzma_crc32(buf: *const u8, size: size_t, crc: u32) -> u32;
+    pub fn lzma_crc64(buf: *const u8, size: size_t, crc: u64) -> u64;
+    pub fn lzma_get_check(strm: *const lzma_stream) -> lzma_check;
+
+    pub fn lzma_filter_encoder_is_supported(id: lzma_vli) -> lzma_bool;
+    pub fn lzma_filter_decoder_is_supported(id: lzma_vli) -> lzma_bool;
+    pub fn lzma_filters_copy(
+        src: *const lzma_filter,
+        dest: *mut lzma_filter,
+        allocator: *const lzma_allocator,
+    ) -> lzma_ret;
+    pub fn lzma_raw_encoder_memusage(filters: *const lzma_filter) -> u64;
+    pub fn lzma_raw_decoder_memusage(filters: *const lzma_filter) -> u64;
+    pub fn lzma_raw_encoder(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret;
+    pub fn lzma_raw_decoder(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret;
+    pub fn lzma_filters_update(strm: *mut lzma_stream, filters: *const lzma_filter) -> lzma_ret;
+    pub fn lzma_raw_buffer_encode(
+        filters: *const lzma_filter,
+        allocator: *const lzma_allocator,
+        input: *const u8,
+        in_size: size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+    pub fn lzma_raw_buffer_decode(
+        filters: *const lzma_filter,
+        allocator: *const lzma_allocator,
+        input: *const u8,
+        in_pos: *mut size_t,
+        in_size: size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+    pub fn lzma_properties_size(size: *mut u32, filter: *const lzma_filter) -> lzma_ret;
+    pub fn lzma_properties_encode(filter: *const lzma_filter, props: *mut u8) -> lzma_ret;
+    pub fn lzma_properties_decode(
+        filter: *mut lzma_filter,
+        allocator: *const lzma_allocator,
+        props: *const u8,
+        props_size: size_t,
+    ) -> lzma_ret;
+    pub fn lzma_physmem() -> u64;
+    pub fn lzma_cputhreads() -> u32;
+
+    pub fn lzma_stream_header_encode(options: *const lzma_stream_flags, out: *mut u8) -> lzma_ret;
+    pub fn lzma_stream_footer_encode(options: *const lzma_stream_flags, out: *mut u8) -> lzma_ret;
+    pub fn lzma_stream_header_decode(options: *mut lzma_stream_flags, input: *const u8)
+        -> lzma_ret;
+    pub fn lzma_stream_footer_decode(options: *mut lzma_stream_flags, input: *const u8)
+        -> lzma_ret;
+    pub fn lzma_stream_flags_compare(
+        a: *const lzma_stream_flags,
+        b: *const lzma_stream_flags,
+    ) -> lzma_ret;
+
+    pub fn lzma_version_number() -> u32;
+    pub fn lzma_version_string() -> *const c_char;
+
+    pub fn lzma_vli_encode(
+        vli: lzma_vli,
+        vli_pos: *mut size_t,
+        out: *mut u8,
+        out_pos: *mut size_t,
+        out_size: size_t,
+    ) -> lzma_ret;
+    pub fn lzma_vli_decode(
+        vli: *mut lzma_vli,
+        vli_pos: *mut size_t,
+        input: *const u8,
+        in_pos: *mut size_t,
+        in_size: size_t,
+    ) -> lzma_ret;
+    pub fn lzma_vli_size(vli: lzma_vli) -> u32;
+
+    pub fn lzma_lzma_preset(options: *mut lzma_options_lzma, preset: u32) -> lzma_bool;
+    pub fn lzma_mf_is_supported(mf: lzma_match_finder) -> lzma_bool;
+}