首页 > 编程语言 >压缩算法_quicklz接口demo

压缩算法_quicklz接口demo

时间：2024-02-02 21:22:05浏览次数：38  
标签：src QLZ demo quicklz unsigned char state hash 压缩算法

1 quicklz

  quicklz是单片机上一个常见的压缩算法，具体原理没有文档和hash表的相关基础我就不去深究了；

  只需要将fileSrc.txt放在桌面，代码可以使用vscode的mingw直接编译；

2 quicklz源码

quicklz.h 

/***quicklz.h***********************************************************************************/
#ifndef QLZ_HEADER
#define QLZ_HEADER


/* Compiler Related Definitions */
#ifdef __CC_ARM                         /* ARM Compiler */
#elif defined (__IAR_SYSTEMS_ICC__)     /* for IAR Compiler */
#define __inline                       inline
#elif defined (__GNUC__)                /* GNU GCC Compiler */
#else
    #error not supported tool chain
#endif

// Fast data compression library
// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
// [email protected]
//
// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
// released into public must be open source) or under a commercial license if such
// has been acquired (see http://www.quicklz.com/order.html). The commercial license
// does not cover derived or ported versions created by third parties under GPL.

// You can edit following user settings. Data must be decompressed with the same
// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed
// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially
// zeroed out (see manual). First #ifndef makes it possible to define settings from
// the outside like the compiler command line.

// 1.5.0 final


// 1 gives fastest compression speed. 3 gives fastest decompression speed and best
// compression ratio.
#ifndef QLZ_COMPRESSION_LEVEL
#define QLZ_COMPRESSION_LEVEL 1
//#define QLZ_COMPRESSION_LEVEL 2
//#define QLZ_COMPRESSION_LEVEL 3
#endif

// If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress()
// and decompress packets in the same order as they were compressed
#ifndef QLZ_STREAMING_BUFFER
#define QLZ_STREAMING_BUFFER 0
//#define QLZ_STREAMING_BUFFER 1000000
//#define QLZ_STREAMING_BUFFER 1000000
#endif

// Guarantees that decompression of corrupted data cannot crash. Decreases decompression
// speed 10-20%. Compression speed not affected.
#define QLZ_MEMORY_SAFE

#define QLZ_VERSION_MAJOR 1
#define QLZ_VERSION_MINOR 5
#define QLZ_VERSION_REVISION 0

// Buffer padding for destination buffer, least size + 400 bytes large because incompressible data may increase in size.
#define QLZ_BUFFER_PADDING    400

// Using size_t, memset() and memcpy()
#include <string.h>


#define PATH_FILE_SRC    "C:/Users/WINDOWS/Desktop/fileSrc.txt"
#define PATH_FILE_CPS    "C:/Users/WINDOWS/Desktop/fileCompress.qlz"
#define PATH_FILE_DST    "C:/Users/WINDOWS/Desktop/fileDst.txt"

// Verify compression level
#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3
#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3
#endif

typedef unsigned int ui32;
typedef unsigned short int ui16;

// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values!
#if QLZ_COMPRESSION_LEVEL == 1
#define QLZ_POINTERS 1
#define QLZ_HASH_VALUES 4096
#elif QLZ_COMPRESSION_LEVEL == 2
#define QLZ_POINTERS 4
#define QLZ_HASH_VALUES 2048
#elif QLZ_COMPRESSION_LEVEL == 3
#define QLZ_POINTERS 16
#define QLZ_HASH_VALUES 4096
#endif

// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization.
#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__
#define QLZ_PTR_64
#endif

// hash entry
typedef struct
{
#if QLZ_COMPRESSION_LEVEL == 1
    ui32 cache;
#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
    unsigned int offset;
#else
    const unsigned char *offset;
#endif
#else
    const unsigned char *offset[QLZ_POINTERS];
#endif

} qlz_hash_compress;

typedef struct
{
#if QLZ_COMPRESSION_LEVEL == 1
    const unsigned char *offset;
#else
    const unsigned char *offset[QLZ_POINTERS];
#endif
} qlz_hash_decompress;


// states
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
    unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
    size_t stream_counter;
    qlz_hash_compress hash[QLZ_HASH_VALUES];
    unsigned char hash_counter[QLZ_HASH_VALUES];
} qlz_state_compress;


#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
    unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
    qlz_hash_decompress hash[QLZ_HASH_VALUES];
    unsigned char hash_counter[QLZ_HASH_VALUES];
    size_t stream_counter;
} qlz_state_decompress;
#elif QLZ_COMPRESSION_LEVEL == 3
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
    unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
#if QLZ_COMPRESSION_LEVEL <= 2
    qlz_hash_decompress hash[QLZ_HASH_VALUES];
#endif
    size_t stream_counter;
} qlz_state_decompress;
#endif


#if defined (__cplusplus)
extern "C" {
#endif

// Public functions of QuickLZ
    size_t qlz_size_decompressed(const char *source);
    size_t qlz_size_compressed(const char *source);
    size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state);
    size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state);
    int qlz_get_setting(int setting);

#if defined (__cplusplus)
}
#endif

#endif
quicklz.c 
 /***quicklz.c*****************************************************************************************/
// Fast data compression library
// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
// [email protected]
//
// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
// released into public must be open source) or under a commercial license if such
// has been acquired (see http://www.quicklz.com/order.html). The commercial license
// does not cover derived or ported versions created by third parties under GPL.

// 1.5.0 final

#include "quicklz.h"


#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
#error quicklz.c and quicklz.h have different versions
#endif

#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
#define X86X64
#endif

#define MINOFFSET 2
#define UNCONDITIONAL_MATCHLEN 6
#define UNCOMPRESSED_END 4
#define CWORD_LEN 4

#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
#define OFFSET_BASE source
#define CAST (ui32)(size_t)
#else
#define OFFSET_BASE 0
#define CAST
#endif

int qlz_get_setting(int setting)
{
    switch(setting)
    {
        case 0:
            return QLZ_COMPRESSION_LEVEL;
        case 1:
            return sizeof(qlz_state_compress);
        case 2:
            return sizeof(qlz_state_decompress);
        case 3:
            return QLZ_STREAMING_BUFFER;
#ifdef QLZ_MEMORY_SAFE
        case 6:
            return 1;
#else
        case 6:
            return 0;
#endif
        case 7:
            return QLZ_VERSION_MAJOR;
        case 8:
            return QLZ_VERSION_MINOR;
        case 9:
            return QLZ_VERSION_REVISION;
    }
    return -1;
}

#if QLZ_COMPRESSION_LEVEL == 1
static int same(const unsigned char *src, size_t n)
{
    while(n > 0 && *(src + n) == *src)
        n--;
    return n == 0 ? 1 : 0;
}
#endif

static void reset_table_compress(qlz_state_compress *state)
{
    int i;
    for(i = 0; i < QLZ_HASH_VALUES; i++)
    {
#if QLZ_COMPRESSION_LEVEL == 1
        state->hash[i].offset = 0;
#else
        state->hash_counter[i] = 0;
#endif
    }
}

static void reset_table_decompress(qlz_state_decompress *state)
{
    int i;
    (void)state;
    (void)i;
#if QLZ_COMPRESSION_LEVEL == 2
    for(i = 0; i < QLZ_HASH_VALUES; i++)
    {
        state->hash_counter[i] = 0;
    }
#endif
}

static __inline ui32 hash_func(ui32 i)
{
#if QLZ_COMPRESSION_LEVEL == 2
    return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
#else
    return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
#endif
}

static __inline ui32 fast_read(void const *src, ui32 bytes)
{
#ifndef X86X64
    unsigned char *p = (unsigned char *)src;
    switch(bytes)
    {
        case 4:
            return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
        case 3:
            return(*p | *(p + 1) << 8 | *(p + 2) << 16);
        case 2:
            return(*p | *(p + 1) << 8);
        case 1:
            return(*p);
    }
    return 0;
#else
    if(bytes >= 1 && bytes <= 4)
        return *((ui32 *)src);
    else
        return 0;
#endif
}

static __inline ui32 hashat(const unsigned char *src)
{
    ui32 fetch, hash;
    fetch = fast_read(src, 3);
    hash = hash_func(fetch);
    return hash;
}

static __inline void fast_write(ui32 f, void *dst, size_t bytes)
{
#ifndef X86X64
    unsigned char *p = (unsigned char *)dst;

    switch(bytes)
    {
        case 4:
            *p = (unsigned char)f;
            *(p + 1) = (unsigned char)(f >> 8);
            *(p + 2) = (unsigned char)(f >> 16);
            *(p + 3) = (unsigned char)(f >> 24);
            return;
        case 3:
            *p = (unsigned char)f;
            *(p + 1) = (unsigned char)(f >> 8);
            *(p + 2) = (unsigned char)(f >> 16);
            return;
        case 2:
            *p = (unsigned char)f;
            *(p + 1) = (unsigned char)(f >> 8);
            return;
        case 1:
            *p = (unsigned char)f;
            return;
    }
#else
    switch(bytes)
    {
        case 4:
            *((ui32 *)dst) = f;
            return;
        case 3:
            *((ui32 *)dst) = f;
            return;
        case 2:
            *((ui16 *)dst) = (ui16)f;
            return;
        case 1:
            *((unsigned char *)dst) = (unsigned char)f;
            return;
    }
#endif
}


size_t qlz_size_decompressed(const char *source)
{
    ui32 n, r;
    n = (((*source) & 2) == 2) ? 4 : 1;
    r = fast_read(source + 1 + n, n);
    r = r & (0xffffffff >> ((4 - n) * 8));
    return r;
}

size_t qlz_size_compressed(const char *source)
{
    ui32 n, r;
    n = (((*source) & 2) == 2) ? 4 : 1;
    r = fast_read(source + 1, n);
    r = r & (0xffffffff >> ((4 - n) * 8));
    return r;
}

size_t qlz_size_header(const char *source)
{
    size_t n = 2 * ((((*source) & 2) == 2) ? 4 : 1) + 1;
    return n;
}


static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
{
    // Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
#ifndef X86X64
    unsigned char *end = dst + n;
    while(dst < end)
    {
        *dst = *src;
        dst++;
        src++;
    }
#else
    ui32 f = 0;
    do
    {
        *(ui32 *)(dst + f) = *(ui32 *)(src + f);
        f += MINOFFSET + 1;
    }
    while(f < n);
#endif
}

static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1
    ui32 hash;
    hash = hashat(s);
    state->hash[hash].offset = s;
    state->hash_counter[hash] = 1;
#elif QLZ_COMPRESSION_LEVEL == 2
    ui32 hash;
    unsigned char c;
    hash = hashat(s);
    c = state->hash_counter[hash];
    state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
    c++;
    state->hash_counter[hash] = c;
#endif
    (void)state;
    (void)s;
}

#if QLZ_COMPRESSION_LEVEL <= 2
static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
{
    while(*lh < max)
    {
        (*lh)++;
        update_hash(state, *lh);
    }
}
#endif

static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
{
    const unsigned char *last_byte = source + size - 1;
    const unsigned char *src = source;
    unsigned char *cword_ptr = destination;
    unsigned char *dst = destination + CWORD_LEN;
    ui32 cword_val = 1U << 31;
    const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
    ui32 fetch = 0;
    unsigned int lits = 0;

    (void) lits;

    if(src <= last_matchstart)
        fetch = fast_read(src, 3);

    while(src <= last_matchstart)
    {
        if((cword_val & 1) == 1)
        {
            // store uncompressed if compression ratio is too low
            if(src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
                return 0;

            fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);

            cword_ptr = dst;
            dst += CWORD_LEN;
            cword_val = 1U << 31;
            fetch = fast_read(src, 3);
        }
#if QLZ_COMPRESSION_LEVEL == 1
        {
            const unsigned char *o;
            ui32 hash, cached;

            hash = hash_func(fetch);
            cached = fetch ^ state->hash[hash].cache;
            state->hash[hash].cache = fetch;

            o = state->hash[hash].offset + OFFSET_BASE;
            state->hash[hash].offset = CAST(src - OFFSET_BASE);

#ifdef X86X64
            if((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
            {
                if(cached != 0)
                {
#else
            if(cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
            {
                if(*(o + 3) != *(src + 3))
                {
#endif
                    hash <<= 4;
                    cword_val = (cword_val >> 1) | (1U << 31);
                    fast_write((3 - 2) | hash, dst, 2);
                    src += 3;
                    dst += 2;
                }
                else
                {
                    const unsigned char *old_src = src;
                    size_t matchlen;
                    hash <<= 4;

                    cword_val = (cword_val >> 1) | (1U << 31);
                    src += 4;

                    if(*(o + (src - old_src)) == *src)
                    {
                        src++;
                        if(*(o + (src - old_src)) == *src)
                        {
                            size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
                            size_t remaining = q > 255 ? 255 : q;
                            src++;
                            while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
                                src++;
                        }
                    }

                    matchlen = src - old_src;
                    if(matchlen < 18)
                    {
                        fast_write((ui32)(matchlen - 2) | hash, dst, 2);
                        dst += 2;
                    }
                    else
                    {
                        fast_write((ui32)(matchlen << 16) | hash, dst, 3);
                        dst += 3;
                    }
                }
                fetch = fast_read(src, 3);
                lits = 0;
            }
            else
            {
                lits++;
                *dst = *src;
                src++;
                dst++;
                cword_val = (cword_val >> 1);
#ifdef X86X64
                fetch = fast_read(src, 3);
#else
                fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
#endif
            }
        }
#elif QLZ_COMPRESSION_LEVEL >= 2
        {
            const unsigned char *o, *offset2;
            ui32 hash, matchlen, k, m, best_k = 0;
            unsigned char c;
            size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
            (void)best_k;


            //hash = hashat(src);
            fetch = fast_read(src, 3);
            hash = hash_func(fetch);

            c = state->hash_counter[hash];

            offset2 = state->hash[hash].offset[0];
            if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
            {
                matchlen = 3;
                if(*(offset2 + matchlen) == *(src + matchlen))
                {
                    matchlen = 4;
                    while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
                        matchlen++;
                }
            }
            else
                matchlen = 0;
            for(k = 1; k < QLZ_POINTERS && c > k; k++)
            {
                o = state->hash[hash].offset[k];
#if QLZ_COMPRESSION_LEVEL == 3
                if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#elif QLZ_COMPRESSION_LEVEL == 2
                if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#endif
                {
                    m = 3;
                    while(*(o + m) == *(src + m) && m < remaining)
                        m++;
#if QLZ_COMPRESSION_LEVEL == 3
                    if((m > matchlen) || (m == matchlen && o > offset2))
#elif QLZ_COMPRESSION_LEVEL == 2
                    if(m > matchlen)
#endif
                    {
                        offset2 = o;
                        matchlen = m;
                        best_k = k;
                    }
                }
            }
            o = offset2;
            state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
            c++;
            state->hash_counter[hash] = c;

#if QLZ_COMPRESSION_LEVEL == 3
            if(matchlen > 2 && src - o < 131071)
            {
                ui32 u;
                size_t offset = src - o;

                for(u = 1; u < matchlen; u++)
                {
                    hash = hashat(src + u);
                    c = state->hash_counter[hash]++;
                    state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
                }

                cword_val = (cword_val >> 1) | (1U << 31);
                src += matchlen;

                if(matchlen == 3 && offset <= 63)
                {
                    *dst = (unsigned char)(offset << 2);
                    dst++;
                }
                else if(matchlen == 3 && offset <= 16383)
                {
                    ui32 f = (ui32)((offset << 2) | 1);
                    fast_write(f, dst, 2);
                    dst += 2;
                }
                else if(matchlen <= 18 && offset <= 1023)
                {
                    ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
                    fast_write(f, dst, 2);
                    dst += 2;
                }

                else if(matchlen <= 33)
                {
                    ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
                    fast_write(f, dst, 3);
                    dst += 3;
                }
                else
                {
                    ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
                    fast_write(f, dst, 4);
                    dst += 4;
                }
            }
            else
            {
                *dst = *src;
                src++;
                dst++;
                cword_val = (cword_val >> 1);
            }
#elif QLZ_COMPRESSION_LEVEL == 2

            if(matchlen > 2)
            {
                cword_val = (cword_val >> 1) | (1U << 31);
                src += matchlen;

                if(matchlen < 10)
                {
                    ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
                    fast_write(f, dst, 2);
                    dst += 2;
                }
                else
                {
                    ui32 f = best_k | (matchlen << 16) | (hash << 5);
                    fast_write(f, dst, 3);
                    dst += 3;
                }
            }
            else
            {
                *dst = *src;
                src++;
                dst++;
                cword_val = (cword_val >> 1);
            }
#endif
        }
#endif
    }
    while(src <= last_byte)
    {
        if((cword_val & 1) == 1)
        {
            fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
            cword_ptr = dst;
            dst += CWORD_LEN;
            cword_val = 1U << 31;
        }
#if QLZ_COMPRESSION_LEVEL < 3
        if(src <= last_byte - 3)
        {
#if QLZ_COMPRESSION_LEVEL == 1
            ui32 hash, fetch;
            fetch = fast_read(src, 3);
            hash = hash_func(fetch);
            state->hash[hash].offset = CAST(src - OFFSET_BASE);
            state->hash[hash].cache = fetch;
#elif QLZ_COMPRESSION_LEVEL == 2
            ui32 hash;
            unsigned char c;
            hash = hashat(src);
            c = state->hash_counter[hash];
            state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
            c++;
            state->hash_counter[hash] = c;
#endif
        }
#endif
        *dst = *src;
        src++;
        dst++;
        cword_val = (cword_val >> 1);
    }

    while((cword_val & 1) != 1)
        cword_val = (cword_val >> 1);

    fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);

    // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
    return dst - destination < 9 ? 9 : dst - destination;
}

static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
{
    const unsigned char *src = source + qlz_size_header((const char *)source);
    unsigned char *dst = destination;
    const unsigned char *last_destination_byte = destination + size - 1;
    ui32 cword_val = 1;
    const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
    unsigned char *last_hashed = destination - 1;
    const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
    static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};

    (void) last_source_byte;
    (void) last_hashed;
    (void) state;
    (void) history;

    for(;;)
    {
        ui32 fetch;

        if(cword_val == 1)
        {
#ifdef QLZ_MEMORY_SAFE
            if(src + CWORD_LEN - 1 > last_source_byte)
                return 0;
#endif
            cword_val = fast_read(src, CWORD_LEN);
            src += CWORD_LEN;
        }

#ifdef QLZ_MEMORY_SAFE
        if(src + 4 - 1 > last_source_byte)
            return 0;
#endif

        fetch = fast_read(src, 4);

        if((cword_val & 1) == 1)
        {
            ui32 matchlen;
            const unsigned char *offset2;

#if QLZ_COMPRESSION_LEVEL == 1
            ui32 hash;
            cword_val = cword_val >> 1;
            hash = (fetch >> 4) & 0xfff;
            offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;

            if((fetch & 0xf) != 0)
            {
                matchlen = (fetch & 0xf) + 2;
                src += 2;
            }
            else
            {
                matchlen = *(src + 2);
                src += 3;
            }

#elif QLZ_COMPRESSION_LEVEL == 2
            ui32 hash;
            unsigned char c;
            cword_val = cword_val >> 1;
            hash = (fetch >> 5) & 0x7ff;
            c = (unsigned char)(fetch & 0x3);
            offset2 = state->hash[hash].offset[c];

            if((fetch & (28)) != 0)
            {
                matchlen = ((fetch >> 2) & 0x7) + 2;
                src += 2;
            }
            else
            {
                matchlen = *(src + 2);
                src += 3;
            }

#elif QLZ_COMPRESSION_LEVEL == 3
            ui32 offset;
            cword_val = cword_val >> 1;
            if((fetch & 3) == 0)
            {
                offset = (fetch & 0xff) >> 2;
                matchlen = 3;
                src++;
            }
            else if((fetch & 2) == 0)
            {
                offset = (fetch & 0xffff) >> 2;
                matchlen = 3;
                src += 2;
            }
            else if((fetch & 1) == 0)
            {
                offset = (fetch & 0xffff) >> 6;
                matchlen = ((fetch >> 2) & 15) + 3;
                src += 2;
            }
            else if((fetch & 127) != 3)
            {
                offset = (fetch >> 7) & 0x1ffff;
                matchlen = ((fetch >> 2) & 0x1f) + 2;
                src += 3;
            }
            else
            {
                offset = (fetch >> 15);
                matchlen = ((fetch >> 7) & 255) + 3;
                src += 4;
            }

            offset2 = dst - offset;
#endif

#ifdef QLZ_MEMORY_SAFE
            if(offset2 < history || offset2 > dst - MINOFFSET - 1)
                return 0;

            if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
                return 0;
#endif

            memcpy_up(dst, offset2, matchlen);
            dst += matchlen;

#if QLZ_COMPRESSION_LEVEL <= 2
            update_hash_upto(state, &last_hashed, dst - matchlen);
            last_hashed = dst - 1;
#endif
        }
        else
        {
            if(dst < last_matchstart)
            {
                unsigned int n = bitlut[cword_val & 0xf];
#ifdef X86X64
                *(ui32 *)dst = *(ui32 *)src;
#else
                memcpy_up(dst, src, 4);
#endif
                cword_val = cword_val >> n;
                dst += n;
                src += n;
#if QLZ_COMPRESSION_LEVEL <= 2
                update_hash_upto(state, &last_hashed, dst - 3);
#endif
            }
            else
            {
                while(dst <= last_destination_byte)
                {
                    if(cword_val == 1)
                    {
                        src += CWORD_LEN;
                        cword_val = 1U << 31;
                    }
#ifdef QLZ_MEMORY_SAFE
                    if(src >= last_source_byte + 1)
                        return 0;
#endif
                    *dst = *src;
                    dst++;
                    src++;
                    cword_val = cword_val >> 1;
                }

#if QLZ_COMPRESSION_LEVEL <= 2
                update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
#endif
                return size;
            }

        }
    }
}

size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
{
    size_t r;
    ui32 compressed;
    size_t base;

    if(size == 0 || size > 0xffffffff - 400)
        return 0;

    if(size < 216)
        base = 3;
    else
        base = 9;

#if QLZ_STREAMING_BUFFER > 0
    if(state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
#endif
    {
        reset_table_compress(state);
        r = base + qlz_compress_core((const unsigned char *)source, (unsigned char *)destination + base, size, state);
#if QLZ_STREAMING_BUFFER > 0
        reset_table_compress(state);
#endif
        if(r == base)
        {
            memcpy(destination + base, source, size);
            r = size + base;
            compressed = 0;
        }
        else
        {
            compressed = 1;
        }
        state->stream_counter = 0;
    }
#if QLZ_STREAMING_BUFFER > 0
    else
    {
        unsigned char *src = state->stream_buffer + state->stream_counter;

        memcpy(src, source, size);
        r = base + qlz_compress_core(src, (unsigned char *)destination + base, size, state);

        if(r == base)
        {
            memcpy(destination + base, src, size);
            r = size + base;
            compressed = 0;
            reset_table_compress(state);
        }
        else
        {
            compressed = 1;
        }
        state->stream_counter += size;
    }
#endif
    if(base == 3)
    {
        *destination = (unsigned char)(0 | compressed);
        *(destination + 1) = (unsigned char)r;
        *(destination + 2) = (unsigned char)size;
    }
    else
    {
        *destination = (unsigned char)(2 | compressed);
        fast_write((ui32)r, destination + 1, 4);
        fast_write((ui32)size, destination + 5, 4);
    }

    *destination |= (QLZ_COMPRESSION_LEVEL << 2);
    *destination |= (1 << 6);
    *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);

// 76543210
// 01SSLLHC

    return r;
}

size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
{
    size_t dsiz = qlz_size_decompressed(source);

#if QLZ_STREAMING_BUFFER > 0
    if(state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
#endif
    {
        if((*source & 1) == 1)
        {
            reset_table_decompress(state);
            dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
        }
        else
        {
            memcpy(destination, source + qlz_size_header(source), dsiz);
        }
        state->stream_counter = 0;
        reset_table_decompress(state);
    }
#if QLZ_STREAMING_BUFFER > 0
    else
    {
        unsigned char *dst = state->stream_buffer + state->stream_counter;
        if((*source & 1) == 1)
        {
            dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
        }
        else
        {
            memcpy(dst, source + qlz_size_header(source), dsiz);
            reset_table_decompress(state);
        }
        memcpy(destination, dst, dsiz);
        state->stream_counter += dsiz;
    }
#endif
    return dsiz;
}
fileSrc.txt 
/***Put this file on Desktop as srcFile for quicklz compress；***/

// attention
//1 compress buff no more than 2048 size;
//2 because compressed transmit buff need to read qlz algrorithm parameter together for decompress,
//   and parameter will count into decompressed buff;
//   may lead to some mistake decompress;

4 quicklz 函数接口


/***main.c**************************************************************************************************/
//vscode直接使用mingw编译的时候没法包含多个c文件查找,需要配置vscode包含路径,不想配先放着;
//所以quicklz.c代码需要放到main函数所在c文件中，不然会查找不到；


#include <stdio.h>  
#include <stdlib.h> 
#include "quicklz.h"


#define PATH_FILE_SRC    "C:/Users/WINDOWS/Desktop/fileSrc.txt"
#define PATH_FILE_CPS    "C:/Users/WINDOWS/Desktop/fileCompress.qlz"
#define PATH_FILE_DST    "C:/Users/WINDOWS/Desktop/fileDst.txt"


int qlzCompress(void)
{
	printf("qlzCompress() \n");
	
	FILE *fsrcFile;
	FILE *fcpsFile;
    unsigned char psrcTrans[2048] ={0};      
    unsigned char pdstTrans[2048] ={0};     
	size_t srcFileLen, cpsFileLen; 
	qlz_state_compress state_compress;
    
	//fsrcFile handle;
    fsrcFile = fopen(PATH_FILE_SRC, "r");

	//fsrcFile length;
    fseek(fsrcFile, 0, SEEK_END);
    srcFileLen = ftell(fsrcFile);
	
	//read fsrcFile >> psrcTrans;
	printf("srcFileLen:(d)%08d  \n",srcFileLen);
	fseek(fsrcFile, 0, SEEK_SET);
    fread(psrcTrans, sizeof(char), srcFileLen, fsrcFile);
	//for(int i =0; i<srcFileLen;i++)  printf("%c",psrcTrans[i]); printf("\n");

	//fcpsFile handle;
	fcpsFile = fopen(PATH_FILE_CPS, "w");

	//compress psrcTrans >> pdstTrans;
    cpsFileLen = qlz_compress(psrcTrans, pdstTrans, srcFileLen, &state_compress);
	
	printf("cpsFileLen:(d)%08d  \n",cpsFileLen);
    fwrite(pdstTrans, sizeof(char), cpsFileLen, fcpsFile);

    fclose(fsrcFile);
    fclose(fcpsFile);
    return 0;
}



int qlzDecompress(void)
{
   printf("qlzDecompress()  \n");
   
   FILE *fcpsFile;
   FILE *fdstFile;
   unsigned char *pcpsTrans;
   unsigned char *pdstTrans;
   qlz_state_decompress state_decompress;
   unsigned int cpsFileLen;
   unsigned int decpsFileLen;

   //fcpsFile handle;
   fcpsFile = fopen(PATH_FILE_CPS, "r");

   //fcpsFile length;
   fseek(fcpsFile, 0, SEEK_END);
   cpsFileLen = ftell(fcpsFile);

   //pcpsTrans buff
   pcpsTrans = (unsigned char *)malloc(cpsFileLen);
   if(pcpsTrans==NULL){
   	   printf("malloc cps buff fail.  \n");
	   return -2;
   }

   //read fcpsFile >> pcpsTrans;
   printf("cpsFileLen:(d)%08d \n",cpsFileLen);
   fseek(fcpsFile, 0, SEEK_SET);
   fread(pcpsTrans, sizeof(char), cpsFileLen, fcpsFile);
   //for(int i =0;i< cpsFileLen;i++)  printf("%02x ",pcpsTrans[i]); printf("\n");

   //pdstTrans length;
   decpsFileLen = qlz_size_decompressed(pcpsTrans);
   pdstTrans = (unsigned char *)malloc(decpsFileLen);
   if(pdstTrans ==NULL){
   	  printf("malloc pdstTrans fail. \n");
	  return -3;
   }

   //bcz pcpsTrans read some more qlz para,
   //so after decompress will have some more space empty in fileDst.txt? bug leave;
   printf("decpsFileLen:(d)%08d \n",decpsFileLen);
   decpsFileLen = qlz_decompress(pcpsTrans, pdstTrans, &state_decompress); 

   fdstFile = fopen(PATH_FILE_DST, "w");
   fwrite(pdstTrans, sizeof(char), decpsFileLen, fdstFile);
   
   fclose(fcpsFile);
   fclose(fdstFile);
   free(pcpsTrans);
   free(pdstTrans);
   return 0;
}


int main()
{
    printf("hello  \n");
	//文件压缩完之后会多出部分qlz的参数,分配内存时需要考虑;
	//因为解压时是通过读取文件长度来解压的，所以解压后结尾部分会把这部分多出的也解压了;qlz本身小bug??

	//这个bug的解决办法是在使用数据的时候只读取解压长度的数据，丢弃掉多余字节；
	//所以在使用数据的时候，需要通过解压前数据长度来使用；
	//上面的接口函数只是压缩解压，不涉及数据处理使用；
	
	//qlz解压最大不超过4096,需要考虑;
	//函数内数组不要超出栈空间大小;
	
    qlzCompress();
    qlzDecompress();
    return 0;
}

5 文件操作读写模式

  

  前两列判断fopen的操作；后两列判断如果可以执行当前函数操作的话pfile的文件偏移位置；

  需要注意的是图中斜杆部分读写无此操作，但执行不会报错，导致不容易发现bug，

  通过判断fopen、fread、fwrite返回值来判断文件操作是否成功失败，可以提高代码的健壮性和稳定性；

6 小结

  这个接口主要还有三个地方需要优化的；

  一是vscode编辑器配置多个c文件的问题，但是我懒的配置了，先放着吧；

  二是解压成源文件之后，解压出来的文件结尾虽然没有数据丢失但是会有部分空格多余，需要丢弃，先放着吧；

  三是读写接口可以优化，加上返回值判断，也放着把；

标签：src,QLZ,demo,quicklz,unsigned,char,state,hash,压缩算法
From： https://www.cnblogs.com/caesura-k/p/18002720

相关文章

  • 通过Demo学WPF—数据绑定（二）
    准备今天学习的Demo是DataBinding中的Linq：创建一个空白解决方案，然后添加现有项目，选择Linq，解决方案如下所示：查看这个Demo的效果：开始学习这个Demoxaml部分查看MainWindow.xaml：<Windowx:Class="Linq.MainWindow"xmlns="http://schemas.microsoft.com/winfx/2006......
  • java flink（二十六） 实战之电商黑名单过滤 Flink CEP编程实现、什么是CEP、CEP组合模式d
    javaflink（二十六）实战之电商黑名单过滤FlinkCEP编程实现、什么是CEP、CEP组合模式demo、CEP循环模式demo什么是CEP：1、复杂事件处理2、Flink中实现复杂事件处理库3、CEP允许在无休止的事件中检测事件模式，让我们有机会掌握数据中的重要部分4、一个或多个由简单事件构成的事......
  • 通过Demo学WPF—数据绑定（一）✨
    前言✨想学习WPF，但是看视频教程觉得太耗时间，直接看文档又觉得似懂非懂，因此想通过看Demo代码+文档的方式进行学习。准备✨微软官方其实提供了WPF的一些Demo，地址为：microsoft/WPF-Samples:RepositoryforWPFrelatedsamples(github.com)将其克隆到本地，有很多的Demo代码：新建......
  • 大文件分片上传demo，前端基于Uppy，
    实现分片上传并且支持断点续传需要基于TusTus是一种开放协议，用于基于HTTP构建的可恢复上传。这意味着意外关闭选项卡或失去连接，让您继续，对于实例，您的10GB上传，而不是重新开始。后端后端变化挺大的，你需要将你的服务器变得支持Tus，刚好官方提供了对应的插件（Java后台、php后......
  • 关系网demo
    树状图显示层级关系，使用d3js竖形<!DOCTYPEhtml><htmllang="en"><head><metacharset="UTF-8"/><metaname="viewport"content="width=device-width,initial-scale=1.0"/><title>......
  • 虹软sdk实现人脸识别小demo
    虹软官网：https://ai.arcsoft.com.cn注册后，下载适配的sdk注意提取下载的sdk中的jar包，代码中需要用到<dependency><groupId>com.arcsoft.face</groupId><artifactId>arcsoft-sdk-face</artifactId><version>3.0.0.0</version><scope&......
  • 不能在此路径中使用此配置节。如果在父级别上锁定了该节，便会出现这种情况。锁定是默认
    今天运行项目的时候出现了这个错误....查了一下解决的方法。具体方案如下： 1、先确认安装IIS的时候有没有装Asp.Net，如果没安装的话，安装上即可。（XTHS：采用这步，就可以了！） 2、IIS采用了更安全的web.config管理机制，默认情况下会锁住配置项不允许更改。用超级管理员的身份执......
  • rocketmq--同步、异步、批量、事务消息demo
    在SpringBoot中使用RocketMQ进行同步和异步消息传输的关键是使用RocketMQTemplate类。下面是两个例子，分别演示了如何实现同步和异步消息传输。首先，确保你已经添加了RocketMQ的依赖到你的pom.xml中，如下所示：<dependency><groupId>org.apache.rocketmq</groupId><artifa......
  • rocketmq--消息顺序消费demo
    在RocketMQ中，要实现消息的顺序消费，你需要确保以下几点：发送消息时，相同业务顺序的消息应该发送到同一个队列（MessageQueue）。消费者在消费时，应该使用顺序消费的方式。下面是一个使用SpringBoot和RocketMQ实现消息顺序消费的例子。添加依赖(pom.xml):<dependencies><de......
  • rocketmq--两种消息模型的区别及demo
    RocketMQ主要支持两种消息模型：集群消费（Clustering）和广播消费（Broadcasting）。集群消费（Clustering）：在集群消费模式下，同一个消费者组（ConsumerGroup）中的消费者实例平均分摊消费消息，即一个消息只会被消费者组中的一个消费者消费一次。这种模式适用于负载均衡场景，可以提高消费的并......

赞助商

阅读排行

网站主页关于我们联系我们网站地图

本网站内容转载自其他媒体，侵权联系[admin##ips99.com]。

Copyright © 2020-2023 IPS99 版权所有 IPS99