Intel® Integrated Performance Primitives
Community support and discussions relating to developing high-performance vision, signal, security, and storage applications.
6633 Discussions

Zlib compress/decompress > 64K buffer fails

robert-laumeyer
Beginner
543 Views
I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("\nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("\nFAILED %d\n", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

0 Kudos
1 Solution
Sergey_K_Intel
Employee
543 Views
Quoting - robert-laumeyer


The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


Please, take deflate.c anddeflate.h They must be OK.

View solution in original post

4 Replies
Sergey_K_Intel
Employee
543 Views
Please, modify line #755 of deflate.c
if( status == ippStsNoErr && ippflush == IppLZ77NoFlush) {
to
if( status == ippStsNoErr && strm->avail_in > 0) {

The problem must go away.

Regards,
Sergey

robert-laumeyer
Beginner
543 Views
Quoting - robert-laumeyer
I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("nFAILED %dn", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}



The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


Sergey_K_Intel
Employee
544 Views
Quoting - robert-laumeyer


The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


Please, take deflate.c anddeflate.h They must be OK.
robert-laumeyer
Beginner
543 Views
Quoting - sergey_kh
Please, take deflate.c anddeflate.h They must be OK.


That works! Thank you.
Reply