Zlib compress/decompress > 64K buffer fails

[robert-laumeyer]

I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("\nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("\nFAILED %d\n", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

[Sergey_K_Intel]

Please, modify line #755 of deflate.c
if( status == ippStsNoErr && ippflush == IppLZ77NoFlush) {
to
if( status == ippStsNoErr && strm->avail_in > 0) {

The problem must go away.

Regards,
Sergey

[robert-laumeyer]

Quoting - robert-laumeyer

I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("nFAILED %dn", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.

[Sergey_K_Intel · Accepted solution]

Quoting - robert-laumeyer

The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.

Please, take deflate.c anddeflate.h They must be OK.

[robert-laumeyer]

Quoting - sergey_kh

Please, take deflate.c anddeflate.h They must be OK.

That works! Thank you.