Community
cancel
Showing results for 
Search instead for 
Did you mean: 
Highlighted
74 Views

Zlib compress/decompress > 64K buffer fails

Jump to solution
I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("\nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("\nFAILED %d\n", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

0 Kudos

Accepted Solutions
Highlighted
Employee
74 Views
Quoting - robert-laumeyer


The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


Please, take deflate.c anddeflate.h They must be OK.

View solution in original post

0 Kudos
4 Replies
Highlighted
Employee
74 Views
Please, modify line #755 of deflate.c
if( status == ippStsNoErr && ippflush == IppLZ77NoFlush) {
to
if( status == ippStsNoErr && strm->avail_in > 0) {

The problem must go away.

Regards,
Sergey

0 Kudos
Highlighted
74 Views
Quoting - robert-laumeyer
I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);


if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;


while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("nFAILED %dn", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}



The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


0 Kudos
Highlighted
Employee
75 Views
Quoting - robert-laumeyer


The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.


Please, take deflate.c anddeflate.h They must be OK.

View solution in original post

0 Kudos
Highlighted
74 Views
Quoting - sergey_kh
Please, take deflate.c anddeflate.h They must be OK.


That works! Thank you.
0 Kudos