Skip to content

Commit

Permalink
handle large inflate output with dynamic buffer, compile with 1 command
Browse files Browse the repository at this point in the history
  • Loading branch information
fangq committed May 4, 2019
1 parent 86a0dea commit bd099b9
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 15 deletions.
11 changes: 0 additions & 11 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -140,17 +140,6 @@ base64 encoding/decoding to strings.
Please run these examples and understand how ZMat works before you use
it to process your data.

=======================
Known Issues and TODOs
=======================

ZMat has several known limitations. We are striving to make it more general
and robust. Hopefully in a few future releases, the limitations become less.

Here are the known issues:

* The default output buffer to store decompressed data is 20x of the input data size. If the compression ratio is higher than that, zmat may produce an error

==========================
Contribution and feedback
==========================
Expand Down
6 changes: 3 additions & 3 deletions src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ OUTPUTFLAG:=-o
OBJSUFFIX=.o
EXESUFFIX=

FILES=zmat
FILES=

ifeq ($(findstring CYGWIN,$(PLATFORM)), CYGWIN)
CC=nvcc
Expand Down Expand Up @@ -75,14 +75,14 @@ else
endif

oct: OUTPUT_DIR=..
oct: AR= CXXFLAGS='-O3' LFLAGS='$(-lz)' LDFLAGS='$(LFLAGS)' mkoctfile
oct: AR= CXXFLAGS='-O3' LFLAGS='$(-lz)' LDFLAGS='$(LFLAGS)' mkoctfile zmat.cpp
oct: BINARY=zmat.mex
oct: LINKOPT+=--mex $(INCLUDEDIRS)
oct: CXX=mkoctfile

mex: CXX=$(MEX)
mex: OUTPUTFLAG:=-output
mex: AR=$(MEX)
mex: AR=$(MEX) zmat.cpp
mex: LINKOPT+= -cxx CXXLIBS='$$CXXLIBS -lz' -outdir $(ZMATDIR)
mex: OUTPUT_DIR=..

Expand Down
8 changes: 7 additions & 1 deletion src/zmat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]){
if(zipid==zmBase64){
temp=base64_decode((const unsigned char*)inputstr, inputsize, &outputsize);
}else{
int count=1;
if(zipid==zmZlib){
if(inflateInit(&zs) != Z_OK)
mexErrMsgTxt("failed to initialize zlib");
Expand All @@ -136,7 +137,12 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]){

zs.next_out = (Bytef *)(temp); //(Bytef *)(); // output char array

ret=inflate(&zs, Z_FINISH);
while((ret=inflate(&zs, Z_SYNC_FLUSH))!=Z_STREAM_END && count<=10){
temp=(unsigned char *)realloc(temp, (buflen[0]<<count));
zs.next_out = (Bytef *)(temp+(buflen[0]<<(count-1))); //(Bytef *)(); // output char array
zs.avail_out = (buflen[0]<<(count-1)); // size of output
count++;
}
outputsize=zs.total_out;

if(ret!=Z_STREAM_END && ret!=Z_OK)
Expand Down

0 comments on commit bd099b9

Please sign in to comment.