hio.zlib.impl source code

1 module hio.zlib.impl;
2 
3 import std.experimental.logger;
4 import std.string;
5 import std.stdio;
6 import std.algorithm;
7 import std.typecons;
8 
9 import etc.c.zlib;
10 
11 import nbuff: Nbuff, NbuffChunk, MutableNbuffChunk;
12 
13 
14 enum OBUFFSZ = 16*1024;
15 
16 alias zInflateResult = Tuple!(ulong, "consumed", NbuffChunk, "result", int, "status");
17 
18 struct ZLib
19 {
20     private
21     {
22         z_stream          _zstr;
23         uint              _obufsz = 16*1024;
24         MutableNbuffChunk _outBuff;
25         int               _state = Z_OK;
26     }
27 
28     void zInit(uint obufsz)
29     {
30         immutable r = inflateInit2(&_zstr, 15+32); // allow autodetect
31         assert(r == Z_OK);
32         _obufsz = obufsz;
33         _outBuff = Nbuff.get(_obufsz);
34         _zstr.next_out = &_outBuff.data.ptr[0];
35         _zstr.avail_out = _obufsz;
36         _state = Z_OK;
37     }
38 
39     zInflateResult zInflate(NbuffChunk b)
40     {
41         // process data in b
42         // after call to inflate we can have
43         // 0) avail_in =  0 - everything processed (input empty)
44         //    avail_out>= 0 - 
45         //
46         // 1) avail_in >   0 - something consumed but...
47         //    avail_out == 0 - no space in out buffer
48         //
49         // 2) 
50         // avail_in == 0
51         // avail_out > 0
52         if (_state != Z_OK)
53         {
54             return zInflateResult(0, NbuffChunk(), _state);
55         }
56         _zstr.next_in = &b.data[0];
57         _zstr.avail_in = cast(int)b.length;
58         int rc = inflate(&_zstr, 0);
59         switch(rc)
60         {
61             case Z_OK, Z_STREAM_END, Z_STREAM_ERROR, Z_DATA_ERROR:
62                 _state = rc;
63                 break;
64             default:
65                 break;
66         }
67         // writefln("r=%d, avail_in=%d, avail_out=%d", rc, _zstr.avail_in, _zstr.avail_out);
68         // writefln("in progress = %d", b.length - _zstr.avail_in);
69         // writefln("out space = %d", _zstr.avail_out);
70         // writefln("data=%s", cast(string)_outBuff.data[0.._obufsz-_zstr.avail_out]);
71         if ( _zstr.avail_out == 0 || rc == Z_STREAM_END)
72         {
73             auto res = zInflateResult(
74                 b.length - _zstr.avail_in,
75                 NbuffChunk(_outBuff, min(_obufsz - _zstr.avail_out, _obufsz)),
76                 rc);
77             _outBuff = Nbuff.get(_obufsz);
78             _zstr.avail_out = _obufsz;
79             _zstr.next_out = &_outBuff.data.ptr[0];
80             return res;
81         }
82         else
83         {
84             return zInflateResult(b.length - _zstr.avail_in, NbuffChunk(), rc);
85         }
86     }
87 
88     void zFlush() @trusted
89     {
90         inflateEnd(&_zstr);
91     }
92 }
93 
94 
95 unittest
96 {
97     import std.range;
98     info("testing zlib");
99     immutable(ubyte)[] hello =
100     [
101         0x1f, 0x8b, 0x08, 0x00, 0x94, 0x82, 0xc1, 0x5e, 0x00, 0x03, 0xcb, 0x48, 0xcd, 0xc9, 0xc9, 0xe7,
102         0x02, 0x00, 0x20, 0x30, 0x3a, 0x36, 0x06, 0x00, 0x00, 0x00
103     ];
104     NbuffChunk b = NbuffChunk(hello);
105     ZLib zlib;
106     zlib.zInit(4);
107     int processed;
108     while(processed<b.length)
109     {
110         auto r = zlib.zInflate(b[processed..$]);
111         processed += r.consumed;
112     }
113     zlib.zFlush();
114 
115     immutable(ubyte)[] compressed =
116     [
117         0x1F,0x8B,0x08,0x00, 0xF0,0xFD,0xC2,0x5E, 0x00,0x03,0x4B,0x4C, 0x4A,0x4E,0x49,0x4D,
118         0x4B,0xCF,0xC8,0xCC, 0xCA,0xCE,0xC9,0xCD, 0xCB,0x2F,0x28,0x2C, 0x2A,0x2E,0x29,0x2D,
119         0x2B,0xAF,0xA8,0xAC, 0xE2,0x72,0x74,0x72, 0x76,0x71,0x75,0x73, 0xF7,0xF0,0xF4,0xF2,
120         0xF6,0xF1,0xF5,0xF3, 0x0F,0x08,0x0C,0x0A, 0x0E,0x09,0x0D,0x0B, 0x8F,0x88,0x8C,0xE2,
121         0x4A,0xA4,0xA3,0x2E, 0x00,0x94,0xA7,0x42, 0x7C,0xA2,0x00,0x00, 0x00,
122         0xff,0xff,0xff // this is 3 garbage bytes
123     ];
124 
125 
126 
127     for(int s=1; s<=1024; s++)
128     for(int c=1; c<=64;   c++)
129     {
130         Nbuff uncompressed;
131         zlib.zInit(s);
132         foreach(chunk; chunks(compressed, c))
133         {
134             processed = 0;
135             b = NbuffChunk(chunk);
136             while(processed<b.length)
137             {
138                 auto r = zlib.zInflate(b[processed..$]);
139                 processed += r.consumed;
140                 if (r.result.length)
141                 {
142                     uncompressed.append(r.result);
143                 }
144                 if (r.status == Z_STREAM_END)
145                 {
146                     break;
147                 }
148             }
149         }
150         zlib.zFlush();
151         assert(uncompressed.data.data ==
152             "abcdefghijklmnopqrstuvwxyz\nABCDEFGHIJKLMNOPQRSTUVWXYZ\nabcdefghijklmnopqrstuvwxyz\n" ~
153             "ABCDEFGHIJKLMNOPQRSTUVWXYZ\nabcdefghijklmnopqrstuvwxyz\nABCDEFGHIJKLMNOPQRSTUVWXYZ\n");
154         //writefln("ok for %d/%d", s, c);
155     }
156     info("ok");
157 }