1 ///
2 module hio.redisd.codec;
3 
4 import std.string;
5 import std.algorithm;
6 import std.stdio;
7 import std.conv;
8 import std.exception;
9 import std.format;
10 import std.typecons;
11 import std.traits;
12 import std.array;
13 import std.range;
14 
15 import std.experimental.logger;
16 
17 import nbuff;
18 
19 /// Type of redis value
20 enum ValueType : ubyte {
21     Incomplete = 0,
22     Null,
23     Integer = ':',
24     String = '+',
25     BulkString = '$',
26     List = '*',
27     Error = '-',
28 }
29 ///
30 class BadDataFormat : Exception {
31     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
32         super(msg, f, l);
33     }
34 }
35 ///
36 class EncodeException : Exception {
37     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
38         super(msg, f, l);
39     }
40 }
41 ///
42 class WrongOffset : Exception {
43     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
44         super(msg, f, l);
45     }
46 }
47 ///
48 class WrongDataAccess : Exception {
49     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
50         super(msg, f, l);
51     }
52 }
53 
54 struct RedisdValue {
55     private {
56         ValueType       _type;
57         string          _svar;
58         long            _ivar;
59         RedisdValue[]    _list;
60     }
61     /// get type of value
62     ValueType type() @safe const {
63         return _type;
64     }
65     /// if this is nil value
66     bool empty() const {
67         return _type == ValueType.Null;
68     }
69     /// get string value
70     string svar() @safe const {
71         switch (_type) {
72         case ValueType.String, ValueType.BulkString, ValueType.Error:
73             return _svar;
74         default:
75             throw new WrongDataAccess("You can't use svar for non-string value");
76         }
77     }
78     /// get integer value
79     auto ivar() @safe const {
80         switch (_type) {
81         case ValueType.Integer:
82             return _ivar;
83         default:
84             throw new WrongDataAccess("You can't use ivar for non-integer value");
85         }
86     }
87 
88     void opAssign(long v) @safe {
89         _type = ValueType.Integer;
90         _ivar = v;
91         _svar = string.init;
92         _list = RedisdValue[].init;
93     }
94 
95     void opAssign(string v) @safe {
96         _type = ValueType.String;
97         _svar = v;
98         _ivar = long.init;
99         _list = RedisdValue[].init;
100     }
101 
102     string toString() {
103         switch(_type) {
104         case ValueType.Incomplete:
105             return "<Incomplete>";
106         case ValueType.String:
107             return "\"%s\"".format(_svar);
108         case ValueType.Error:
109             return "<%s>".format(_svar);
110         case ValueType.Integer:
111             return "%d".format(_ivar);
112         case ValueType.BulkString:
113             return "'%s'".format(_svar);
114         case ValueType.List:
115             return "[%(%s,%)]".format(_list);
116         case ValueType.Null:
117             return "(nil)";
118         default:
119             return "unknown type " ~ to!string(_type);
120         }
121     }
122 }
123 
124 /// Build redis value from string or integer
125 RedisdValue redisdValue(T)(T v)
126 if (isSomeString!T || isIntegral!T) {
127     RedisdValue _v;
128     static if (isIntegral!T) {
129         _v._type = ValueType.Integer;
130         _v._ivar = v;
131     } else
132     static if (isSomeString!T) {
133         _v._type = ValueType.BulkString;
134         _v._svar = v;
135     } else {
136         static assert(0, T.stringof);
137     }
138     return _v;
139 }
140 
141 /// Build redis value from tuple
142 RedisdValue redisdValue(T)(T v) if (isTuple!T) {
143     RedisdValue _v;
144     _v._type = ValueType.List;
145     RedisdValue[] l;
146     foreach (element; v) {
147         l ~= redisdValue(element);
148     }
149     _v._list = l;
150     return _v;
151 }
152 
153 /// Build redis value from array
154 RedisdValue redisdValue(T:U[], U)(T v)
155 if (isArray!T && !isSomeString!T) {
156     RedisdValue _v;
157     _v._type = ValueType.List;
158     RedisdValue[] l;
159     foreach (element; v) {
160         l ~= redisdValue(element);
161     }
162     _v._list = l;
163     return _v;
164 }
165 /// serialize RedisdValue to byte array
166 immutable(ubyte)[] encode(RedisdValue v) @safe {
167     string encoded;
168     switch(v._type) {
169     case ValueType.Error:
170         encoded.reserve(1 + v._svar.length + 2);
171         encoded ~= "-";
172         encoded ~= v._svar;
173         encoded ~= "\r\n";
174         return encoded.representation;
175     case ValueType.String:
176         encoded.reserve(1+v._svar.length+2);
177         encoded ~= "+";
178         encoded ~= v._svar;
179         encoded ~= "\r\n";
180         return encoded.representation;
181     case ValueType.Integer:
182         encoded.reserve(15);
183         encoded ~= ":";
184         encoded ~= to!string(v._ivar);
185         encoded ~= "\r\n";
186         return encoded.representation;
187     case ValueType.BulkString:
188         encoded.reserve(v._svar.length+1+20+2+2);
189         encoded ~= "$";
190         encoded ~= to!string(v._svar.length);
191         encoded ~= "\r\n";
192         encoded ~= to!string(v._svar);
193         encoded ~= "\r\n";
194         return encoded.representation;
195     case ValueType.List:
196         Appender!(immutable(ubyte)[]) appender;
197         appender.put(("*" ~ to!string(v._list.length) ~ "\r\n").representation);
198         foreach(element; v._list) {
199             appender.put(element.encode);
200         }
201         return cast(immutable(ubyte)[])appender.data;
202     default:
203         throw new EncodeException("Failed to encode");
204     }
205 }
206 
207 ///
208 @safe unittest {
209     RedisdValue v;
210     v = "abc";
211     assert(encode(v) == "+abc\r\n".representation);
212     v = -1234567890;
213     assert(encode(v) == ":-1234567890\r\n".representation);
214     v = -0;
215     assert(encode(v) == ":0\r\n".representation);
216 
217     v = -1234567890;
218     assert(v.encode.decode.value == v);
219     v = redisdValue("abc");
220     assert(v.encode.decode.value == v);
221     v = redisdValue([1,2]);
222     assert(v.encode.decode.value == v);
223     v = redisdValue(tuple("abc", 1));
224     assert(v.encode.decode.value == v);
225 }
226 
227 alias DecodeResult = Tuple!(RedisdValue, "value", immutable(ubyte)[], "rest");
228 
229 /// deserialize from byte array
230 DecodeResult decode(immutable(ubyte)[] data) @safe {
231     assert(data.length >= 1);
232     RedisdValue v;
233     switch(data[0]) {
234     case '+':
235         // simple string
236         v._type = ValueType.String;
237         auto s = data[1..$].findSplit([13,10]);
238         v._svar = cast(string)s[0];
239         return DecodeResult(v, s[2]);
240     case '-':
241         // error
242         v._type = ValueType.Error;
243         auto s = data[1 .. $].findSplit([13, 10]);
244         v._svar = cast(string) s[0];
245         return DecodeResult(v, s[2]);
246     case ':':
247         // integer
248         v._type = ValueType.Integer;
249         auto s = data[1 .. $].findSplit([13, 10]);
250         v._ivar = to!long(cast(string)s[0]);
251         return DecodeResult(v, s[2]);
252     case '$':
253         // bulk string
254         // skip $, then try to split on first \r\n:
255         // s now: [length],[\r\n],[data\r\n]
256         v._type = ValueType.BulkString;
257         auto s = data[1..$].findSplit([13, 10]);
258         if (s[1].length != 2) {
259             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
260         }
261         auto len = to!long(cast(string)s[0]);
262         if ( s[2].length < len + 2 ) {
263             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
264         }
265         v._svar = cast(string)s[2][0..len];
266         return DecodeResult(v, s[2][len+2..$]);
267     case '*':
268         // list
269         v._type = ValueType.List;
270         auto s = data[1 .. $].findSplit([13, 10]);
271         if (s[1].length != 2) {
272             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
273         }
274         auto len = to!long(cast(string) s[0]);
275         if ( len == 0 ) {
276             return DecodeResult(v, s[2]);
277         }
278         if ( len == -1 ){
279             v._type = ValueType.Null;
280             return DecodeResult(v, s[2]);
281         }
282         auto rest = s[2];
283         RedisdValue[] array;
284 
285         while(len>0) {
286             auto d = decode(rest);
287             auto v0 = d.value;
288             array ~= v0;
289             rest = d.rest;
290             len--;
291         }
292         v._list = array;
293         return DecodeResult(v, rest);
294     default:
295         break;
296     }
297     assert(0);
298 }
299 
300 @safe unittest {
301     RedisdValue v;
302     v = "a";
303     v = 1;
304 }
305 ///
306 @safe unittest {
307     DecodeResult d;
308     d = decode("+OK\r\n ".representation);
309     auto v = d.value;
310     auto r = d.rest;
311 
312     assert(v._type == ValueType.String);
313     assert(v._svar == "OK");
314     assert(r == " ".representation);
315 
316     d = decode("-ERROR\r\ngarbage\r\n".representation);
317     v = d.value;
318     r = d.rest;
319     assert(v._type == ValueType.Error);
320     assert(v._svar == "ERROR");
321     assert(r == "garbage\r\n".representation);
322 
323     d = decode(":100\r\n".representation);
324     v = d.value;
325     r = d.rest;
326     assert(v._type == ValueType.Integer);
327     assert(v._ivar == 100);
328     assert(r == "".representation);
329 
330     d = decode("$8\r\nfoobar\r\n\r\n:41\r\n".representation);
331     v = d.value;
332     r = d.rest;
333     assert(v._svar == "foobar\r\n", format("<%s>", v._svar));
334     assert(r ==  ":41\r\n".representation);
335     assert(v._type == ValueType.BulkString);
336 
337     d = decode("*3\r\n:1\r\n:2\r\n$6\r\nfoobar\r\nxyz".representation);
338     v = d.value;
339     r = d.rest;
340     assert(v._type == ValueType.List);
341     assert(r == "xyz".representation);
342 }
343 
344 /// decode redis values from stream of ubyte chunks
345 class Decoder {
346     enum State {
347         Init,
348         Type,
349     }
350     private {
351         Nbuff           _chunks;
352         State           _state;
353         ValueType       _frontChunkType;
354         size_t          _parsedPosition;
355         size_t          _list_len;
356         RedisdValue[]   _list;
357     }
358     /// put next data chunk to decoder
359     bool put(NbuffChunk chunk) @safe {
360         assert(chunk.length > 0, "Chunk must not be emplty");
361         if ( _chunks.length == 0 ) {
362             switch( chunk[0]) {
363             case '+','-',':','*','$':
364                 _frontChunkType = cast(ValueType)chunk[0];
365                 debug(redisd) tracef("new chunk type %s", _frontChunkType);
366                 break;
367             default:
368                 throw new BadDataFormat("on chunk" ~ to!string(chunk));
369             }
370         }
371         _chunks.append(chunk);
372         //_len += chunk.length;
373         return false;
374     }
375 
376     private RedisdValue _handleListElement(RedisdValue v) @safe {
377         // we processing list element
378         () @trusted {debug(redisd) tracef("appending %s to list", v);} ();
379         _list ~= v;
380         _list_len--;
381         if (_list_len == 0) {
382             RedisdValue result = {_type:ValueType.List, _list : _list};
383             _list.length = 0;
384             return result;
385         }
386         return RedisdValue();
387     }
388     /// try to fetch decoded value from decoder.
389     /// Return next value or value of type $(B Incomplete).
390     RedisdValue get() @safe {
391         RedisdValue v;
392     // debug(redisd)writefln("on entry:\n◇\n%s\n◆", _chunks.dump());
393     start:
394         if (_chunks.length == 0 ) {
395             return v;
396         }
397         // Flat f = Flat(_chunks);
398         // Flat[] s;
399         debug(redisd) tracef("check var type '%c'", cast(char)_chunks[0]);
400         switch (_chunks[0]) {
401         case ':':
402             auto s = _chunks.findSplitOn(['\r', '\n']);
403             //debug(redisd) tracef("check var type %s", s[0].data.asString!(s=>s));
404             if ( s[1].length ) {
405                 v = to!long(s[0].data[1..$].toString);
406                 _chunks = s[2];
407                 // debug(redisd)writefln("after:\n%s\n", _chunks.dump());
408                 if (_list_len > 0) {
409                     v = _handleListElement(v);
410                     if (_list_len)
411                         goto start;
412                 }
413                 debug(redisd) tracef("value: %s", v);
414                 return v;
415             }
416             break;
417         case '+':
418             auto s = _chunks.findSplitOn(['\r', '\n']);
419             if ( s[1].length ) {
420                 v = s[0].data[1 .. $].toString;
421                 _chunks = s[2];
422                 if (_list_len > 0) {
423                     v = _handleListElement(v);
424                     if (_list_len)
425                         goto start;
426                 }
427                 return v;
428             }
429             break;
430         case '-':
431             auto s = _chunks.findSplitOn(['\r', '\n']);
432             if ( s[1].length ) {
433                 v._type = ValueType.Error;
434                 v._svar = s[0].data[1 .. $].toString;
435                 _chunks = s[2];
436                 if (_list_len > 0) {
437                     v = _handleListElement(v);
438                     if (_list_len)
439                         goto start;
440                 }
441                 return v;
442             }
443             break;
444         case '$':
445             auto s = _chunks.findSplitOn(['\r', '\n']);
446             if ( s[1].length ) {
447                 long len = (s[0].data[1..$].toString.to!long);
448                 if ( len == -1 ) {
449                     v._type = ValueType.Null;
450                     _chunks = s[2];
451                     if (_list_len > 0) {
452                         v = _handleListElement(v);
453                         if (_list_len)
454                             goto start;
455                     }
456                     return v;
457                 }
458                 if (s[2].length < len + 2) {
459                     return v;
460                 }
461                 string data = s[2][0..len].toString;
462                 v = data;
463                 _chunks = s[2][data.length+2..s[2].length];
464                 if (_list_len > 0) {
465                     v = _handleListElement(v);
466                     if (_list_len) goto start;
467                 }
468             }
469             break;
470         case '*':
471             _list_len = -1;
472             auto s = _chunks.findSplitOn(['\r', '\n']);
473             if ( s[1].length ) {
474                 long len = s[0].data[1 .. $].toString.to!long;
475                 _list_len = len;
476                 _chunks = s[2];
477                 if (len == 0) {
478                     v._type = ValueType.Null;
479                     return v;
480                 }
481                 debug (redisd)
482                     tracef("list of length %d detected", _list_len);
483                 goto start;            
484             }
485             break;
486         default:
487             break;
488         }
489         return v;
490     }
491 }
492 ///
493 @safe unittest {
494     globalLogLevel = LogLevel.trace;
495     RedisdValue str = {_type:ValueType.String, _svar : "abc"};
496     RedisdValue err = {_type:ValueType.Error, _svar : "err"};
497     immutable(ubyte)[] b = redisdValue(1001).encode 
498             ~ redisdValue(1002).encode
499             ~ str.encode
500             ~ err.encode
501             ~ redisdValue("\r\nBulkString\r\n").encode
502             ~ redisdValue(1002).encode
503             ~ redisdValue([1,2,3]).encode
504             ~ redisdValue(["abc", "def"]).encode
505             ~ redisdValue(1002).encode;
506 
507     foreach(chunkSize; 1..b.length) {
508         auto s = new Decoder();
509         foreach (c; b.chunks(chunkSize)) {
510             s.put(NbuffChunk(c));
511         }
512         auto v = s.get();
513         assert(v._ivar == 1001);
514 
515         v = s.get();
516         assert(v._ivar == 1002);
517 
518         v = s.get();
519         assert(v._svar == "abc");
520 
521         v = s.get();
522         assert(v._svar == "err");
523 
524         v = s.get();
525         assert(v._svar == "\r\nBulkString\r\n");
526         v = s.get();
527         debug(redisd)trace(v);
528         assert(v._ivar == 1002);
529         int lists_to_get = 2;
530         while( lists_to_get>0 ) {
531             v = s.get();
532             debug(redisd)trace(v);
533             debug (redisd)
534                 () @trusted { tracef("%s: %s, %s", v._type, v, lists_to_get); }();
535             if (v._type == ValueType.List) {
536                 lists_to_get--;
537             }
538         }
539         v = s.get();
540         debug(redisd)trace(v);
541         assert(v._ivar == 1002);
542         v = s.get();
543         debug(redisd)trace(v);
544         assert(v._type == ValueType.Incomplete);
545     }
546     info("test ok");
547 }