1 /// 2 module hio.redisd.codec; 3 4 import std.string; 5 import std.algorithm; 6 import std.stdio; 7 import std.conv; 8 import std.exception; 9 import std.format; 10 import std.typecons; 11 import std.traits; 12 import std.array; 13 import std.range; 14 15 import std.experimental.logger; 16 17 import nbuff; 18 19 /// Type of redis value 20 enum ValueType : ubyte { 21 Incomplete = 0, 22 Null, 23 Integer = ':', 24 String = '+', 25 BulkString = '$', 26 List = '*', 27 Error = '-', 28 } 29 /// 30 class BadDataFormat : Exception { 31 this(string msg, string f = __FILE__, ulong l = __LINE__) @safe { 32 super(msg, f, l); 33 } 34 } 35 /// 36 class EncodeException : Exception { 37 this(string msg, string f = __FILE__, ulong l = __LINE__) @safe { 38 super(msg, f, l); 39 } 40 } 41 /// 42 class WrongOffset : Exception { 43 this(string msg, string f = __FILE__, ulong l = __LINE__) @safe { 44 super(msg, f, l); 45 } 46 } 47 /// 48 class WrongDataAccess : Exception { 49 this(string msg, string f = __FILE__, ulong l = __LINE__) @safe { 50 super(msg, f, l); 51 } 52 } 53 54 struct RedisdValue { 55 private { 56 ValueType _type; 57 string _svar; 58 long _ivar; 59 RedisdValue[] _list; 60 } 61 /// get type of value 62 ValueType type() @safe const { 63 return _type; 64 } 65 /// if this is nil value 66 bool empty() const { 67 return _type == ValueType.Null; 68 } 69 /// get string value 70 string svar() @safe const { 71 switch (_type) { 72 case ValueType.String, ValueType.BulkString, ValueType.Error: 73 return _svar; 74 default: 75 throw new WrongDataAccess("You can't use svar for non-string value"); 76 } 77 } 78 /// get integer value 79 auto ivar() @safe const { 80 switch (_type) { 81 case ValueType.Integer: 82 return _ivar; 83 default: 84 throw new WrongDataAccess("You can't use ivar for non-integer value"); 85 } 86 } 87 88 void opAssign(long v) @safe { 89 _type = ValueType.Integer; 90 _ivar = v; 91 _svar = string.init; 92 _list = RedisdValue[].init; 93 } 94 95 void opAssign(string v) @safe { 96 _type = ValueType.String; 97 _svar = v; 98 _ivar = long.init; 99 _list = RedisdValue[].init; 100 } 101 102 string toString() { 103 switch(_type) { 104 case ValueType.Incomplete: 105 return "<Incomplete>"; 106 case ValueType.String: 107 return "\"%s\"".format(_svar); 108 case ValueType.Error: 109 return "<%s>".format(_svar); 110 case ValueType.Integer: 111 return "%d".format(_ivar); 112 case ValueType.BulkString: 113 return "'%s'".format(_svar); 114 case ValueType.List: 115 return "[%(%s,%)]".format(_list); 116 case ValueType.Null: 117 return "(nil)"; 118 default: 119 return "unknown type " ~ to!string(_type); 120 } 121 } 122 } 123 124 /// Build redis value from string or integer 125 RedisdValue redisdValue(T)(T v) 126 if (isSomeString!T || isIntegral!T) { 127 RedisdValue _v; 128 static if (isIntegral!T) { 129 _v._type = ValueType.Integer; 130 _v._ivar = v; 131 } else 132 static if (isSomeString!T) { 133 _v._type = ValueType.BulkString; 134 _v._svar = v; 135 } else { 136 static assert(0, T.stringof); 137 } 138 return _v; 139 } 140 141 /// Build redis value from tuple 142 RedisdValue redisdValue(T)(T v) if (isTuple!T) { 143 RedisdValue _v; 144 _v._type = ValueType.List; 145 RedisdValue[] l; 146 foreach (element; v) { 147 l ~= redisdValue(element); 148 } 149 _v._list = l; 150 return _v; 151 } 152 153 /// Build redis value from array 154 RedisdValue redisdValue(T:U[], U)(T v) 155 if (isArray!T && !isSomeString!T) { 156 RedisdValue _v; 157 _v._type = ValueType.List; 158 RedisdValue[] l; 159 foreach (element; v) { 160 l ~= redisdValue(element); 161 } 162 _v._list = l; 163 return _v; 164 } 165 /// serialize RedisdValue to byte array 166 immutable(ubyte)[] encode(RedisdValue v) @safe { 167 string encoded; 168 switch(v._type) { 169 case ValueType.Error: 170 encoded.reserve(1 + v._svar.length + 2); 171 encoded ~= "-"; 172 encoded ~= v._svar; 173 encoded ~= "\r\n"; 174 return encoded.representation; 175 case ValueType.String: 176 encoded.reserve(1+v._svar.length+2); 177 encoded ~= "+"; 178 encoded ~= v._svar; 179 encoded ~= "\r\n"; 180 return encoded.representation; 181 case ValueType.Integer: 182 encoded.reserve(15); 183 encoded ~= ":"; 184 encoded ~= to!string(v._ivar); 185 encoded ~= "\r\n"; 186 return encoded.representation; 187 case ValueType.BulkString: 188 encoded.reserve(v._svar.length+1+20+2+2); 189 encoded ~= "$"; 190 encoded ~= to!string(v._svar.length); 191 encoded ~= "\r\n"; 192 encoded ~= to!string(v._svar); 193 encoded ~= "\r\n"; 194 return encoded.representation; 195 case ValueType.List: 196 Appender!(immutable(ubyte)[]) appender; 197 appender.put(("*" ~ to!string(v._list.length) ~ "\r\n").representation); 198 foreach(element; v._list) { 199 appender.put(element.encode); 200 } 201 return cast(immutable(ubyte)[])appender.data; 202 default: 203 throw new EncodeException("Failed to encode"); 204 } 205 } 206 207 /// 208 @safe unittest { 209 RedisdValue v; 210 v = "abc"; 211 assert(encode(v) == "+abc\r\n".representation); 212 v = -1234567890; 213 assert(encode(v) == ":-1234567890\r\n".representation); 214 v = -0; 215 assert(encode(v) == ":0\r\n".representation); 216 217 v = -1234567890; 218 assert(v.encode.decode.value == v); 219 v = redisdValue("abc"); 220 assert(v.encode.decode.value == v); 221 v = redisdValue([1,2]); 222 assert(v.encode.decode.value == v); 223 v = redisdValue(tuple("abc", 1)); 224 assert(v.encode.decode.value == v); 225 } 226 227 alias DecodeResult = Tuple!(RedisdValue, "value", immutable(ubyte)[], "rest"); 228 229 /// deserialize from byte array 230 DecodeResult decode(immutable(ubyte)[] data) @safe { 231 assert(data.length >= 1); 232 RedisdValue v; 233 switch(data[0]) { 234 case '+': 235 // simple string 236 v._type = ValueType.String; 237 auto s = data[1..$].findSplit([13,10]); 238 v._svar = cast(string)s[0]; 239 return DecodeResult(v, s[2]); 240 case '-': 241 // error 242 v._type = ValueType.Error; 243 auto s = data[1 .. $].findSplit([13, 10]); 244 v._svar = cast(string) s[0]; 245 return DecodeResult(v, s[2]); 246 case ':': 247 // integer 248 v._type = ValueType.Integer; 249 auto s = data[1 .. $].findSplit([13, 10]); 250 v._ivar = to!long(cast(string)s[0]); 251 return DecodeResult(v, s[2]); 252 case '$': 253 // bulk string 254 // skip $, then try to split on first \r\n: 255 // s now: [length],[\r\n],[data\r\n] 256 v._type = ValueType.BulkString; 257 auto s = data[1..$].findSplit([13, 10]); 258 if (s[1].length != 2) { 259 throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data)); 260 } 261 auto len = to!long(cast(string)s[0]); 262 if ( s[2].length < len + 2 ) { 263 throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data)); 264 } 265 v._svar = cast(string)s[2][0..len]; 266 return DecodeResult(v, s[2][len+2..$]); 267 case '*': 268 // list 269 v._type = ValueType.List; 270 auto s = data[1 .. $].findSplit([13, 10]); 271 if (s[1].length != 2) { 272 throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data)); 273 } 274 auto len = to!long(cast(string) s[0]); 275 if ( len == 0 ) { 276 return DecodeResult(v, s[2]); 277 } 278 if ( len == -1 ){ 279 v._type = ValueType.Null; 280 return DecodeResult(v, s[2]); 281 } 282 auto rest = s[2]; 283 RedisdValue[] array; 284 285 while(len>0) { 286 auto d = decode(rest); 287 auto v0 = d.value; 288 array ~= v0; 289 rest = d.rest; 290 len--; 291 } 292 v._list = array; 293 return DecodeResult(v, rest); 294 default: 295 break; 296 } 297 assert(0); 298 } 299 300 @safe unittest { 301 RedisdValue v; 302 v = "a"; 303 v = 1; 304 } 305 /// 306 @safe unittest { 307 DecodeResult d; 308 d = decode("+OK\r\n ".representation); 309 auto v = d.value; 310 auto r = d.rest; 311 312 assert(v._type == ValueType.String); 313 assert(v._svar == "OK"); 314 assert(r == " ".representation); 315 316 d = decode("-ERROR\r\ngarbage\r\n".representation); 317 v = d.value; 318 r = d.rest; 319 assert(v._type == ValueType.Error); 320 assert(v._svar == "ERROR"); 321 assert(r == "garbage\r\n".representation); 322 323 d = decode(":100\r\n".representation); 324 v = d.value; 325 r = d.rest; 326 assert(v._type == ValueType.Integer); 327 assert(v._ivar == 100); 328 assert(r == "".representation); 329 330 d = decode("$8\r\nfoobar\r\n\r\n:41\r\n".representation); 331 v = d.value; 332 r = d.rest; 333 assert(v._svar == "foobar\r\n", format("<%s>", v._svar)); 334 assert(r == ":41\r\n".representation); 335 assert(v._type == ValueType.BulkString); 336 337 d = decode("*3\r\n:1\r\n:2\r\n$6\r\nfoobar\r\nxyz".representation); 338 v = d.value; 339 r = d.rest; 340 assert(v._type == ValueType.List); 341 assert(r == "xyz".representation); 342 } 343 344 /// decode redis values from stream of ubyte chunks 345 class Decoder { 346 enum State { 347 Init, 348 Type, 349 } 350 private { 351 Nbuff _chunks; 352 State _state; 353 ValueType _frontChunkType; 354 size_t _parsedPosition; 355 size_t _list_len; 356 RedisdValue[] _list; 357 } 358 /// put next data chunk to decoder 359 bool put(NbuffChunk chunk) @safe { 360 assert(chunk.length > 0, "Chunk must not be emplty"); 361 if ( _chunks.length == 0 ) { 362 switch( chunk[0]) { 363 case '+','-',':','*','$': 364 _frontChunkType = cast(ValueType)chunk[0]; 365 debug(redisd) tracef("new chunk type %s", _frontChunkType); 366 break; 367 default: 368 throw new BadDataFormat("on chunk" ~ to!string(chunk)); 369 } 370 } 371 _chunks.append(chunk); 372 //_len += chunk.length; 373 return false; 374 } 375 376 private RedisdValue _handleListElement(RedisdValue v) @safe { 377 // we processing list element 378 () @trusted {debug(redisd) tracef("appending %s to list", v);} (); 379 _list ~= v; 380 _list_len--; 381 if (_list_len == 0) { 382 RedisdValue result = {_type:ValueType.List, _list : _list}; 383 _list.length = 0; 384 return result; 385 } 386 return RedisdValue(); 387 } 388 /// try to fetch decoded value from decoder. 389 /// Return next value or value of type $(B Incomplete). 390 RedisdValue get() @safe { 391 RedisdValue v; 392 // debug(redisd)writefln("on entry:\n◇\n%s\n◆", _chunks.dump()); 393 start: 394 if (_chunks.length == 0 ) { 395 return v; 396 } 397 // Flat f = Flat(_chunks); 398 // Flat[] s; 399 debug(redisd) tracef("check var type '%c'", cast(char)_chunks[0]); 400 switch (_chunks[0]) { 401 case ':': 402 auto s = _chunks.findSplitOn(['\r', '\n']); 403 //debug(redisd) tracef("check var type %s", s[0].data.asString!(s=>s)); 404 if ( s[1].length ) { 405 v = to!long(s[0].data[1..$].toString); 406 _chunks = s[2]; 407 // debug(redisd)writefln("after:\n%s\n", _chunks.dump()); 408 if (_list_len > 0) { 409 v = _handleListElement(v); 410 if (_list_len) 411 goto start; 412 } 413 debug(redisd) tracef("value: %s", v); 414 return v; 415 } 416 break; 417 case '+': 418 auto s = _chunks.findSplitOn(['\r', '\n']); 419 if ( s[1].length ) { 420 v = s[0].data[1 .. $].toString; 421 _chunks = s[2]; 422 if (_list_len > 0) { 423 v = _handleListElement(v); 424 if (_list_len) 425 goto start; 426 } 427 return v; 428 } 429 break; 430 case '-': 431 auto s = _chunks.findSplitOn(['\r', '\n']); 432 if ( s[1].length ) { 433 v._type = ValueType.Error; 434 v._svar = s[0].data[1 .. $].toString; 435 _chunks = s[2]; 436 if (_list_len > 0) { 437 v = _handleListElement(v); 438 if (_list_len) 439 goto start; 440 } 441 return v; 442 } 443 break; 444 case '$': 445 auto s = _chunks.findSplitOn(['\r', '\n']); 446 if ( s[1].length ) { 447 long len = (s[0].data[1..$].toString.to!long); 448 if ( len == -1 ) { 449 v._type = ValueType.Null; 450 _chunks = s[2]; 451 if (_list_len > 0) { 452 v = _handleListElement(v); 453 if (_list_len) 454 goto start; 455 } 456 return v; 457 } 458 if (s[2].length < len + 2) { 459 return v; 460 } 461 string data = s[2][0..len].toString; 462 v = data; 463 _chunks = s[2][data.length+2..s[2].length]; 464 if (_list_len > 0) { 465 v = _handleListElement(v); 466 if (_list_len) goto start; 467 } 468 } 469 break; 470 case '*': 471 _list_len = -1; 472 auto s = _chunks.findSplitOn(['\r', '\n']); 473 if ( s[1].length ) { 474 long len = s[0].data[1 .. $].toString.to!long; 475 _list_len = len; 476 _chunks = s[2]; 477 if (len == 0) { 478 v._type = ValueType.Null; 479 return v; 480 } 481 debug (redisd) 482 tracef("list of length %d detected", _list_len); 483 goto start; 484 } 485 break; 486 default: 487 break; 488 } 489 return v; 490 } 491 } 492 /// 493 @safe unittest { 494 globalLogLevel = LogLevel.trace; 495 RedisdValue str = {_type:ValueType.String, _svar : "abc"}; 496 RedisdValue err = {_type:ValueType.Error, _svar : "err"}; 497 immutable(ubyte)[] b = redisdValue(1001).encode 498 ~ redisdValue(1002).encode 499 ~ str.encode 500 ~ err.encode 501 ~ redisdValue("\r\nBulkString\r\n").encode 502 ~ redisdValue(1002).encode 503 ~ redisdValue([1,2,3]).encode 504 ~ redisdValue(["abc", "def"]).encode 505 ~ redisdValue(1002).encode; 506 507 foreach(chunkSize; 1..b.length) { 508 auto s = new Decoder(); 509 foreach (c; b.chunks(chunkSize)) { 510 s.put(NbuffChunk(c)); 511 } 512 auto v = s.get(); 513 assert(v._ivar == 1001); 514 515 v = s.get(); 516 assert(v._ivar == 1002); 517 518 v = s.get(); 519 assert(v._svar == "abc"); 520 521 v = s.get(); 522 assert(v._svar == "err"); 523 524 v = s.get(); 525 assert(v._svar == "\r\nBulkString\r\n"); 526 v = s.get(); 527 debug(redisd)trace(v); 528 assert(v._ivar == 1002); 529 int lists_to_get = 2; 530 while( lists_to_get>0 ) { 531 v = s.get(); 532 debug(redisd)trace(v); 533 debug (redisd) 534 () @trusted { tracef("%s: %s, %s", v._type, v, lists_to_get); }(); 535 if (v._type == ValueType.List) { 536 lists_to_get--; 537 } 538 } 539 v = s.get(); 540 debug(redisd)trace(v); 541 assert(v._ivar == 1002); 542 v = s.get(); 543 debug(redisd)trace(v); 544 assert(v._type == ValueType.Incomplete); 545 } 546 info("test ok"); 547 }