Closes #51 and #57: large inputs and test suite

README.md

@@ -33,6 +33,28 @@ $ go run main.go
 
 Because Dice speaks Redis' dialect, you can connect to it with any Redis Client and the simplest way it to use a [Redis CLI](https://redis.io/docs/manual/cli/). Programmatically, depending on the language you prefer, you can use your favourite Redis library to connect.
 
+## Running Tests
+
+To run all the unit tests fire the following command
+
+```sh
+$ go test github.com/dicedb/dice/tests
+```
+
+### Running a single test
+
+```sh
+$ go test -timeout 30s -run <pattern> github.com/dicedb/dice/tests
+$ go test -timeout 30s -run ^TestHugeResponse$ github.com/dicedb/dice/tests
+```
+
+## Running Benchmark
+
+```sh
+$ go test -test.bench <pattern>
+$ go test -test.bench BenchmarkListRedis
+```
+
 ## Getting Started
 
 To get started with building and contributing to DiceDB, please refer to the [issues](https://github.com/DiceDB/dice/issues) created in this repository.

config/main.go

@@ -10,3 +10,6 @@ var EvictionRatio float64 = 0.40
 
 var EvictionStrategy string = "allkeys-lru"
 var AOFFile string = "./dice-master.aof"
+
+// Network
+var IOBufferLength int = 512

core/eval.go

@@ -295,6 +295,8 @@ func executeCommand(cmd *RedisCmd, c *Client) []byte {
 		}
 		c.TxnDiscard()
 		return RESP_OK
+	case "ABORT":
+		return RESP_OK
 	default:
 		return evalPING(cmd.Args)
 	}

core/io.go

@@ -0,0 +1,98 @@
+package core
+
+import (
+	"bytes"
+	"errors"
+	"io"
+
+	"github.com/dicedb/dice/config"
+)
+
+type RESPParser struct {
+	c   io.ReadWriter
+	buf *bytes.Buffer
+}
+
+func NewRESPParser(c io.ReadWriter) *RESPParser {
+	return NewRESPParserWithBytes(c, []byte{})
+}
+
+func NewRESPParserWithBytes(c io.ReadWriter, initBytes []byte) *RESPParser {
+	var b []byte
+	var buf *bytes.Buffer = bytes.NewBuffer(b)
+	buf.Write(initBytes)
+	return &RESPParser{
+		c:   c,
+		buf: buf,
+	}
+}
+
+func (rp *RESPParser) DecodeOne() (interface{}, error) {
+	// assigning temporary buffer to read 512 bytes in one shot
+	// and reading them in a loop until we have all the data
+	// we want.
+	// note: the size 512 is arbitrarily chosen, and we can put
+	// a decent thought into deciding the optimal value (in case it affects the perf)
+	var buf []byte = make([]byte, config.IOBufferLength)
+
+	// keep reading the bytes from the buffer until the first /r is found
+	// note: there may be extra bytes read over the socket post /r
+	// but that is fine.
+	for {
+		n, err := rp.c.Read(buf)
+		if n <= 0 {
+			break
+		}
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		rp.buf.Write(buf[:n])
+
+		if bytes.Contains(buf, []byte{'\r', '\n'}) {
+			break
+		}
+	}
+
+	b, err := rp.buf.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+
+	switch b {
+	case '+':
+		return readSimpleString(rp.c, rp.buf)
+	case '-':
+		return readError(rp.c, rp.buf)
+	case ':':
+		return readInt64(rp.c, rp.buf)
+	case '$':
+		return readBulkString(rp.c, rp.buf)
+	case '*':
+		return readArray(rp.c, rp.buf, rp)
+	}
+
+	// this also captures the Cross Protocol Scripting attack.
+	// Since we do not support simple strings, anything that does
+	// not start with any of the above special chars will be a potential
+	// attack.
+	// Details: https://bou.ke/blog/hacking-developers/
+	return nil, errors.New("possible cross protocol scripting attack detected")
+}
+
+func (rp *RESPParser) DecodeMultiple() ([]interface{}, error) {
+	var values []interface{} = make([]interface{}, 0)
+	for {
+		value, err := rp.DecodeOne()
+		if err != nil {
+			return nil, err
+		}
+		values = append(values, value)
+		if rp.buf.Len() == 0 {
+			break
+		}
+	}
+	return values, nil
+}

core/resp.go

@@ -2,134 +2,124 @@ package core
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
+	"io"
+	"strconv"
 )
 
-// reads the length typically the first integer of the string
-// until hit by an non-digit byte and returns
-// the integer and the delta = length + 2 (CRLF)
-// TODO: Make it simpler and read until we get `\r` just like other functions
-func readLength(data []byte) (int, int) {
-	pos, length := 0, 0
-	for pos = range data {
-		b := data[pos]
-		if !(b >= '0' && b <= '9') {
-			return length, pos + 2
-		}
-		length = length*10 + int(b-'0')
+func readLength(buf *bytes.Buffer) (int64, error) {
+	s, err := readStringUntilSr(buf)
+	if err != nil {
+		return 0, err
 	}
-	return 0, 0
-}
 
-// reads a RESP encoded simple string from data and returns
-// the string, the delta, and the error
-func readSimpleString(data []byte) (string, int, error) {
-	// first character +
-	pos := 1
+	v, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, err
+	}
 
-	for ; data[pos] != '\r'; pos++ {
+	return v, nil
+}
+
+func readStringUntilSr(buf *bytes.Buffer) (string, error) {
+	s, err := buf.ReadString('\r')
+	if err != nil {
+		return "", err
 	}
+	// increamenting to skip `\n`
+	buf.ReadByte()
+	return s[:len(s)-1], nil
+}
 
-	return string(data[1:pos]), pos + 2, nil
+// reads a RESP encoded simple string from data and returns
+// the string and the error
+// the function internally manipulates the buffer pointer
+// and keepts it at a point where the subsequent value can be read from.
+func readSimpleString(c io.ReadWriter, buf *bytes.Buffer) (string, error) {
+	return readStringUntilSr(buf)
 }
 
 // reads a RESP encoded error from data and returns
-// the error string, the delta, and the error
-func readError(data []byte) (string, int, error) {
-	return readSimpleString(data)
+// the error string and the error
+// the function internally manipulates the buffer pointer
+// and keepts it at a point where the subsequent value can be read from.
+func readError(c io.ReadWriter, buf *bytes.Buffer) (string, error) {
+	return readStringUntilSr(buf)
 }
 
 // reads a RESP encoded integer from data and returns
-// the intger value, the delta, and the error
-func readInt64(data []byte) (int64, int, error) {
-	// first character :
-	pos := 1
-	var value int64 = 0
-
-	for ; data[pos] != '\r'; pos++ {
-		value = value*10 + int64(data[pos]-'0')
+// the intger value and the error
+// the function internally manipulates the buffer pointer
+// and keepts it at a point where the subsequent value can be read from.
+func readInt64(c io.ReadWriter, buf *bytes.Buffer) (int64, error) {
+	s, err := readStringUntilSr(buf)
+	if err != nil {
+		return 0, err
+	}
+
+	v, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, err
 	}
 
-	return value, pos + 2, nil
+	return v, nil
 }
 
 // reads a RESP encoded string from data and returns
-// the string, the delta, and the error
-func readBulkString(data []byte) (string, int, error) {
-	// first character $
-	pos := 1
+// the string, and the error
+// the function internally manipulates the buffer pointer
+// and keepts it at a point where the subsequent value can be read from.
+func readBulkString(c io.ReadWriter, buf *bytes.Buffer) (string, error) {
+	len, err := readLength(buf)
+	if err != nil {
+		return "", err
+	}
 
-	// reading the length and forwarding the pos by
-	// the lenth of the integer + the first special character
-	len, delta := readLength(data[pos:])
-	pos += delta
+	var bytesRem int64 = len + 2 // 2 for \r\n
+	bytesRem = bytesRem - int64(buf.Len())
+	for bytesRem > 0 {
+		tbuf := make([]byte, bytesRem)
+		n, err := c.Read(tbuf)
+		if err != nil {
+			return "", nil
+		}
+		buf.Write(tbuf[:n])
+		bytesRem = bytesRem - int64(n)
+	}
+
+	bulkStr := make([]byte, len)
+	_, err = buf.Read(bulkStr)
+	if err != nil {
+		return "", err
+	}
+
+	// moving buffer pointer by 2 for \r and \n
+	buf.ReadByte()
+	buf.ReadByte()
 
 	// reading `len` bytes as string
-	return string(data[pos:(pos + len)]), pos + len + 2, nil
+	return string(bulkStr), nil
 }
 
 // reads a RESP encoded array from data and returns
 // the array, the delta, and the error
-func readArray(data []byte) (interface{}, int, error) {
-	// first character *
-	pos := 1
-
-	// reading the length
-	count, delta := readLength(data[pos:])
-	pos += delta
+// the function internally manipulates the buffer pointer
+// and keepts it at a point where the subsequent value can be read from.
+func readArray(c io.ReadWriter, buf *bytes.Buffer, rp *RESPParser) (interface{}, error) {
+	count, err := readLength(buf)
+	if err != nil {
+		return nil, err
+	}
 
 	var elems []interface{} = make([]interface{}, count)
 	for i := range elems {
-		elem, delta, err := DecodeOne(data[pos:])
+		elem, err := rp.DecodeOne()
 		if err != nil {
-			return nil, 0, err
+			return nil, err
 		}
 		elems[i] = elem
-		pos += delta
-	}
-	return elems, pos, nil
-}
-
-func DecodeOne(data []byte) (interface{}, int, error) {
-	if len(data) == 0 {
-		return nil, 0, errors.New("no data")
-	}
-	switch data[0] {
-	case '+':
-		return readSimpleString(data)
-	case '-':
-		return readError(data)
-	case ':':
-		return readInt64(data)
-	case '$':
-		return readBulkString(data)
-	case '*':
-		return readArray(data)
 	}
-	return nil, 0, nil
-}
-
-func Decode(data []byte) ([]interface{}, bool, error) {
-	if len(data) == 0 {
-		return nil, false, errors.New("no data")
-	}
-	var values []interface{} = make([]interface{}, 0)
-	var index int = 0
-	var maliciousFlag bool
-	for index < len(data) {
-		value, delta, err := DecodeOne(data[index:])
-		if err != nil {
-			return values, false, err
-		}
-		if delta == 0 {
-			maliciousFlag = true
-			break
-		}
-		index = index + delta
-		values = append(values, value)
-	}
-	return values, maliciousFlag, nil
+	return elems, nil
 }
 
 func encodeString(v string) []byte {
@@ -152,6 +142,13 @@ func Encode(value interface{}, isSimple bool) []byte {
 			buf.Write(encodeString(b))
 		}
 		return []byte(fmt.Sprintf("*%d\r\n%s", len(v), buf.Bytes()))
+	case []interface{}:
+		var b []byte
+		buf := bytes.NewBuffer(b)
+		for _, b := range value.([]string) {
+			buf.Write(Encode(b, false))
+		}
+		return []byte(fmt.Sprintf("*%d\r\n%s", len(v), buf.Bytes()))
 	case error:
 		return []byte(fmt.Sprintf("-%s\r\n", v))
 	default: