Support Tair modules (#404)

* Support tairstring tairzset tairhash module

* Update README

Makefile

@@ -1,4 +1,4 @@
-all: build runtest
+all: build
 
 runtest:
 	./test.sh

README.md

@@ -32,6 +32,13 @@ Supports `Standalone`, `Cluster` and some proxies type like `Codis`, `twemproxy`
 
 For `codis` and `twemproxy`, there maybe some constraints, please checkout this [question](https://github.com/alibaba/RedisShake/wiki/FAQ#q-does-redisshake-supports-codis-and-twemproxy).
 
+
+Support Redis Modules:   
+[TairHash](https://github.com/alibaba/TairHash): A redis module, similar to redis hash, but you can set expire and version for the field  
+[TairZset](https://github.com/alibaba/TairZset): A redis module, similar to redis zset, but you can set multiple scores for each member to support multi-dimensional sorting  
+[TairString](https://github.com/alibaba/TairString): A redis module, similar to redis string, but you can set expire and version for the value. It also provides many very useful commands, such as cas/cad, etc.  
+
+
 # Configuration
 
 Redis-shake has several parameters in the configuration `conf/redis-shake.conf`, that maybe confusing, if this is your first time using, please visit this [tutorial](https://github.com/alibaba/RedisShake/wiki/tutorial-about-how-to-set-up).

src/pkg/rdb/module.go

@@ -0,0 +1,102 @@
+package rdb
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// this function is used to parse rdb module
+
+func (r *rdbReader) parseModule(moduleName string, moduleId uint64, t byte, b *bytes.Buffer) ([]byte, error) {
+	// the module providing the 10 bit encoding version in the lower 10 bits of the module ID.
+	encodeVersion := moduleId & 1023
+	switch moduleName {
+	case "tairhash-":
+		// length
+		length, err := r.moduleLoadUnsigned(t)
+		if err != nil {
+			return nil, err
+		}
+
+		// key
+		if _, err := r.moduleLoadString(t); err != nil {
+			return nil, err
+		}
+
+		for i := uint64(0); i < length; i++ {
+			// skey
+			if _, err := r.moduleLoadString(t); err != nil {
+				return nil, err
+			}
+
+			// version
+			if _, err := r.moduleLoadUnsigned(t); err != nil {
+				return nil, err
+			}
+
+			// expire
+			if _, err := r.moduleLoadUnsigned(t); err != nil {
+				return nil, err
+			}
+
+			// value
+			if _, err := r.moduleLoadString(t); err != nil {
+				return nil, err
+			}
+		}
+	case "exstrtype":
+		// version
+		if _, err := r.moduleLoadUnsigned(t); err != nil {
+			return nil, err
+		}
+
+		if encodeVersion == 1 {
+			//flag
+			if _, err := r.moduleLoadUnsigned(t); err != nil {
+				return nil, err
+			}
+		}
+
+		// value
+		if _, err := r.moduleLoadString(t); err != nil {
+			return nil, err
+		}
+
+	case "tairzset_":
+		length, err := r.moduleLoadUnsigned(t)
+		if err != nil {
+			return nil, err
+		}
+
+		score_num, err := r.moduleLoadUnsigned(t)
+		if err != nil {
+			return nil, err
+		}
+
+		for i := uint64(0); i < length; i++ {
+			if _, err := r.moduleLoadString(t); err != nil {
+				return nil, err
+			}
+
+			for j := uint64(0); j < score_num; j++ {
+				if _, err := r.moduleLoadDouble(t); err != nil {
+					return nil, err
+				}
+			}
+		}
+
+	default:
+		return nil, fmt.Errorf("unknown module name[%v] with modue id[%v]", moduleName, moduleId)
+	}
+
+	if t == RdbTypeModule2 {
+		code, err := r.ReadLength()
+		if err != nil {
+			return nil, err
+		} else if code != rdbModuleOpcodeEof {
+			return nil, fmt.Errorf("illegal end code[%v] in module type", code)
+		}
+	}
+
+	return b.Bytes(), nil
+}

src/pkg/rdb/reader.go

@@ -9,22 +9,26 @@ import (
 	"fmt"
 	"io"
 	"math"
+
 	// "runtime/debug"
 	"strconv"
 
 	"github.com/alibaba/RedisShake/pkg/libs/errors"
+	"github.com/alibaba/RedisShake/pkg/libs/log"
 )
 
 var FromVersion int64 = 9
 var ToVersion int64 = 6
 
 const (
-	RdbTypeString = 0
-	RdbTypeList   = 1
-	RdbTypeSet    = 2
-	RdbTypeZSet   = 3
-	RdbTypeHash   = 4
-	RdbTypeZSet2  = 5
+	RdbTypeString  = 0
+	RdbTypeList    = 1
+	RdbTypeSet     = 2
+	RdbTypeZSet    = 3
+	RdbTypeHash    = 4
+	RdbTypeZSet2   = 5
+	RdbTypeModule  = 6
+	RdbTypeModule2 = 7
 
 	RdbTypeHashZipmap      = 9
 	RdbTypeListZiplist     = 10
@@ -51,6 +55,8 @@ const (
 	rdbModuleOpcodeFloat  = 3
 	rdbModuleOpcodeDouble = 4
 	rdbModuleOpcodeString = 5
+
+	moduleTypeNameCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 )
 
 const (
@@ -298,6 +304,16 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) {
 				}
 			}
 		}
+	case RdbTypeModule2:
+		// skip 64 bit
+		if moduleId, err := r.ReadLength64(); err != nil {
+			return nil, err
+		} else {
+			moduleName := moduleTypeNameByID(moduleId)
+			log.Debugf("handle module id[%v] name[%v]", moduleId, moduleName)
+			lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0
+			return r.parseModule(moduleName, moduleId, t, &b)
+		}
 	}
 	return b.Bytes(), nil
 }
@@ -664,3 +680,54 @@ func (r *rdbReader) CountZipmapItems(buf *sliceBuffer) (int, error) {
 	_, err := buf.Seek(0, 0)
 	return n, err
 }
+
+func moduleTypeNameByID(moduleId uint64) string {
+	nameList := make([]byte, 9)
+	moduleId >>= 10
+	for i := 8; i >= 0; i-- {
+		nameList[i] = moduleTypeNameCharSet[moduleId&63]
+		moduleId >>= 6
+	}
+	return string(nameList)
+}
+
+func (r *rdbReader) moduleLoadUnsigned(rdbType byte) (uint64, error) {
+	if rdbType == RdbTypeModule2 {
+		// ver == 2
+		if opCode, err := r.ReadLength(); err != nil {
+			return 0, err
+		} else if opCode != rdbModuleOpcodeUint {
+			return 0, fmt.Errorf("opcode[%v] != rdbModuleOpcodeUint[%v]", opCode, rdbModuleOpcodeUint)
+		}
+	}
+
+	val, err := r.ReadLength()
+	return uint64(val), err
+}
+
+func (r *rdbReader) moduleLoadDouble(rdbType byte) (float64, error) {
+	if rdbType == RdbTypeModule2 {
+		// ver == 2
+		if opCode, err := r.ReadLength(); err != nil {
+			return 0, err
+		} else if opCode != rdbModuleOpcodeDouble {
+			return 0, fmt.Errorf("opcode[%v] != rdbModuleOpcodeDouble[%v]", opCode,
+				rdbModuleOpcodeDouble)
+		}
+	}
+
+	return r.ReadDouble()
+}
+
+func (r *rdbReader) moduleLoadString(rdbType byte) (string, error) {
+	if rdbType == RdbTypeModule2 {
+		if opCode, err := r.ReadLength(); err != nil {
+			return "", err
+		} else if opCode != rdbModuleOpcodeString {
+			return "", fmt.Errorf("opcode[%v] != rdbModuleOpcodeString[%v]", opCode,
+				rdbModuleOpcodeString)
+		}
+	}
+	ret, err := r.ReadString()
+	return string(ret), err
+}