diff --git a/Makefile b/Makefile index 22c62e6..e4c8878 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: build runtest +all: build runtest: ./test.sh diff --git a/README.md b/README.md index 4747732..e5018d5 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,13 @@ Supports `Standalone`, `Cluster` and some proxies type like `Codis`, `twemproxy` For `codis` and `twemproxy`, there maybe some constraints, please checkout this [question](https://github.com/alibaba/RedisShake/wiki/FAQ#q-does-redisshake-supports-codis-and-twemproxy). + +Support Redis Modules: +[TairHash](https://github.com/alibaba/TairHash): A redis module, similar to redis hash, but you can set expire and version for the field +[TairZset](https://github.com/alibaba/TairZset): A redis module, similar to redis zset, but you can set multiple scores for each member to support multi-dimensional sorting +[TairString](https://github.com/alibaba/TairString): A redis module, similar to redis string, but you can set expire and version for the value. It also provides many very useful commands, such as cas/cad, etc. + + # Configuration Redis-shake has several parameters in the configuration `conf/redis-shake.conf`, that maybe confusing, if this is your first time using, please visit this [tutorial](https://github.com/alibaba/RedisShake/wiki/tutorial-about-how-to-set-up). diff --git a/src/pkg/rdb/module.go b/src/pkg/rdb/module.go new file mode 100644 index 0000000..7cdf663 --- /dev/null +++ b/src/pkg/rdb/module.go @@ -0,0 +1,102 @@ +package rdb + +import ( + "bytes" + "fmt" +) + +// this function is used to parse rdb module + +func (r *rdbReader) parseModule(moduleName string, moduleId uint64, t byte, b *bytes.Buffer) ([]byte, error) { + // the module providing the 10 bit encoding version in the lower 10 bits of the module ID. + encodeVersion := moduleId & 1023 + switch moduleName { + case "tairhash-": + // length + length, err := r.moduleLoadUnsigned(t) + if err != nil { + return nil, err + } + + // key + if _, err := r.moduleLoadString(t); err != nil { + return nil, err + } + + for i := uint64(0); i < length; i++ { + // skey + if _, err := r.moduleLoadString(t); err != nil { + return nil, err + } + + // version + if _, err := r.moduleLoadUnsigned(t); err != nil { + return nil, err + } + + // expire + if _, err := r.moduleLoadUnsigned(t); err != nil { + return nil, err + } + + // value + if _, err := r.moduleLoadString(t); err != nil { + return nil, err + } + } + case "exstrtype": + // version + if _, err := r.moduleLoadUnsigned(t); err != nil { + return nil, err + } + + if encodeVersion == 1 { + //flag + if _, err := r.moduleLoadUnsigned(t); err != nil { + return nil, err + } + } + + // value + if _, err := r.moduleLoadString(t); err != nil { + return nil, err + } + + case "tairzset_": + length, err := r.moduleLoadUnsigned(t) + if err != nil { + return nil, err + } + + score_num, err := r.moduleLoadUnsigned(t) + if err != nil { + return nil, err + } + + for i := uint64(0); i < length; i++ { + if _, err := r.moduleLoadString(t); err != nil { + return nil, err + } + + for j := uint64(0); j < score_num; j++ { + if _, err := r.moduleLoadDouble(t); err != nil { + return nil, err + } + } + } + + default: + return nil, fmt.Errorf("unknown module name[%v] with modue id[%v]", moduleName, moduleId) + } + + if t == RdbTypeModule2 { + code, err := r.ReadLength() + if err != nil { + return nil, err + } else if code != rdbModuleOpcodeEof { + return nil, fmt.Errorf("illegal end code[%v] in module type", code) + } + } + + return b.Bytes(), nil +} diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index c5f5112..1838049 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -9,22 +9,26 @@ import ( "fmt" "io" "math" + // "runtime/debug" "strconv" "github.com/alibaba/RedisShake/pkg/libs/errors" + "github.com/alibaba/RedisShake/pkg/libs/log" ) var FromVersion int64 = 9 var ToVersion int64 = 6 const ( - RdbTypeString = 0 - RdbTypeList = 1 - RdbTypeSet = 2 - RdbTypeZSet = 3 - RdbTypeHash = 4 - RdbTypeZSet2 = 5 + RdbTypeString = 0 + RdbTypeList = 1 + RdbTypeSet = 2 + RdbTypeZSet = 3 + RdbTypeHash = 4 + RdbTypeZSet2 = 5 + RdbTypeModule = 6 + RdbTypeModule2 = 7 RdbTypeHashZipmap = 9 RdbTypeListZiplist = 10 @@ -51,6 +55,8 @@ const ( rdbModuleOpcodeFloat = 3 rdbModuleOpcodeDouble = 4 rdbModuleOpcodeString = 5 + + moduleTypeNameCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" ) const ( @@ -298,6 +304,16 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { } } } + case RdbTypeModule2: + // skip 64 bit + if moduleId, err := r.ReadLength64(); err != nil { + return nil, err + } else { + moduleName := moduleTypeNameByID(moduleId) + log.Debugf("handle module id[%v] name[%v]", moduleId, moduleName) + lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0 + return r.parseModule(moduleName, moduleId, t, &b) + } } return b.Bytes(), nil } @@ -664,3 +680,54 @@ func (r *rdbReader) CountZipmapItems(buf *sliceBuffer) (int, error) { _, err := buf.Seek(0, 0) return n, err } + +func moduleTypeNameByID(moduleId uint64) string { + nameList := make([]byte, 9) + moduleId >>= 10 + for i := 8; i >= 0; i-- { + nameList[i] = moduleTypeNameCharSet[moduleId&63] + moduleId >>= 6 + } + return string(nameList) +} + +func (r *rdbReader) moduleLoadUnsigned(rdbType byte) (uint64, error) { + if rdbType == RdbTypeModule2 { + // ver == 2 + if opCode, err := r.ReadLength(); err != nil { + return 0, err + } else if opCode != rdbModuleOpcodeUint { + return 0, fmt.Errorf("opcode[%v] != rdbModuleOpcodeUint[%v]", opCode, rdbModuleOpcodeUint) + } + } + + val, err := r.ReadLength() + return uint64(val), err +} + +func (r *rdbReader) moduleLoadDouble(rdbType byte) (float64, error) { + if rdbType == RdbTypeModule2 { + // ver == 2 + if opCode, err := r.ReadLength(); err != nil { + return 0, err + } else if opCode != rdbModuleOpcodeDouble { + return 0, fmt.Errorf("opcode[%v] != rdbModuleOpcodeDouble[%v]", opCode, + rdbModuleOpcodeDouble) + } + } + + return r.ReadDouble() +} + +func (r *rdbReader) moduleLoadString(rdbType byte) (string, error) { + if rdbType == RdbTypeModule2 { + if opCode, err := r.ReadLength(); err != nil { + return "", err + } else if opCode != rdbModuleOpcodeString { + return "", fmt.Errorf("opcode[%v] != rdbModuleOpcodeString[%v]", opCode, + rdbModuleOpcodeString) + } + } + ret, err := r.ReadString() + return string(ret), err +}