Merge pull request #5 from alibaba/develop

update

.gitignore

@@ -1,4 +1,3 @@
-pkg
 .gopath
 .idea
 *.iml

ChangeLog

@@ -1,3 +1,7 @@
+2019-04-03 Alibaba Cloud.
+	* version: 1.2.2
+	* BUGFIX: support 5.0 rdb RDB_OPCODE_MODULE_AUX, RDB_OPCODE_IDLE and
+	RDB_OPCODE_FREQ type.
 2019-03-27 Alibaba Cloud.
 	* version: 1.2.1
 	* IMPROVE: support syncing lua script in RDB syncing.

src/pkg/rdb/loader.go

@@ -63,6 +63,8 @@ type BinEntry struct {
 	ExpireAt        uint64
 	RealMemberCount uint32
 	NeedReadLen     byte
+	IdleTime        uint32
+	Freq            uint8
 }
 
 func (e *BinEntry) ObjEntry() (*ObjEntry, error) {
@@ -157,8 +159,30 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) {
 			l.db = dbnum
 		case rdbFlagEOF:
 			return nil, nil
-		case rdbFlagOnlyValue:
+		case rdbFlagModuleAux:
-			fallthrough
+			// currently, ignore this filed
+			_, err := l.ReadLength() // module-id
+			if err != nil {
+				return nil, err
+			}
+			// skip module
+			if err = rdbLoadCheckModuleValue(l); err != nil {
+				return nil, err
+			}
+		case rdbFlagIdle:
+			// ignore idle because target redis doesn't support this for given key
+			idle, err := l.ReadLength()
+			if err != nil {
+				return nil, err
+			}
+			entry.IdleTime = idle
+		case rdbFlagFreq:
+			// ignore freq because target redis doesn't support this for given key
+			freq, err := l.readUint8()
+			if err != nil {
+				return nil, err
+			}
+			entry.Freq = freq
 		default:
 			var key []byte
 			if l.remainMember == 0 {

src/pkg/rdb/mix.go

@@ -0,0 +1,37 @@
+package rdb
+
+func rdbLoadCheckModuleValue(l *Loader) error {
+	var opcode uint32
+	var err error
+	for {
+		if opcode, err = l.ReadLength(); err != nil {
+			return err
+		} else if opcode == rdbModuleOpcodeEof {
+			break
+		}
+
+		switch opcode {
+		case rdbModuleOpcodeSint:
+			fallthrough
+		case rdbModuleOpcodeUint:
+			if _, err = l.ReadLength(); err != nil {
+				return err
+			}
+		case rdbModuleOpcodeString:
+			if _, err = l.ReadString(); err != nil {
+				return err
+			}
+		case rdbModuleOpcodeFloat:
+			// float 32 bits
+			if _, err = l.ReadFloat(); err != nil {
+				return err
+			}
+		case rdbModuleOpcodeDouble:
+			// double 64 bits
+			if _, err = l.ReadDouble(); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

src/pkg/rdb/reader.go

@@ -35,13 +35,23 @@ const (
 	RdbTypeQuicklist       = 14
 	RDBTypeStreamListPacks = 15 // stream
 
-	rdbFlagOnlyValue = 0xf9
+	rdbFlagModuleAux = 0xf7
+	rdbFlagIdle      = 0xf8
+	rdbFlagFreq      = 0xf9
 	RdbFlagAUX       = 0xfa
 	rdbFlagResizeDB  = 0xfb
 	rdbFlagExpiryMS  = 0xfc
 	rdbFlagExpiry    = 0xfd
 	rdbFlagSelectDB  = 0xfe
 	rdbFlagEOF       = 0xff
+
+	// Module serialized values sub opcodes
+	rdbModuleOpcodeEof    = 0
+	rdbModuleOpcodeSint   = 1
+	rdbModuleOpcodeUint   = 2
+	rdbModuleOpcodeFloat  = 3
+	rdbModuleOpcodeDouble = 4
+	rdbModuleOpcodeString = 5
 )
 
 const (

src/redis-shake/common/utils.go

@@ -711,12 +711,22 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
 		return
 	}
 
-	// fmt.Printf("kkkey: %v, value: %v\n", string(e.Key), e.Value)
+	params := []interface{}{e.Key, ttlms, e.Value}
-	s, err := redigo.String(c.Do("restore", e.Key, ttlms, e.Value))
+	if e.IdleTime != 0 {
+		params = append(params, "IDLETIME")
+		params = append(params, e.IdleTime)
+	}
+	if e.Freq != 0 {
+		params = append(params, "FREQ")
+		params = append(params, e.Freq)
+	}
+	// fmt.Printf("key: %v, value: %v params: %v\n", string(e.Key), e.Value, params)
+	s, err := redigo.String(c.Do("restore", params...))
 	if err != nil {
 		/*The reply value of busykey in 2.8 kernel is "target key name is busy",
 		  but in 4.0 kernel is "BUSYKEY Target key name already exists"*/
-		if strings.Contains(err.Error(), "Target key name is busy") || strings.Contains(err.Error(), "BUSYKEY Target key name already exists") {
+		if strings.Contains(err.Error(), "Target key name is busy") ||
+			strings.Contains(err.Error(), "BUSYKEY Target key name already exists") {
 			if conf.Options.Rewrite {
 				if !conf.Options.Metric {
 					log.Infof("warning, rewrite key: %v", string(e.Key))
@@ -724,10 +734,11 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
 				var s2 string
 				var rerr error
 				if conf.Options.TargetReplace {
-					s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value, "replace"))
+					params = append(params, "REPLACE")
+					s2, rerr = redigo.String(c.Do("restore", params...))
 				} else {
 					_, _ = redigo.String(c.Do("del", e.Key))
-					s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value))
+					s2, rerr = redigo.String(c.Do("restore", params...))
 				}
 				if rerr != nil {
 					log.Info(s2, rerr, "key ", string(e.Key))