From beb0f2a06d5d786b5e3ee085f514307ec482f9b7 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 3 Apr 2019 19:18:28 +0800 Subject: [PATCH 1/4] support 5.0 rdb RDB_OPCODE_MODULE_AUX, RDB_OPCODE_IDLE and RDB_OPCODE_FREQ type --- src/pkg/rdb/loader.go | 24 ++++++++++++++++++++++-- src/pkg/rdb/reader.go | 12 +++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go index 5f84ac9..7ca4faf 100644 --- a/src/pkg/rdb/loader.go +++ b/src/pkg/rdb/loader.go @@ -157,8 +157,28 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { l.db = dbnum case rdbFlagEOF: return nil, nil - case rdbFlagOnlyValue: - fallthrough + case rdbFlagModuleAux: + // currently, ignore this filed + _, err := l.ReadLength() // module-id + if err != nil { + return nil, err + } + // skip module + if err = rdbLoadCheckModuleValue(l); err != nil { + return nil, err + } + case rdbFlagIdle: + // ignore idle because target redis doesn't support this for given key + _, err := l.ReadLength() + if err != nil { + return nil, err + } + case rdbFlagFreq: + // ignore freq because target redis doesn't support this for given key + _, err := l.readUint8() + if err != nil { + return nil, err + } default: var key []byte if l.remainMember == 0 { diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index 068eb94..f9a5c5e 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -35,13 +35,23 @@ const ( RdbTypeQuicklist = 14 RDBTypeStreamListPacks = 15 // stream - rdbFlagOnlyValue = 0xf9 + rdbFlagModuleAux = 0xf7 + rdbFlagIdle = 0xf8 + rdbFlagFreq = 0xf9 RdbFlagAUX = 0xfa rdbFlagResizeDB = 0xfb rdbFlagExpiryMS = 0xfc rdbFlagExpiry = 0xfd rdbFlagSelectDB = 0xfe rdbFlagEOF = 0xff + + // Module serialized values sub opcodes + rdbModuleOpcodeEof = 0 + rdbModuleOpcodeSint = 1 + rdbModuleOpcodeUint = 2 + rdbModuleOpcodeFloat = 3 + rdbModuleOpcodeDouble = 4 + rdbModuleOpcodeString = 5 ) const ( From 9fa0f79cca319c0485dff85137c16e35ff4791d7 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 3 Apr 2019 19:20:33 +0800 Subject: [PATCH 2/4] add 1.2.2 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8073bc9..f6648d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2019-04-03 Alibaba Cloud. + * version: 1.2.2 + * BUGFIX: support 5.0 rdb RDB_OPCODE_MODULE_AUX, RDB_OPCODE_IDLE and + RDB_OPCODE_FREQ type. 2019-03-27 Alibaba Cloud. * version: 1.2.1 * IMPROVE: support syncing lua script in RDB syncing. From 0132d359df1fd09d372e537564eed44bcf40ea18 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 3 Apr 2019 19:33:20 +0800 Subject: [PATCH 3/4] update, forget mix.go --- .gitignore | 1 - src/pkg/rdb/mix.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/pkg/rdb/mix.go diff --git a/.gitignore b/.gitignore index 543e80e..c2f7e30 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -pkg .gopath .idea *.iml diff --git a/src/pkg/rdb/mix.go b/src/pkg/rdb/mix.go new file mode 100644 index 0000000..850e387 --- /dev/null +++ b/src/pkg/rdb/mix.go @@ -0,0 +1,37 @@ +package rdb + +func rdbLoadCheckModuleValue(l *Loader) error { + var opcode uint32 + var err error + for { + if opcode, err = l.ReadLength(); err != nil { + return err + } else if opcode == rdbModuleOpcodeEof { + break + } + + switch opcode { + case rdbModuleOpcodeSint: + fallthrough + case rdbModuleOpcodeUint: + if _, err = l.ReadLength(); err != nil { + return err + } + case rdbModuleOpcodeString: + if _, err = l.ReadString(); err != nil { + return err + } + case rdbModuleOpcodeFloat: + // float 32 bits + if _, err = l.ReadFloat(); err != nil { + return err + } + case rdbModuleOpcodeDouble: + // double 64 bits + if _, err = l.ReadDouble(); err != nil { + return err + } + } + } + return nil +} \ No newline at end of file From e90826302ed48fc14a33c7dea22af2c440abbd9d Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 4 Apr 2019 14:19:42 +0800 Subject: [PATCH 4/4] use restore to recovery key --- src/pkg/rdb/loader.go | 8 ++++++-- src/redis-shake/common/utils.go | 21 ++++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go index 7ca4faf..c7ec8d4 100644 --- a/src/pkg/rdb/loader.go +++ b/src/pkg/rdb/loader.go @@ -63,6 +63,8 @@ type BinEntry struct { ExpireAt uint64 RealMemberCount uint32 NeedReadLen byte + IdleTime uint32 + Freq uint8 } func (e *BinEntry) ObjEntry() (*ObjEntry, error) { @@ -169,16 +171,18 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { } case rdbFlagIdle: // ignore idle because target redis doesn't support this for given key - _, err := l.ReadLength() + idle, err := l.ReadLength() if err != nil { return nil, err } + entry.IdleTime = idle case rdbFlagFreq: // ignore freq because target redis doesn't support this for given key - _, err := l.readUint8() + freq, err := l.readUint8() if err != nil { return nil, err } + entry.Freq = freq default: var key []byte if l.remainMember == 0 { diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index f8ef081..e4b30df 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -711,12 +711,22 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { return } - // fmt.Printf("kkkey: %v, value: %v\n", string(e.Key), e.Value) - s, err := redigo.String(c.Do("restore", e.Key, ttlms, e.Value)) + params := []interface{}{e.Key, ttlms, e.Value} + if e.IdleTime != 0 { + params = append(params, "IDLETIME") + params = append(params, e.IdleTime) + } + if e.Freq != 0 { + params = append(params, "FREQ") + params = append(params, e.Freq) + } + // fmt.Printf("key: %v, value: %v params: %v\n", string(e.Key), e.Value, params) + s, err := redigo.String(c.Do("restore", params...)) if err != nil { /*The reply value of busykey in 2.8 kernel is "target key name is busy", but in 4.0 kernel is "BUSYKEY Target key name already exists"*/ - if strings.Contains(err.Error(), "Target key name is busy") || strings.Contains(err.Error(), "BUSYKEY Target key name already exists") { + if strings.Contains(err.Error(), "Target key name is busy") || + strings.Contains(err.Error(), "BUSYKEY Target key name already exists") { if conf.Options.Rewrite { if !conf.Options.Metric { log.Infof("warning, rewrite key: %v", string(e.Key)) @@ -724,10 +734,11 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { var s2 string var rerr error if conf.Options.TargetReplace { - s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value, "replace")) + params = append(params, "REPLACE") + s2, rerr = redigo.String(c.Do("restore", params...)) } else { _, _ = redigo.String(c.Do("del", e.Key)) - s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value)) + s2, rerr = redigo.String(c.Do("restore", params...)) } if rerr != nil { log.Info(s2, rerr, "key ", string(e.Key))