diff --git a/ChangeLog b/ChangeLog index f974001..dce2ee4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2019-11-12 Alibaba Cloud. + * VERSION: 1.6.21 + * BUGFIX: update redis-go-cluster to solve the MOVED error when target + redis type is cluster and key is unicode encoding. Thanks + @shuff1e(sfxu@foxmail.com). see#68. + * IMPROVE: rump: filter key in fetching stage instead of writing stage. 2019-10-18 Alibaba Cloud. * VERSION: 1.6.20 * IMPROVE: add progress bar in rump mode. see #174. diff --git a/README.md b/README.md index db3dca3..a44c6b2 100644 --- a/README.md +++ b/README.md @@ -103,3 +103,4 @@ Plus, we have a DingDing(钉钉) group so that users can join and discuss, pleas | wangyiyang | wangyiyang.kk@gmail.com | | muicoder | muicoder@gmail.com | | zhklcf | huikangzhu@126.com | +| shuff1e | sfxu@foxmail.com | diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index edb21ae..d102ec3 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -781,7 +781,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { // TODO, need to judge big key if e.Type != rdb.RDBTypeStreamListPacks && - (uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) { + (uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) { + log.Debugf("restore big key[%s] with length[%v] and member count[%v]", e.Key, len(e.Value), e.RealMemberCount) //use command if conf.Options.Rewrite && e.NeedReadLen == 1 { if !conf.Options.Metric { diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index fe34842..3277249 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -349,9 +349,9 @@ func (dre *dbRumperExecutor) writer() { preDb := 0 preBigKeyDb := 0 for ele := range dre.keyChan { - if filter.FilterKey(ele.key) { + /*if filter.FilterKey(ele.key) { continue - } + }*/ // QoS, limit the qps <-bucket @@ -494,11 +494,26 @@ func (dre *dbRumperExecutor) doFetch(db int) error { log.Infof("dbRumper[%v] executor[%v] start fetching node db[%v]", dre.rumperId, dre.executorId, db) for { - keys, err := dre.scanner.ScanKey() + rawKeys, err := dre.scanner.ScanKey() if err != nil { return err } + var keys []string + if len(conf.Options.FilterKeyBlacklist) != 0 || len(conf.Options.FilterKeyWhitelist) != 0 { + // filter keys + keys = make([]string, 0, len(rawKeys)) + for _, key := range rawKeys { + if filter.FilterKey(key) { + log.Infof("dbRumper[%v] executor[%v] key[%v] filter", dre.rumperId, dre.executorId, key) + continue + } + keys = append(keys, key) + } + } else { + keys = rawKeys + } + log.Debugf("dbRumper[%v] executor[%v] scanned keys number: %v", dre.rumperId, dre.executorId, len(keys)) if len(keys) != 0 { diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index 2b5de40..6a2a38d 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "OF+Rwv9+X+XcgrBZLxpjrO9koKI=", + "checksumSHA1": "GYdlJZBp2zfh0QEV/MN4M0L1nPE=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "4b1a36f93b18e5c0fd7cf5294e3dd6c52c10542c", - "revisionTime": "2019-09-19T05:56:24Z" + "revision": "83c0ea01d33aea24bb40b9d46a85fb56dac543cb", + "revisionTime": "2019-11-12T06:34:43Z" }, { "checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=",