Merge pull request #186 from alibaba/feature-1.6

Feature 1.6

ChangeLog

@@ -1,3 +1,9 @@
+2019-11-12 Alibaba Cloud.
+	* VERSION: 1.6.21
+	* BUGFIX: update redis-go-cluster to solve the MOVED error when target
+	redis type is cluster and key is unicode encoding. Thanks
+	@shuff1e(sfxu@foxmail.com). see#68.
+	* IMPROVE: rump: filter key in fetching stage instead of writing stage.
 2019-10-18 Alibaba Cloud.
 	* VERSION: 1.6.20
 	* IMPROVE: add progress bar in rump mode. see #174.

README.md

@@ -103,3 +103,4 @@ Plus, we have a DingDing(钉钉) group so that users can join and discuss, pleas
 | wangyiyang | wangyiyang.kk@gmail.com |
 | muicoder | muicoder@gmail.com |
 | zhklcf | huikangzhu@126.com |
+| shuff1e | sfxu@foxmail.com |

src/redis-shake/common/utils.go

@@ -781,7 +781,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
 
 	// TODO, need to judge big key
 	if e.Type != rdb.RDBTypeStreamListPacks &&
-		(uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) {
+			(uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) {
+		log.Debugf("restore big key[%s] with length[%v] and member count[%v]", e.Key, len(e.Value), e.RealMemberCount)
 		//use command
 		if conf.Options.Rewrite && e.NeedReadLen == 1 {
 			if !conf.Options.Metric {

src/redis-shake/rump.go

@@ -349,9 +349,9 @@ func (dre *dbRumperExecutor) writer() {
 	preDb := 0
 	preBigKeyDb := 0
 	for ele := range dre.keyChan {
-		if filter.FilterKey(ele.key) {
+		/*if filter.FilterKey(ele.key) {
 			continue
-		}
+		}*/
 		// QoS, limit the qps
 		<-bucket
 
@@ -494,11 +494,26 @@ func (dre *dbRumperExecutor) doFetch(db int) error {
 	log.Infof("dbRumper[%v] executor[%v] start fetching node db[%v]", dre.rumperId, dre.executorId, db)
 
 	for {
-		keys, err := dre.scanner.ScanKey()
+		rawKeys, err := dre.scanner.ScanKey()
 		if err != nil {
 			return err
 		}
 
+		var keys []string
+		if len(conf.Options.FilterKeyBlacklist) != 0 || len(conf.Options.FilterKeyWhitelist) != 0 {
+			// filter keys
+			keys = make([]string, 0, len(rawKeys))
+			for _, key := range rawKeys {
+				if filter.FilterKey(key) {
+					log.Infof("dbRumper[%v] executor[%v] key[%v] filter", dre.rumperId, dre.executorId, key)
+					continue
+				}
+				keys = append(keys, key)
+			}
+		} else {
+			keys = rawKeys
+		}
+
 		log.Debugf("dbRumper[%v] executor[%v] scanned keys number: %v", dre.rumperId, dre.executorId, len(keys))
 
 		if len(keys) != 0 {

src/vendor/vendor.json

@@ -159,10 +159,10 @@
 			"revisionTime": "2019-03-04T09:57:49Z"
 		},
 		{
-			"checksumSHA1": "OF+Rwv9+X+XcgrBZLxpjrO9koKI=",
+			"checksumSHA1": "GYdlJZBp2zfh0QEV/MN4M0L1nPE=",
 			"path": "github.com/vinllen/redis-go-cluster",
-			"revision": "4b1a36f93b18e5c0fd7cf5294e3dd6c52c10542c",
-			"revisionTime": "2019-09-19T05:56:24Z"
+			"revision": "83c0ea01d33aea24bb40b9d46a85fb56dac543cb",
+			"revisionTime": "2019-11-12T06:34:43Z"
 		},
 		{
 			"checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=",