Merge pull request #129 from alibaba/improve-1.6.13

Improve 1.6.13

ChangeLog

@@ -1,3 +1,11 @@
+2019-07-24 Alibaba Cloud.
+	* VERSION: 1.6.13
+	* IMPROVE: support `filter.db.whitelist` and `filter.db.blacklist` to let
+	different db syncing to db0 even when target type is cluster. see #127.
+	* BUGFIX: fix bug of connection url in automatic discovery in cluster. see
+	#124.
+	* IMPROVE: support `target.db` in rump mode.
+	* IMPROVE: add debug log in RDB syncing.
 2019-07-11 Alibaba Cloud.
 	* VERSION: 1.6.12
 	* IMPROVE: support filter key with whitelist and blacklist.

src/redis-shake/main/main.go

@@ -328,13 +328,6 @@ func sanitizeOptions(tp string) error {
 		return fmt.Errorf("only one of 'filter.key.whitelist' and 'filter.key.blacklist' can be given")
 	}
 
-	// if the target is "cluster", only allow pass db 0
-	if conf.Options.TargetType == conf.RedisTypeCluster {
-		conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0
-		conf.Options.FilterDBBlacklist = []string{}  // reset blacklist
-		log.Info("the target redis type is cluster, only pass db0")
-	}
-
 	if len(conf.Options.FilterSlot) > 0 {
 		for i, val := range conf.Options.FilterSlot {
 			if _, err := strconv.Atoi(val); err != nil {
@@ -353,8 +346,18 @@ func sanitizeOptions(tp string) error {
 		conf.Options.TargetDB = v
 	}
 
-	if conf.Options.TargetDB > 0 && conf.Options.TargetType == conf.RedisTypeCluster {
-		return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB)
+	// if the target is "cluster", only allow pass db 0
+	if conf.Options.TargetType == conf.RedisTypeCluster {
+		if conf.Options.TargetDB == -1 {
+			conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0
+			conf.Options.FilterDBBlacklist = []string{}    // reset blacklist
+			log.Info("the target redis type is cluster, only pass db0")
+		} else if conf.Options.TargetDB == 0 {
+			log.Info("the target redis type is cluster, all db syncing to db0")
+		} else {
+			// > 0
+			return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB)
+		}
 	}
 
 	if conf.Options.HttpProfile < 0 || conf.Options.HttpProfile > 65535 {

src/redis-shake/restore.go

@@ -158,6 +158,9 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth
 						dr.ignore.Incr()
 					} else {
 						dr.nentry.Incr()
+
+						log.Debugf("routine[%v] try restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value))
+
 						if conf.Options.TargetDB != -1 {
 							if conf.Options.TargetDB != int(lastdb) {
 								lastdb = uint32(conf.Options.TargetDB)
@@ -173,9 +176,11 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth
 						if filter.FilterKey(string(e.Key)) {
 							continue
 						}
-						utils.RestoreRdbEntry(c, e)
 
-						log.Debugf("routine[%v] restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value))
+						log.Debugf("routine[%v] start restoring key[%s] with value length[%v]", dr.id, e.Key, len(e.Value))
+
+						utils.RestoreRdbEntry(c, e)
+						log.Debugf("routine[%v] restore key[%s] ok", dr.id, e.Key)
 					}
 				}
 			}()

src/redis-shake/sync.go

@@ -414,6 +414,9 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type
 						ds.ignore.Incr()
 					} else {
 						ds.nentry.Incr()
+
+						log.Debugf("dbSyncer[%v] try restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value))
+
 						if conf.Options.TargetDB != -1 {
 							if conf.Options.TargetDB != int(lastdb) {
 								lastdb = uint32(conf.Options.TargetDB)
@@ -438,8 +441,11 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type
 								continue
 							}
 						}
+
+						log.Debugf("dbSyncer[%v] start restoring key[%s] with value length[%v]", ds.id, e.Key, len(e.Value))
+
 						utils.RestoreRdbEntry(c, e)
-						log.Debugf("dbSyncer[%v] restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value))
+						log.Debugf("dbSyncer[%v] restore key[%s] ok", ds.id, e.Key)
 					}
 				}
 			}()