set big_key_threshold to 1 to avoid some sync failed cases. see #173

ChangeLog

@@ -1,3 +1,9 @@
+2019-10-xx Alibaba Cloud.
+	* VERSION: 1.6.20
+	* IMPROVE: add progress bar in rump mode. see #174.
+	* IMPROVE: add run_direct.py script.
+	* IMRPOVE: set big_key_threshold to 1 to avoid some sync failed cases. see
+	#173.
 2019-09-19 Alibaba Cloud.
 	* VERSION: 1.6.19
 	* BUGFIX: update "redis-go-cluster" driver to fix bug to throw CROSSSLOT

src/redis-shake/common/common.go

@@ -14,6 +14,7 @@ import (
 
 	logRotate "gopkg.in/natefinch/lumberjack.v2"
 	"github.com/cupcake/rdb/crc64"
+	"strconv"
 )
 
 const (
@@ -163,4 +164,52 @@ func GetMetric(input int64) string {
 	default:
 		return fmt.Sprintf("%dB", input)
 	}
+}
+
+/*
+ * compare the version with given level. e.g.,
+ * 2.0.1, 2.0.3, level = 2 => equal: 0
+ * 2.0.1, 2.0.3, level = 3 => smaller: 1
+ * 3.1.1, 2.1 level = 2 => bigger: 2
+ * 3, 3.2, level = 2 => smaller: 1
+ * 3.a, 3.2, level = 2 => unknown: 3
+ */
+func CompareVersion(a, b string, level int) int {
+	if level <= 0 {
+		return 0
+	}
+
+	var err error
+	as := strings.Split(a, ".")
+	bs := strings.Split(b, ".")
+	for l := 0; l < level; l++ {
+		var av, bv int
+		// parse av
+		if l > len(as) {
+			av = 0
+		} else {
+			av, err = strconv.Atoi(as[l])
+			if err != nil {
+				return 3
+			}
+		}
+
+		// parse bv
+		if l > len(bs) {
+			bv = 0
+		} else {
+			bv, err = strconv.Atoi(bs[l])
+			if err != nil {
+				return 3
+			}
+		}
+
+		if av > bv {
+			return 2
+		} else if av < bv {
+			return 1
+		}
+	}
+
+	return 0
 }

src/redis-shake/common/utils_test.go

@@ -84,4 +84,20 @@ func TestCutRedisInfoSegment(t *testing.T) {
 		assert.Equal(t, len(expect), len(ret), "should be equal")
 		assert.Equal(t, expect, ret, "should be equal")
 	}
+}
+
+func TestCompareVersion(t *testing.T) {
+	var nr int
+	{
+		fmt.Printf("TestCompareVersion case %d.\n", nr)
+		nr++
+
+		assert.Equal(t, 1, CompareVersion("1.2", "1.3", 2), "should be equal")
+		assert.Equal(t, 0, CompareVersion("1.2", "1.3", 1), "should be equal")
+		assert.Equal(t, 2, CompareVersion("1.4", "1.3", 2), "should be equal")
+		assert.Equal(t, 2, CompareVersion("1.4.x", "1.3", 2), "should be equal")
+		assert.Equal(t, 3, CompareVersion("1.4.x", "1.4", 3), "should be equal")
+		assert.Equal(t, 0, CompareVersion("1.4.x", "1.4", 0), "should be equal")
+		assert.Equal(t, 2, CompareVersion("2.4", "1.1", 2), "should be equal")
+	}
 }

src/redis-shake/configure/configure.go

@@ -68,6 +68,7 @@ type Configuration struct {
 	// generated variables
 	SourceAddressList []string      // source address list
 	TargetAddressList []string      // target address list
+	SourceVersion     string        // source version
 	HeartbeatIp       string        // heartbeat ip
 	ShiftTime         time.Duration // shift
 	TargetReplace     bool          // to_replace

src/redis-shake/main/main.go

@@ -414,6 +414,7 @@ func sanitizeOptions(tp string) error {
 	}
 
 	if tp == conf.TypeRestore || tp == conf.TypeSync || tp == conf.TypeRump {
+		// version check is useless, we only want to verify the correctness of configuration.
 		if conf.Options.TargetVersion == "" {
 			// get target redis version and set TargetReplace.
 			for _, address := range conf.Options.TargetAddressList {
@@ -427,6 +428,12 @@ func sanitizeOptions(tp string) error {
 					conf.Options.TargetVersion = v
 				}
 			}
+		} else {
+			/*
+			 * see github issue #173.
+			 * set 1 if target is target version can't be fetched just like twemproxy.
+			 */
+			conf.Options.BigKeyThreshold = 1
 		}
 
 		if strings.HasPrefix(conf.Options.TargetVersion, "4.") ||
@@ -438,6 +445,28 @@ func sanitizeOptions(tp string) error {
 		}
 	}
 
+	// check version and set big_key_threshold. see #173
+	if tp == conf.TypeSync || tp == conf.TypeRump { // "tp == restore" hasn't been handled
+		// fetch source version
+		for _, address := range conf.Options.SourceAddressList {
+			// single connection even if the target is cluster
+			if v, err := utils.GetRedisVersion(address, conf.Options.SourceAuthType,
+				conf.Options.SourcePasswordRaw, conf.Options.SourceTLSEnable); err != nil {
+				return fmt.Errorf("get source redis version failed[%v]", err)
+			} else if conf.Options.SourceVersion != "" && conf.Options.SourceVersion != v {
+				return fmt.Errorf("source redis version is different: [%v %v]", conf.Options.SourceVersion, v)
+			} else {
+				conf.Options.SourceVersion = v
+			}
+		}
+
+		// compare version. see github issue #173.
+		if ret := utils.CompareVersion(conf.Options.SourceVersion, conf.Options.TargetVersion, 2); ret != 0 && ret != 1 {
+			// target version is smaller than source version, or unknown
+			conf.Options.BigKeyThreshold = 1
+		}
+	}
+
 	if tp == conf.TypeRump {
 		if conf.Options.ScanKeyNumber == 0 {
 			conf.Options.ScanKeyNumber = 100