From 2b1ece66624e8cfbf81b29e1e3ed3c9915efcd93 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 16 Oct 2019 11:46:34 +0800 Subject: [PATCH] set big_key_threshold to 1 to avoid some sync failed cases. see #173 --- ChangeLog | 6 ++++ src/redis-shake/common/common.go | 49 ++++++++++++++++++++++++++ src/redis-shake/common/utils_test.go | 16 +++++++++ src/redis-shake/configure/configure.go | 1 + src/redis-shake/main/main.go | 29 +++++++++++++++ 5 files changed, 101 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7af45cf..a0025ef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2019-10-xx Alibaba Cloud. + * VERSION: 1.6.20 + * IMPROVE: add progress bar in rump mode. see #174. + * IMPROVE: add run_direct.py script. + * IMRPOVE: set big_key_threshold to 1 to avoid some sync failed cases. see + #173. 2019-09-19 Alibaba Cloud. * VERSION: 1.6.19 * BUGFIX: update "redis-go-cluster" driver to fix bug to throw CROSSSLOT diff --git a/src/redis-shake/common/common.go b/src/redis-shake/common/common.go index 74b6d46..8f75dc3 100644 --- a/src/redis-shake/common/common.go +++ b/src/redis-shake/common/common.go @@ -14,6 +14,7 @@ import ( logRotate "gopkg.in/natefinch/lumberjack.v2" "github.com/cupcake/rdb/crc64" + "strconv" ) const ( @@ -163,4 +164,52 @@ func GetMetric(input int64) string { default: return fmt.Sprintf("%dB", input) } +} + +/* + * compare the version with given level. e.g., + * 2.0.1, 2.0.3, level = 2 => equal: 0 + * 2.0.1, 2.0.3, level = 3 => smaller: 1 + * 3.1.1, 2.1 level = 2 => bigger: 2 + * 3, 3.2, level = 2 => smaller: 1 + * 3.a, 3.2, level = 2 => unknown: 3 + */ +func CompareVersion(a, b string, level int) int { + if level <= 0 { + return 0 + } + + var err error + as := strings.Split(a, ".") + bs := strings.Split(b, ".") + for l := 0; l < level; l++ { + var av, bv int + // parse av + if l > len(as) { + av = 0 + } else { + av, err = strconv.Atoi(as[l]) + if err != nil { + return 3 + } + } + + // parse bv + if l > len(bs) { + bv = 0 + } else { + bv, err = strconv.Atoi(bs[l]) + if err != nil { + return 3 + } + } + + if av > bv { + return 2 + } else if av < bv { + return 1 + } + } + + return 0 } \ No newline at end of file diff --git a/src/redis-shake/common/utils_test.go b/src/redis-shake/common/utils_test.go index 240b540..ababcac 100644 --- a/src/redis-shake/common/utils_test.go +++ b/src/redis-shake/common/utils_test.go @@ -84,4 +84,20 @@ func TestCutRedisInfoSegment(t *testing.T) { assert.Equal(t, len(expect), len(ret), "should be equal") assert.Equal(t, expect, ret, "should be equal") } +} + +func TestCompareVersion(t *testing.T) { + var nr int + { + fmt.Printf("TestCompareVersion case %d.\n", nr) + nr++ + + assert.Equal(t, 1, CompareVersion("1.2", "1.3", 2), "should be equal") + assert.Equal(t, 0, CompareVersion("1.2", "1.3", 1), "should be equal") + assert.Equal(t, 2, CompareVersion("1.4", "1.3", 2), "should be equal") + assert.Equal(t, 2, CompareVersion("1.4.x", "1.3", 2), "should be equal") + assert.Equal(t, 3, CompareVersion("1.4.x", "1.4", 3), "should be equal") + assert.Equal(t, 0, CompareVersion("1.4.x", "1.4", 0), "should be equal") + assert.Equal(t, 2, CompareVersion("2.4", "1.1", 2), "should be equal") + } } \ No newline at end of file diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index ec5343d..79e4ab6 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -68,6 +68,7 @@ type Configuration struct { // generated variables SourceAddressList []string // source address list TargetAddressList []string // target address list + SourceVersion string // source version HeartbeatIp string // heartbeat ip ShiftTime time.Duration // shift TargetReplace bool // to_replace diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index ff2c05b..f615f50 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -414,6 +414,7 @@ func sanitizeOptions(tp string) error { } if tp == conf.TypeRestore || tp == conf.TypeSync || tp == conf.TypeRump { + // version check is useless, we only want to verify the correctness of configuration. if conf.Options.TargetVersion == "" { // get target redis version and set TargetReplace. for _, address := range conf.Options.TargetAddressList { @@ -427,6 +428,12 @@ func sanitizeOptions(tp string) error { conf.Options.TargetVersion = v } } + } else { + /* + * see github issue #173. + * set 1 if target is target version can't be fetched just like twemproxy. + */ + conf.Options.BigKeyThreshold = 1 } if strings.HasPrefix(conf.Options.TargetVersion, "4.") || @@ -438,6 +445,28 @@ func sanitizeOptions(tp string) error { } } + // check version and set big_key_threshold. see #173 + if tp == conf.TypeSync || tp == conf.TypeRump { // "tp == restore" hasn't been handled + // fetch source version + for _, address := range conf.Options.SourceAddressList { + // single connection even if the target is cluster + if v, err := utils.GetRedisVersion(address, conf.Options.SourceAuthType, + conf.Options.SourcePasswordRaw, conf.Options.SourceTLSEnable); err != nil { + return fmt.Errorf("get source redis version failed[%v]", err) + } else if conf.Options.SourceVersion != "" && conf.Options.SourceVersion != v { + return fmt.Errorf("source redis version is different: [%v %v]", conf.Options.SourceVersion, v) + } else { + conf.Options.SourceVersion = v + } + } + + // compare version. see github issue #173. + if ret := utils.CompareVersion(conf.Options.SourceVersion, conf.Options.TargetVersion, 2); ret != 0 && ret != 1 { + // target version is smaller than source version, or unknown + conf.Options.BigKeyThreshold = 1 + } + } + if tp == conf.TypeRump { if conf.Options.ScanKeyNumber == 0 { conf.Options.ScanKeyNumber = 100