From 3e811b7c3bbf67cc509e83a7ff12a5619df74ad2 Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 14 Nov 2019 20:05:14 +0800 Subject: [PATCH 1/9] add more log in redis-go-cluster --- src/vendor/vendor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index 6a2a38d..01db81c 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "GYdlJZBp2zfh0QEV/MN4M0L1nPE=", + "checksumSHA1": "mMb6xH+BqaCwZ3vANz4DjVBttKs=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "83c0ea01d33aea24bb40b9d46a85fb56dac543cb", - "revisionTime": "2019-11-12T06:34:43Z" + "revision": "cc60a08c6758e2baecc90c8e952f0cf976b29671", + "revisionTime": "2019-11-14T12:00:39Z" }, { "checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=", From b1b608206583a710756b61e55665371bf278192b Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 19 Nov 2019 11:54:08 +0800 Subject: [PATCH 2/9] bugfix: flushcount=4097 > 4096 => flushcount=4096 >= 4096 --- src/redis-shake/sync.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index b5ae439..9ef8aa4 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -588,7 +588,7 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type decoder := redis.NewDecoder(reader) - log.Infof("dbSyncer[%v] Event:IncrSyncStart\tId:%s\t", ds.id, conf.Options.Id) + log.Infof("dbSyncer[%v] FlushEvent:IncrSyncStart\tId:%s\t", ds.id, conf.Options.Id) for { ignorecmd := false @@ -688,8 +688,8 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type ds.addDelayChan(sendId.Get()) } - if noFlushCount > conf.Options.SenderCount || cachedSize > conf.Options.SenderSize || - len(ds.sendBuf) == 0 { // 5000 ds in a batch + if noFlushCount >= conf.Options.SenderCount || cachedSize >= conf.Options.SenderSize || + len(ds.sendBuf) == 0 { // 5000 ds in a batch err := c.Flush() noFlushCount = 0 cachedSize = 0 From c628ed7a0b3736f3501ef25403f21f097605ba8d Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 20 Nov 2019 15:25:12 +0800 Subject: [PATCH 3/9] update redis-go-cluster to solve the bug of \r\n. see #73 in redis-full-check --- src/vendor/vendor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index 01db81c..7e153fd 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "mMb6xH+BqaCwZ3vANz4DjVBttKs=", + "checksumSHA1": "q574+fUCCQIVoIEmTGky+segYFw=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "cc60a08c6758e2baecc90c8e952f0cf976b29671", - "revisionTime": "2019-11-14T12:00:39Z" + "revision": "30a947c1a707090d95437d853a9766d12625ffbb", + "revisionTime": "2019-11-20T07:22:19Z" }, { "checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=", From 3769770db28c6594ec510825e9ac821748036e6e Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 20 Nov 2019 15:30:09 +0800 Subject: [PATCH 4/9] update redis-go-cluster to solve the bug of \r\n. see #73 in redis-full-check 2 --- src/vendor/vendor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index 7e153fd..744c766 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "q574+fUCCQIVoIEmTGky+segYFw=", + "checksumSHA1": "TCoDN4L8oq/S41a1VR2iONqN+mE=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "30a947c1a707090d95437d853a9766d12625ffbb", - "revisionTime": "2019-11-20T07:22:19Z" + "revision": "6e304ef1b7fc2db6f913570112aa2a2c4d80ef81", + "revisionTime": "2019-11-20T07:29:33Z" }, { "checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=", From 1c56a336a67e348a5021745e1b596bdbcf9256bb Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 25 Nov 2019 18:40:52 +0800 Subject: [PATCH 5/9] release v1.6.22 --- ChangeLog | 9 +++++++++ src/redis-shake/common/crc16.go | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index e57b535..f2ed9d3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2019-11-25 Alibaba Cloud. + * VERSION: 1.6.22 + * BUGFIX: solve MOVED error when key is unicode encoding which is not + solved completed in v1.6.21. + * BUGFIX: update redis-go-cluster to solve the bug of \r\n. see #73 in + redis-full-check. + * BUGFIX: solve flushcount comparison: "flushcount=4097 > 4096" => + "flushcount=4096 >= 4096". + * IMPROVE: add more log in redis-go-cluster. 2019-11-12 Alibaba Cloud. * VERSION: 1.6.21 * BUGFIX: update redis-go-cluster to solve the MOVED error when target diff --git a/src/redis-shake/common/crc16.go b/src/redis-shake/common/crc16.go index 5f4f9c4..b238905 100644 --- a/src/redis-shake/common/crc16.go +++ b/src/redis-shake/common/crc16.go @@ -81,8 +81,8 @@ var crc16tab = [256]uint16{ func crc16(buf string) uint16 { var crc uint16 - for _, n := range buf { - crc = (crc << uint16(8)) ^ crc16tab[((crc>>uint16(8))^uint16(n))&0x00FF] + for i := 0; i < len(buf); i++ { + crc = (crc << uint16(8)) ^ crc16tab[((crc>>uint16(8))^uint16(buf[i]))&0x00FF] } return crc -} +} \ No newline at end of file From 37dcc7dea37cc2cc50aaed155f430511dea3e3c4 Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 28 Nov 2019 12:18:12 +0800 Subject: [PATCH 6/9] update driver bugfix: 1. bulk string return -1 is legal; 2. handle lua script in cluster --- ChangeLog | 6 ++++++ src/redis-shake/sync.go | 4 ++-- src/vendor/vendor.json | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index f2ed9d3..52d0c77 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2019-11-28 Alibaba Cloud. + * VERSION: 1.6.23 + * BUGFIX: update redis-go-cluster driver to solve MOVED error in lua + script. + * BUGFIX: update redis-go-clister driver to remove the error when meets + nil bluk string. 2019-11-25 Alibaba Cloud. * VERSION: 1.6.22 * BUGFIX: solve MOVED error when key is unicode encoding which is not diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 9ef8aa4..83bbc44 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -535,7 +535,7 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type id := recvId.Get() // receive id // print debug log of receive reply - log.Debugf("receive reply-id[%v]: [%v], error:[%v]", id, reply, err) + log.Debugf("dbSyncer[%v] receive reply-id[%v]: [%v], error:[%v]", ds.id, id, reply, err) if conf.Options.Metric == false { continue @@ -601,7 +601,7 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type metric.GetMetric(ds.id).AddPullCmdCount(ds.id, 1) // print debug log of send command - if conf.Options.LogLevel == utils.LogLevelDebug { + if conf.Options.LogLevel == utils.LogLevelDebug || conf.Options.LogLevel == utils.LogLevelAll { strArgv := make([]string, len(argv)) for i, ele := range argv { strArgv[i] = *(*string)(unsafe.Pointer(&ele)) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index 744c766..d3a0e47 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "TCoDN4L8oq/S41a1VR2iONqN+mE=", + "checksumSHA1": "Y95c1YwyU+rB+mFMSB4zhxCpt+Q=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "6e304ef1b7fc2db6f913570112aa2a2c4d80ef81", - "revisionTime": "2019-11-20T07:29:33Z" + "revision": "f33d5a7283f2dc28233aa0069d7ec1798323cdad", + "revisionTime": "2019-11-28T04:11:03Z" }, { "checksumSHA1": "U4rR1I0MXcvJz3zSxTp3hb3Y0I0=", From 386a8c4684b0a74e789757a889fee87981de3d9f Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 28 Nov 2019 14:47:50 +0800 Subject: [PATCH 7/9] modify connect timeout and add keepAlive in cluster connection --- src/redis-shake/common/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index d102ec3..96af6ef 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -41,11 +41,11 @@ func OpenRedisConnWithTimeout(target []string, auth_type, passwd string, readTim cluster, err := redigoCluster.NewCluster( &redigoCluster.Options{ StartNodes: target, - ConnTimeout: 1 * time.Second, + ConnTimeout: 10 * time.Second, ReadTimeout: readTimeout, WriteTimeout: writeTimeout, KeepAlive: 16, - AliveTime: 60 * time.Second, + AliveTime: time.Duration(conf.Options.KeepAlive) * time.Second, Password: passwd, }) if err != nil { From f2c5a1d13de8755ac9e3cd390890ca754230e556 Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 28 Nov 2019 14:48:53 +0800 Subject: [PATCH 8/9] release v1.6.23 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 52d0c77..1a4130b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ script. * BUGFIX: update redis-go-clister driver to remove the error when meets nil bluk string. + * IMPROVE: add keep_alive in cluster connection. 2019-11-25 Alibaba Cloud. * VERSION: 1.6.22 * BUGFIX: solve MOVED error when key is unicode encoding which is not From b40bd294be9892fd9319f88b6b7e7ea978b2cb62 Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 28 Nov 2019 15:17:39 +0800 Subject: [PATCH 9/9] update connTimeout to 30s in cluster --- README.md | 2 +- src/redis-shake/common/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a44c6b2..fb73a8f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Supports `Standalone`, `Cluster`, and some proxies type like `Codis`, `twemproxy For `codis` and `twemproxy`, there maybe some constraints, please checkout this [question](https://github.com/alibaba/RedisShake/wiki/FAQ#q-does-redisshake-supports-codis-and-twemproxy). # Configuration -Redis-shake has several parameters in the configuration(`conf/redis-shake.conf`) that maybe confusing, if this is your first time using, just configure the `source.address` and `target.address` parameters. +Redis-shake has several parameters in the configuration(`conf/redis-shake.conf`) that maybe confusing, if this is your first time using, please visit this [tutorial](https://github.com/alibaba/RedisShake/wiki/tutorial-about-how-to-set-up). # Verification --- diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index 96af6ef..09ae7ef 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -41,7 +41,7 @@ func OpenRedisConnWithTimeout(target []string, auth_type, passwd string, readTim cluster, err := redigoCluster.NewCluster( &redigoCluster.Options{ StartNodes: target, - ConnTimeout: 10 * time.Second, + ConnTimeout: 30 * time.Second, ReadTimeout: readTimeout, WriteTimeout: writeTimeout, KeepAlive: 16,