From 0490facfe81d11fd47bd91e1a537a0cd4c0592fd Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:55:30 +0800 Subject: [PATCH 01/33] polish README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7ae280e..3c5cc3f 100644 --- a/README.md +++ b/README.md @@ -98,3 +98,4 @@ Plus, we have a WeChat group so that users can join and discuss, but the group u | :------: | :------: | | ceshihao | davidzheng23@gmail.com | | wangyiyang | wangyiyang.kk@gmail.com | +| muicoder | muicoder@gmail.com | From 5589fa33de6d4d4ba0cf3477facd6881a1e6d9f8 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 14:40:10 +0800 Subject: [PATCH 02/33] test circleci --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ .gitignore | 1 + 2 files changed, 27 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..a2328c0 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,26 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: + docker: + # specify the version + - image: circleci/golang:1.9 + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + # - image: circleci/postgres:9.4 + + #### TEMPLATE_NOTE: go expects specific checkout path representing url + #### expecting it in the form of + #### /go/src/github.com/circleci/go-tool + #### /go/src/bitbucket.org/circleci/go-tool + working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} + steps: + - checkout + + # specify any bash command here prefixed with `run: ` + - run: go get -v -t -d ./... + - run: go test -v ./... diff --git a/.gitignore b/.gitignore index 40c2ac9..0d3ceb7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ result.db.* bin/* conf/* !conf/redis-shake.conf +!.circleci/config.yml dump.data runtime.trace From a7ce7d58b9056a8e2a63f369fe34eaee5c027185 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:03:53 +0800 Subject: [PATCH 03/33] test circleci2 --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a2328c0..628d4de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: build: docker: # specify the version - - image: circleci/golang:1.9 + - image: circleci/golang:1.10 # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images @@ -22,5 +22,4 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` - - run: go get -v -t -d ./... - - run: go test -v ./... + - run: sh build.sh From 20f5df5432fa0ebc9f943b16384f222d3256baee Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:07:25 +0800 Subject: [PATCH 04/33] test circleci3 --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 628d4de..265654c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,4 +22,8 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` - - run: sh build.sh + - run: export GOPATH=`pwd` + - run: go get -u github.com/kardianos/govendor + - run: cd src/vendor + - run: govendor sync + - run: cd ../../ && sh build.sh From 9857c8f7c4ffc6d4549f62b410fc059135751fbc Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:13:05 +0800 Subject: [PATCH 05/33] test circleci4 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 265654c..fdcb8e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,6 @@ jobs: # specify any bash command here prefixed with `run: ` - run: export GOPATH=`pwd` - run: go get -u github.com/kardianos/govendor - - run: cd src/vendor + - run: cd src/vendor && ls - run: govendor sync - run: cd ../../ && sh build.sh From f86e93b35d08d362827777874e33a8e5ee1e8828 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:15:34 +0800 Subject: [PATCH 06/33] test circleci5 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fdcb8e5..d217f1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,6 @@ jobs: # specify any bash command here prefixed with `run: ` - run: export GOPATH=`pwd` - run: go get -u github.com/kardianos/govendor - - run: cd src/vendor && ls + - run: cd src && ls - run: govendor sync - run: cd ../../ && sh build.sh From 4d97c4e00c11cd26eacecddc8b10d58f393bf531 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:41:11 +0800 Subject: [PATCH 07/33] test circleci6 --- .circleci/config.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d217f1f..ff6c48a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,13 +17,16 @@ jobs: #### expecting it in the form of #### /go/src/github.com/circleci/go-tool #### /go/src/bitbucket.org/circleci/go-tool - working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} + working_directory: /go/src/github.com/alibaba/RedisShake steps: - checkout # specify any bash command here prefixed with `run: ` - - run: export GOPATH=`pwd` - - run: go get -u github.com/kardianos/govendor - - run: cd src && ls - - run: govendor sync - - run: cd ../../ && sh build.sh + - run: + name: prepare environment + command: | + export GOPATH=`pwd` + go get -u github.com/kardianos/govendor + cd src && tree + govendor sync -v + cd .. && sh build.sh From a56163a9716b992c0441cd4d19cbeea8100a5655 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:43:49 +0800 Subject: [PATCH 08/33] test circleci7 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff6c48a..5982986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,8 +25,8 @@ jobs: - run: name: prepare environment command: | - export GOPATH=`pwd` - go get -u github.com/kardianos/govendor - cd src && tree - govendor sync -v - cd .. && sh build.sh + export GOPATH=`pwd` + go get -u github.com/kardianos/govendor + cd src && tree + govendor sync -v + cd .. && sh build.sh From 8a7eaba41aeccd74f3fad6d2de0c9f42aa9df8dd Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:47:52 +0800 Subject: [PATCH 09/33] test circleci8 --- .circleci/config.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5982986..84538dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,11 +22,8 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` - - run: - name: prepare environment - command: | - export GOPATH=`pwd` - go get -u github.com/kardianos/govendor - cd src && tree - govendor sync -v - cd .. && sh build.sh + - run: export GOPATH=`pwd` + - run: go get -u github.com/kardianos/govendor + - run: cd src && tree + - run: govendor sync -v + - run: cd .. && sh build.sh From f8a2c6be5318b0b4fe003852a25c4e80451e9ca9 Mon Sep 17 00:00:00 2001 From: vinllen Date: Sat, 10 Aug 2019 15:49:59 +0800 Subject: [PATCH 10/33] test circleci9 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84538dc..d926c18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,6 @@ jobs: # specify any bash command here prefixed with `run: ` - run: export GOPATH=`pwd` - run: go get -u github.com/kardianos/govendor - - run: cd src && tree + - run: cd src && ls && ls vendor - run: govendor sync -v - run: cd .. && sh build.sh From 4c96006d2ec4e0c4d17190e4ad50ff22519aa783 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:34:32 +0800 Subject: [PATCH 11/33] transaction syncing panic when target redis is cluster. see #145 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 68fe4a1..8fb97d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2019-08-12 Alibaba Cloud. + * VERSION: 1.6.17 + * BUGFIX: transaction syncing panic when target redis is cluster. see + #145. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. From ded322871d4faedde883b889efc60d9e9d157ad5 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:52:45 +0800 Subject: [PATCH 12/33] set sender count to 4096 if target redis type is cluster --- conf/redis-shake.conf | 2 +- src/redis-shake/main/main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 08b30dc..281578f 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -189,7 +189,7 @@ sender.size = 104857600 # sender flush buffer size of oplog number. # used in `sync`. # 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 -sender.count = 5000 +sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving # ack from target redis. diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 3ee087c..a6fa959 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,6 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } + if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { + conf.Options.SenderCount = utils.RecvChanSize + } if conf.Options.SenderDelayChannelSize == 0 { conf.Options.SenderDelayChannelSize = 32 From 89f99c839a54820f29706c4ffe207d572c808e96 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:57:22 +0800 Subject: [PATCH 13/33] set max sender.count to 4096 if target redis type is cluster --- ChangeLog | 1 + conf/redis-shake.conf | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8fb97d5..646cb1a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. + * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 281578f..1214700 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -187,8 +187,9 @@ heartbeat.network_interface = # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 +# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, +# the maximum value is 4096. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving From e040724afaacc72023c8a7621e7d53d01c2747e7 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 18:31:39 +0800 Subject: [PATCH 14/33] add more debug in rdb restore --- src/redis-shake/common/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index b429299..6fb9b35 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -811,6 +811,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { params = append(params, "FREQ") params = append(params, e.Freq) } + + log.Debugf("restore key[%s] with params[%v]", e.Key, params) // fmt.Printf("key: %v, value: %v params: %v\n", string(e.Key), e.Value, params) // s, err := redigo.String(c.Do("restore", params...)) s, err := redigoCluster.String(c.Do("restore", params...)) From bece38f2ca10c1b63b4e606eb4d847bd117710c1 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 19:19:47 +0800 Subject: [PATCH 15/33] adjust RecvChanSize based on 'sender.count' or 'scan.key_number' if target redis type is cluster --- ChangeLog | 3 ++- src/redis-shake/common/cluster.go | 2 +- src/redis-shake/main/main.go | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 646cb1a..ea8ffd7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,7 +2,8 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. - * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. + * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` + if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/src/redis-shake/common/cluster.go b/src/redis-shake/common/cluster.go index 8bf9df5..2abb1e7 100644 --- a/src/redis-shake/common/cluster.go +++ b/src/redis-shake/common/cluster.go @@ -7,7 +7,7 @@ import ( "pkg/libs/log" ) -const( +var ( RecvChanSize = 4096 ) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index a6fa959..ea58fc1 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,8 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } - if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { - conf.Options.SenderCount = utils.RecvChanSize + if conf.Options.TargetType == conf.RedisTypeCluster && int(conf.Options.SenderCount) > utils.RecvChanSize { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.SenderCount)) + utils.RecvChanSize = int(conf.Options.SenderCount) } if conf.Options.SenderDelayChannelSize == 0 { @@ -451,8 +452,9 @@ func sanitizeOptions(tp string) error { conf.Options.ScanSpecialCloud, conf.Options.ScanKeyFile) } - if conf.Options.ScanKeyNumber > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { - return fmt.Errorf("scan.key_number should less than [%v] when target type is cluster", utils.RecvChanSize) + if int(conf.Options.ScanKeyNumber) > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.ScanKeyNumber)) + utils.RecvChanSize = int(conf.Options.ScanKeyNumber) } //if len(conf.Options.SourceAddressList) == 1 { From da69c052cd22d1d18cde176838bb4066aaf90cec Mon Sep 17 00:00:00 2001 From: zhangwei <26432832+arugal@users.noreply.github.com> Date: Wed, 14 Aug 2019 10:51:59 +0800 Subject: [PATCH 16/33] fix wrong character --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4292327..d7cf427 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The type can be one of the followings:
* **decode**: Decode dumped payload to human readable format (hex-encoding). * **restore**: Restore RDB file to target redis. -* **dump**: Dump RDB file from souce redis. +* **dump**: Dump RDB file from source redis. * **sync**: Sync data from source redis to target redis by `sync` or `psync` command. Including full synchronization and incremental synchronization. * **rump**: Sync data from source redis to target redis by `scan` command. Only support full synchronization. Plus, RedisShake also supports fetching data from given keys in the input file when `scan` command is not supported on the source side. This mode is usually used when `sync` and `psync` redis commands aren't supported. From 0fccb31eff9f1230270a23a818d2b21f03985c67 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:05 +0800 Subject: [PATCH 17/33] remove some useless variables in conf --- conf/redis-shake.conf | 30 +------ src/redis-shake/configure/configure.go | 108 ++++++++++++------------- 2 files changed, 57 insertions(+), 81 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 1214700..0d941a4 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -19,9 +19,6 @@ system_profile = 9310 # restful port，查看metric端口, -1表示不启用，如果是`restore`模式，只有设置为-1才会在完成RDB恢复后退出，否则会一直block。 http_profile = 9320 -# runtime.GOMAXPROCS, 0 means use cpu core number: runtime.NumCPU() -ncpu = 0 - # parallel routines number used in RDB file syncing. default is 64. # 启动多少个并发线程同步一个RDB文件。 parallel = 32 @@ -165,31 +162,15 @@ metric = true # 是否将metric打印到log中 metric.print_log = false -# heartbeat -# send heartbeat to this url -# used in `sync`. -# 心跳的url地址，redis-shake将会发送到这个地址 -#heartbeat.url = http://127.0.0.1:8000 -heartbeat.url = -# interval by seconds -# 心跳保活周期 -heartbeat.interval = 3 -# external info which will be included in heartbeat data. -# 在心跳报文中添加额外的信息 -heartbeat.external = test external -# local network card to get ip address, e.g., "lo", "eth0", "en0" -# 获取ip的网卡 -heartbeat.network_interface = - # sender information. # sender flush buffer size of byte. # used in `sync`. # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, -# the maximum value is 4096. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 +# used in `sync`. flush sender buffer when bigger than this threshold. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，对于目的端是cluster的情况，这个值 +# 的调大将会占用部分内存。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving @@ -208,13 +189,11 @@ keep_alive = 0 # number of keys captured each time. default is 100. # 每次scan的个数，不配置则默认100. scan.key_number = 50 - # used in `rump`. # we support some special redis types that don't use default `scan` command like alibaba cloud and tencent cloud. # 有些版本具有特殊的格式，与普通的scan命令有所不同，我们进行了特殊的适配。目前支持腾讯云的集群版"tencent_cluster" # 和阿里云的集群版"aliyun_cluster"。 scan.special_cloud = - # used in `rump`. # we support to fetching data from given file which marks the key list. # 有些云版本，既不支持sync/psync，也不支持scan，我们支持从文件中进行读取所有key列表并进行抓取：一行一个key。 @@ -230,6 +209,3 @@ qps = 200000 # replace hash tag. # used in `sync`. replace_hash_tag = false - -# used in `restore` and `dump`. -extra = false diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index 5a8cc85..ec5343d 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -4,65 +4,65 @@ import "time" type Configuration struct { // config file variables - Id string `config:"id"` - LogFile string `config:"log.file"` - LogLevel string `config:"log.level"` - SystemProfile int `config:"system_profile"` - HttpProfile int `config:"http_profile"` + Id string `config:"id"` + LogFile string `config:"log.file"` + LogLevel string `config:"log.level"` + SystemProfile int `config:"system_profile"` + HttpProfile int `config:"http_profile"` + Parallel int `config:"parallel"` + SourceType string `config:"source.type"` + SourceAddress string `config:"source.address"` + SourcePasswordRaw string `config:"source.password_raw"` + SourcePasswordEncoding string `config:"source.password_encoding"` + SourceAuthType string `config:"source.auth_type"` + SourceTLSEnable bool `config:"source.tls_enable"` + SourceRdbInput []string `config:"source.rdb.input"` + SourceRdbParallel int `config:"source.rdb.parallel"` + SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` + TargetAddress string `config:"target.address"` + TargetPasswordRaw string `config:"target.password_raw"` + TargetPasswordEncoding string `config:"target.password_encoding"` + TargetDBString string `config:"target.db"` + TargetAuthType string `config:"target.auth_type"` + TargetType string `config:"target.type"` + TargetTLSEnable bool `config:"target.tls_enable"` + TargetRdbOutput string `config:"target.rdb.output"` + TargetVersion string `config:"target.version"` + FakeTime string `config:"fake_time"` + Rewrite bool `config:"rewrite"` + FilterDBWhitelist []string `config:"filter.db.whitelist"` + FilterDBBlacklist []string `config:"filter.db.blacklist"` + FilterKeyWhitelist []string `config:"filter.key.whitelist"` + FilterKeyBlacklist []string `config:"filter.key.blacklist"` + FilterSlot []string `config:"filter.slot"` + FilterLua bool `config:"filter.lua"` + BigKeyThreshold uint64 `config:"big_key_threshold"` + Psync bool `config:"psync"` + Metric bool `config:"metric"` + MetricPrintLog bool `config:"metric.print_log"` + SenderSize uint64 `config:"sender.size"` + SenderCount uint `config:"sender.count"` + SenderDelayChannelSize uint `config:"sender.delay_channel_size"` + KeepAlive uint `config:"keep_alive"` + PidPath string `config:"pid_path"` + ScanKeyNumber uint32 `config:"scan.key_number"` + ScanSpecialCloud string `config:"scan.special_cloud"` + ScanKeyFile string `config:"scan.key_file"` + Qps int `config:"qps"` + + /*---------------------------------------------------------*/ + // inner variables NCpu int `config:"ncpu"` - Parallel int `config:"parallel"` - SourceType string `config:"source.type"` - SourceAddress string `config:"source.address"` - SourcePasswordRaw string `config:"source.password_raw"` - SourcePasswordEncoding string `config:"source.password_encoding"` - SourceAuthType string `config:"source.auth_type"` - SourceTLSEnable bool `config:"source.tls_enable"` - SourceRdbInput []string `config:"source.rdb.input"` - SourceRdbParallel int `config:"source.rdb.parallel"` - SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` - TargetAddress string `config:"target.address"` - TargetPasswordRaw string `config:"target.password_raw"` - TargetPasswordEncoding string `config:"target.password_encoding"` - TargetDBString string `config:"target.db"` - TargetAuthType string `config:"target.auth_type"` - TargetType string `config:"target.type"` - TargetTLSEnable bool `config:"target.tls_enable"` - TargetRdbOutput string `config:"target.rdb.output"` - TargetVersion string `config:"target.version"` - FakeTime string `config:"fake_time"` - Rewrite bool `config:"rewrite"` - FilterDBWhitelist []string `config:"filter.db.whitelist"` - FilterDBBlacklist []string `config:"filter.db.blacklist"` - FilterKeyWhitelist []string `config:"filter.key.whitelist"` - FilterKeyBlacklist []string `config:"filter.key.blacklist"` - FilterSlot []string `config:"filter.slot"` - FilterLua bool `config:"filter.lua"` - BigKeyThreshold uint64 `config:"big_key_threshold"` - Psync bool `config:"psync"` - Metric bool `config:"metric"` - MetricPrintLog bool `config:"metric.print_log"` HeartbeatUrl string `config:"heartbeat.url"` HeartbeatInterval uint `config:"heartbeat.interval"` HeartbeatExternal string `config:"heartbeat.external"` HeartbeatNetworkInterface string `config:"heartbeat.network_interface"` - SenderSize uint64 `config:"sender.size"` - SenderCount uint `config:"sender.count"` - SenderDelayChannelSize uint `config:"sender.delay_channel_size"` - KeepAlive uint `config:"keep_alive"` - PidPath string `config:"pid_path"` - ScanKeyNumber uint32 `config:"scan.key_number"` - ScanSpecialCloud string `config:"scan.special_cloud"` - ScanKeyFile string `config:"scan.key_file"` - Qps int `config:"qps"` - - /*---------------------------------------------------------*/ - // inner variables - ReplaceHashTag bool `config:"replace_hash_tag"` - ExtraInfo bool `config:"extra"` - SockFileName string `config:"sock.file_name"` - SockFileSize uint `config:"sock.file_size"` - FilterKey []string `config:"filter.key"` // compatible with older versions - FilterDB string `config:"filter.db"` // compatible with older versions + ReplaceHashTag bool `config:"replace_hash_tag"` + ExtraInfo bool `config:"extra"` + SockFileName string `config:"sock.file_name"` + SockFileSize uint `config:"sock.file_size"` + FilterKey []string `config:"filter.key"` // compatible with older versions + FilterDB string `config:"filter.db"` // compatible with older versions /*---------------------------------------------------------*/ // generated variables From 5bd145b229ca3062944dd8b1c8ee2faa7c94d9cf Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:50 +0800 Subject: [PATCH 18/33] remove some useless variables in conf2 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ea8ffd7..44629c8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ #145. * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` if target redis type is cluster. + * IMPROVE: remove some useless variables in conf like `heartbeat`, `ncpu`. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. From b83be9b3467d007d11279831d1c7237c5eabf331 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:26:03 +0800 Subject: [PATCH 19/33] update redis-go-cluster driver --- src/vendor/vendor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index f382246..1760128 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "IjUkzECf826IH8K1bexM/uockIk=", + "checksumSHA1": "1PrwQi6VvhLG4ovw1B7zD/afGZ4=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "ffc34c0f905b65e6b71e07fd68a1d50b435c60f5", - "revisionTime": "2019-07-04T07:53:38Z" + "revision": "0799101ddfdb7d2bc9d47948f8a0cc17d23a216e", + "revisionTime": "2019-08-12T11:22:58Z" }, { "checksumSHA1": "TM3Neoy1xRAKyZYMGzKc41sDFW4=", From 09d5211536f0a7dfccdfa486f5095a6db558b2a5 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:34:32 +0800 Subject: [PATCH 20/33] transaction syncing panic when target redis is cluster. see #145 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 68fe4a1..8fb97d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2019-08-12 Alibaba Cloud. + * VERSION: 1.6.17 + * BUGFIX: transaction syncing panic when target redis is cluster. see + #145. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. From 24e8c4f712f95d7586b87bfaef48658a17653414 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:52:45 +0800 Subject: [PATCH 21/33] set sender count to 4096 if target redis type is cluster --- conf/redis-shake.conf | 2 +- src/redis-shake/main/main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 08b30dc..281578f 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -189,7 +189,7 @@ sender.size = 104857600 # sender flush buffer size of oplog number. # used in `sync`. # 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 -sender.count = 5000 +sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving # ack from target redis. diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 3ee087c..a6fa959 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,6 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } + if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { + conf.Options.SenderCount = utils.RecvChanSize + } if conf.Options.SenderDelayChannelSize == 0 { conf.Options.SenderDelayChannelSize = 32 From 2ef34ac60509ff3d1aeaad3c02492c0b741dda61 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:57:22 +0800 Subject: [PATCH 22/33] set max sender.count to 4096 if target redis type is cluster --- ChangeLog | 1 + conf/redis-shake.conf | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8fb97d5..646cb1a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. + * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 281578f..1214700 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -187,8 +187,9 @@ heartbeat.network_interface = # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 +# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, +# the maximum value is 4096. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving From ffde9f65b5cecf5ea8c60f6849728eb3e14b88e3 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 18:31:39 +0800 Subject: [PATCH 23/33] add more debug in rdb restore --- src/redis-shake/common/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index b429299..6fb9b35 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -811,6 +811,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { params = append(params, "FREQ") params = append(params, e.Freq) } + + log.Debugf("restore key[%s] with params[%v]", e.Key, params) // fmt.Printf("key: %v, value: %v params: %v\n", string(e.Key), e.Value, params) // s, err := redigo.String(c.Do("restore", params...)) s, err := redigoCluster.String(c.Do("restore", params...)) From e2153a12a22d12e1b64be1ccff878e0803616cc8 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 19:19:47 +0800 Subject: [PATCH 24/33] adjust RecvChanSize based on 'sender.count' or 'scan.key_number' if target redis type is cluster --- ChangeLog | 3 ++- src/redis-shake/common/cluster.go | 2 +- src/redis-shake/main/main.go | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 646cb1a..ea8ffd7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,7 +2,8 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. - * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. + * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` + if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/src/redis-shake/common/cluster.go b/src/redis-shake/common/cluster.go index 8bf9df5..2abb1e7 100644 --- a/src/redis-shake/common/cluster.go +++ b/src/redis-shake/common/cluster.go @@ -7,7 +7,7 @@ import ( "pkg/libs/log" ) -const( +var ( RecvChanSize = 4096 ) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index a6fa959..ea58fc1 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,8 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } - if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { - conf.Options.SenderCount = utils.RecvChanSize + if conf.Options.TargetType == conf.RedisTypeCluster && int(conf.Options.SenderCount) > utils.RecvChanSize { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.SenderCount)) + utils.RecvChanSize = int(conf.Options.SenderCount) } if conf.Options.SenderDelayChannelSize == 0 { @@ -451,8 +452,9 @@ func sanitizeOptions(tp string) error { conf.Options.ScanSpecialCloud, conf.Options.ScanKeyFile) } - if conf.Options.ScanKeyNumber > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { - return fmt.Errorf("scan.key_number should less than [%v] when target type is cluster", utils.RecvChanSize) + if int(conf.Options.ScanKeyNumber) > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.ScanKeyNumber)) + utils.RecvChanSize = int(conf.Options.ScanKeyNumber) } //if len(conf.Options.SourceAddressList) == 1 { From 7ef07a0e69a0fa1d2267a508ac2797cd8e66a8a6 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:05 +0800 Subject: [PATCH 25/33] remove some useless variables in conf --- conf/redis-shake.conf | 30 +------ src/redis-shake/configure/configure.go | 108 ++++++++++++------------- 2 files changed, 57 insertions(+), 81 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 1214700..0d941a4 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -19,9 +19,6 @@ system_profile = 9310 # restful port，查看metric端口, -1表示不启用，如果是`restore`模式，只有设置为-1才会在完成RDB恢复后退出，否则会一直block。 http_profile = 9320 -# runtime.GOMAXPROCS, 0 means use cpu core number: runtime.NumCPU() -ncpu = 0 - # parallel routines number used in RDB file syncing. default is 64. # 启动多少个并发线程同步一个RDB文件。 parallel = 32 @@ -165,31 +162,15 @@ metric = true # 是否将metric打印到log中 metric.print_log = false -# heartbeat -# send heartbeat to this url -# used in `sync`. -# 心跳的url地址，redis-shake将会发送到这个地址 -#heartbeat.url = http://127.0.0.1:8000 -heartbeat.url = -# interval by seconds -# 心跳保活周期 -heartbeat.interval = 3 -# external info which will be included in heartbeat data. -# 在心跳报文中添加额外的信息 -heartbeat.external = test external -# local network card to get ip address, e.g., "lo", "eth0", "en0" -# 获取ip的网卡 -heartbeat.network_interface = - # sender information. # sender flush buffer size of byte. # used in `sync`. # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, -# the maximum value is 4096. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 +# used in `sync`. flush sender buffer when bigger than this threshold. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，对于目的端是cluster的情况，这个值 +# 的调大将会占用部分内存。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving @@ -208,13 +189,11 @@ keep_alive = 0 # number of keys captured each time. default is 100. # 每次scan的个数，不配置则默认100. scan.key_number = 50 - # used in `rump`. # we support some special redis types that don't use default `scan` command like alibaba cloud and tencent cloud. # 有些版本具有特殊的格式，与普通的scan命令有所不同，我们进行了特殊的适配。目前支持腾讯云的集群版"tencent_cluster" # 和阿里云的集群版"aliyun_cluster"。 scan.special_cloud = - # used in `rump`. # we support to fetching data from given file which marks the key list. # 有些云版本，既不支持sync/psync，也不支持scan，我们支持从文件中进行读取所有key列表并进行抓取：一行一个key。 @@ -230,6 +209,3 @@ qps = 200000 # replace hash tag. # used in `sync`. replace_hash_tag = false - -# used in `restore` and `dump`. -extra = false diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index 5a8cc85..ec5343d 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -4,65 +4,65 @@ import "time" type Configuration struct { // config file variables - Id string `config:"id"` - LogFile string `config:"log.file"` - LogLevel string `config:"log.level"` - SystemProfile int `config:"system_profile"` - HttpProfile int `config:"http_profile"` + Id string `config:"id"` + LogFile string `config:"log.file"` + LogLevel string `config:"log.level"` + SystemProfile int `config:"system_profile"` + HttpProfile int `config:"http_profile"` + Parallel int `config:"parallel"` + SourceType string `config:"source.type"` + SourceAddress string `config:"source.address"` + SourcePasswordRaw string `config:"source.password_raw"` + SourcePasswordEncoding string `config:"source.password_encoding"` + SourceAuthType string `config:"source.auth_type"` + SourceTLSEnable bool `config:"source.tls_enable"` + SourceRdbInput []string `config:"source.rdb.input"` + SourceRdbParallel int `config:"source.rdb.parallel"` + SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` + TargetAddress string `config:"target.address"` + TargetPasswordRaw string `config:"target.password_raw"` + TargetPasswordEncoding string `config:"target.password_encoding"` + TargetDBString string `config:"target.db"` + TargetAuthType string `config:"target.auth_type"` + TargetType string `config:"target.type"` + TargetTLSEnable bool `config:"target.tls_enable"` + TargetRdbOutput string `config:"target.rdb.output"` + TargetVersion string `config:"target.version"` + FakeTime string `config:"fake_time"` + Rewrite bool `config:"rewrite"` + FilterDBWhitelist []string `config:"filter.db.whitelist"` + FilterDBBlacklist []string `config:"filter.db.blacklist"` + FilterKeyWhitelist []string `config:"filter.key.whitelist"` + FilterKeyBlacklist []string `config:"filter.key.blacklist"` + FilterSlot []string `config:"filter.slot"` + FilterLua bool `config:"filter.lua"` + BigKeyThreshold uint64 `config:"big_key_threshold"` + Psync bool `config:"psync"` + Metric bool `config:"metric"` + MetricPrintLog bool `config:"metric.print_log"` + SenderSize uint64 `config:"sender.size"` + SenderCount uint `config:"sender.count"` + SenderDelayChannelSize uint `config:"sender.delay_channel_size"` + KeepAlive uint `config:"keep_alive"` + PidPath string `config:"pid_path"` + ScanKeyNumber uint32 `config:"scan.key_number"` + ScanSpecialCloud string `config:"scan.special_cloud"` + ScanKeyFile string `config:"scan.key_file"` + Qps int `config:"qps"` + + /*---------------------------------------------------------*/ + // inner variables NCpu int `config:"ncpu"` - Parallel int `config:"parallel"` - SourceType string `config:"source.type"` - SourceAddress string `config:"source.address"` - SourcePasswordRaw string `config:"source.password_raw"` - SourcePasswordEncoding string `config:"source.password_encoding"` - SourceAuthType string `config:"source.auth_type"` - SourceTLSEnable bool `config:"source.tls_enable"` - SourceRdbInput []string `config:"source.rdb.input"` - SourceRdbParallel int `config:"source.rdb.parallel"` - SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` - TargetAddress string `config:"target.address"` - TargetPasswordRaw string `config:"target.password_raw"` - TargetPasswordEncoding string `config:"target.password_encoding"` - TargetDBString string `config:"target.db"` - TargetAuthType string `config:"target.auth_type"` - TargetType string `config:"target.type"` - TargetTLSEnable bool `config:"target.tls_enable"` - TargetRdbOutput string `config:"target.rdb.output"` - TargetVersion string `config:"target.version"` - FakeTime string `config:"fake_time"` - Rewrite bool `config:"rewrite"` - FilterDBWhitelist []string `config:"filter.db.whitelist"` - FilterDBBlacklist []string `config:"filter.db.blacklist"` - FilterKeyWhitelist []string `config:"filter.key.whitelist"` - FilterKeyBlacklist []string `config:"filter.key.blacklist"` - FilterSlot []string `config:"filter.slot"` - FilterLua bool `config:"filter.lua"` - BigKeyThreshold uint64 `config:"big_key_threshold"` - Psync bool `config:"psync"` - Metric bool `config:"metric"` - MetricPrintLog bool `config:"metric.print_log"` HeartbeatUrl string `config:"heartbeat.url"` HeartbeatInterval uint `config:"heartbeat.interval"` HeartbeatExternal string `config:"heartbeat.external"` HeartbeatNetworkInterface string `config:"heartbeat.network_interface"` - SenderSize uint64 `config:"sender.size"` - SenderCount uint `config:"sender.count"` - SenderDelayChannelSize uint `config:"sender.delay_channel_size"` - KeepAlive uint `config:"keep_alive"` - PidPath string `config:"pid_path"` - ScanKeyNumber uint32 `config:"scan.key_number"` - ScanSpecialCloud string `config:"scan.special_cloud"` - ScanKeyFile string `config:"scan.key_file"` - Qps int `config:"qps"` - - /*---------------------------------------------------------*/ - // inner variables - ReplaceHashTag bool `config:"replace_hash_tag"` - ExtraInfo bool `config:"extra"` - SockFileName string `config:"sock.file_name"` - SockFileSize uint `config:"sock.file_size"` - FilterKey []string `config:"filter.key"` // compatible with older versions - FilterDB string `config:"filter.db"` // compatible with older versions + ReplaceHashTag bool `config:"replace_hash_tag"` + ExtraInfo bool `config:"extra"` + SockFileName string `config:"sock.file_name"` + SockFileSize uint `config:"sock.file_size"` + FilterKey []string `config:"filter.key"` // compatible with older versions + FilterDB string `config:"filter.db"` // compatible with older versions /*---------------------------------------------------------*/ // generated variables From 97d7543146d2031101198d291f848976b24742b5 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:50 +0800 Subject: [PATCH 26/33] remove some useless variables in conf2 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ea8ffd7..44629c8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ #145. * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` if target redis type is cluster. + * IMPROVE: remove some useless variables in conf like `heartbeat`, `ncpu`. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. From 40426643aeace9444e772fe3a496a416b557c4da Mon Sep 17 00:00:00 2001 From: vinllen Date: Fri, 16 Aug 2019 19:58:15 +0800 Subject: [PATCH 27/33] print inner error message from redigo driver return message --- ChangeLog | 1 + src/redis-shake/common/command.go | 19 +++++++++++++++++++ src/redis-shake/rump.go | 9 +++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 44629c8..c96a717 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` if target redis type is cluster. * IMPROVE: remove some useless variables in conf like `heartbeat`, `ncpu`. + * IMPROVE: print inner error message from redigo driver return message. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/src/redis-shake/common/command.go b/src/redis-shake/common/command.go index d7b7ab1..97ef810 100644 --- a/src/redis-shake/common/command.go +++ b/src/redis-shake/common/command.go @@ -10,6 +10,11 @@ import ( "strings" ) +const ( + ReplayString = "string" + ReplayInt64s = "int64s" +) + type ClusterNodeInfo struct { Id string Address string @@ -172,4 +177,18 @@ func GetAllClusterNode(client redigo.Conn, role string, choose string) ([]string } return result, nil +} + +func WrapCommand(tp string, reply interface{}, err error) (interface{}, error) { + if err != nil { + return nil, fmt.Errorf("inner error: %v", err) + } + switch tp { + case ReplayString: + return redigo.Strings(reply, err) + case ReplayInt64s: + return redigo.Int64s(reply, err) + default: + return nil, fmt.Errorf("command type[%v] not support", tp) + } } \ No newline at end of file diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index 247ec64..ce28efb 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -483,19 +483,24 @@ func (dre *dbRumperExecutor) doFetch(db int) error { log.Debugf("dbRumper[%v] executor[%v] scan key: %v", dre.rumperId, dre.executorId, key) dre.sourceClient.Send("DUMP", key) } - dumps, err := redis.Strings(dre.sourceClient.Do("")) + + reply, err := dre.sourceClient.Do("") + dumpsRet, err := utils.WrapCommand(utils.ReplayString, reply, err) if err != nil && err != redis.ErrNil { return fmt.Errorf("do dump with failed[%v]", err) } + dumps := dumpsRet.([]string) // pipeline ttl for _, key := range keys { dre.sourceClient.Send("PTTL", key) } - pttls, err := redis.Int64s(dre.sourceClient.Do("")) + reply, err = dre.sourceClient.Do("") + pttlsRet, err := utils.WrapCommand(utils.ReplayInt64s, reply, err) if err != nil && err != redis.ErrNil { return fmt.Errorf("do ttl with failed[%v]", err) } + pttls := pttlsRet.([]int64) dre.stat.rCommands.Add(int64(len(keys))) for i, k := range keys { From fadbdba6bd5f60dc2398c6cd12dfa1f3164be127 Mon Sep 17 00:00:00 2001 From: vinllen Date: Fri, 16 Aug 2019 20:10:16 +0800 Subject: [PATCH 28/33] print inner error message from redigo driver return message2 --- src/redis-shake/common/command.go | 14 -------------- src/redis-shake/rump.go | 10 ++++------ 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/redis-shake/common/command.go b/src/redis-shake/common/command.go index 97ef810..678e95d 100644 --- a/src/redis-shake/common/command.go +++ b/src/redis-shake/common/command.go @@ -177,18 +177,4 @@ func GetAllClusterNode(client redigo.Conn, role string, choose string) ([]string } return result, nil -} - -func WrapCommand(tp string, reply interface{}, err error) (interface{}, error) { - if err != nil { - return nil, fmt.Errorf("inner error: %v", err) - } - switch tp { - case ReplayString: - return redigo.Strings(reply, err) - case ReplayInt64s: - return redigo.Int64s(reply, err) - default: - return nil, fmt.Errorf("command type[%v] not support", tp) - } } \ No newline at end of file diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index ce28efb..a921fcb 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -485,22 +485,20 @@ func (dre *dbRumperExecutor) doFetch(db int) error { } reply, err := dre.sourceClient.Do("") - dumpsRet, err := utils.WrapCommand(utils.ReplayString, reply, err) + dumps, err := redis.Strings(reply, err) if err != nil && err != redis.ErrNil { - return fmt.Errorf("do dump with failed[%v]", err) + return fmt.Errorf("do dump with failed[%v], reply[%v]", err, reply) } - dumps := dumpsRet.([]string) // pipeline ttl for _, key := range keys { dre.sourceClient.Send("PTTL", key) } reply, err = dre.sourceClient.Do("") - pttlsRet, err := utils.WrapCommand(utils.ReplayInt64s, reply, err) + pttls, err := redis.Int64s(reply, err) if err != nil && err != redis.ErrNil { - return fmt.Errorf("do ttl with failed[%v]", err) + return fmt.Errorf("do ttl with failed[%v], reply[%v]", err, reply) } - pttls := pttlsRet.([]int64) dre.stat.rCommands.Add(int64(len(keys))) for i, k := range keys { From 9a9eb7aa5b833a885393cbcdab6770f7f3d74764 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 19 Aug 2019 15:17:20 +0800 Subject: [PATCH 29/33] polish conf --- ChangeLog | 2 +- conf/redis-shake.conf | 3 ++- src/redis-shake/rump.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c96a717..a6159d6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ #145. * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` if target redis type is cluster. - * IMPROVE: remove some useless variables in conf like `heartbeat`, `ncpu`. + * IMPROVE: remove some variables in conf like `heartbeat`, `ncpu`. * IMPROVE: print inner error message from redigo driver return message. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 0d941a4..9a7f76e 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -38,7 +38,8 @@ source.type = standalone # 2. ${sentinel_master_name}:${master or slave}@sentinel single/cluster address, e.g., mymaster:master@127.0.0.1:26379;127.0.0.1:26380, or @127.0.0.1:26379;127.0.0.1:26380. for "sentinel" type. # 3. cluster that has several db nodes split by semicolon(;). for "cluster" type. e.g., 10.1.1.1:20331;10.1.1.2:20441. # 4. proxy address(used in "rump" mode only). for "proxy" type. -# 源redis地址。对于sentinel模式，输入格式为"master名字:拉取角色为master或者slave@sentinel的地址" +# 源redis地址。对于sentinel或者开源cluster模式，输入格式为"master名字:拉取角色为master或者slave@sentinel的地址"，别的cluster +# 架构，比如codis, twemproxy, aliyun proxy等需要配置所有master或者slave的db地址。 source.address = 127.0.0.1:20441 # password of db/proxy. even if type is sentinel. source.password_raw = 123456 diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index a921fcb..c6a0db4 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -13,9 +13,9 @@ import ( "redis-shake/configure" "redis-shake/metric" "redis-shake/scanner" + "redis-shake/filter" "github.com/garyburd/redigo/redis" - "redis-shake/filter" ) type CmdRump struct { From aee3c3fe840fa2661c32e5a75a2d6ef8dce50270 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 21 Aug 2019 11:21:37 +0800 Subject: [PATCH 30/33] debug zset panic --- src/pkg/rdb/loader.go | 6 +++--- src/pkg/rdb/reader.go | 3 +++ src/redis-shake/restore.go | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go index c7ec8d4..28760f8 100644 --- a/src/pkg/rdb/loader.go +++ b/src/pkg/rdb/loader.go @@ -196,9 +196,9 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { } else { key = l.lastEntry.Key } - // log.Infof("l %p r %p", l, l.rdbReader) - // log.Info("remainMember:", l.remainMember, " key:", string(key[:]), " type:", t) - // log.Info("r.remainMember:", l.rdbReader.remainMember) + log.Debugf("l %p r %p", l, l.rdbReader) + log.Debugf("remainMember:", l.remainMember, " key:", string(key[:]), " type:", t) + log.Debugf("r.remainMember:", l.rdbReader.remainMember) val, err := l.readObjectValue(t, l) if err != nil { return nil, err diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index f9a5c5e..7fe4eea 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -14,6 +14,7 @@ import ( "pkg/libs/errors" // "libs/log" + "pkg/libs/log" ) var FromVersion int64 = 9 @@ -144,10 +145,12 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { if n, err := r.ReadLength(); err != nil { return nil, err } else { + log.Debug("zset length: ", n) for i := 0; i < int(n); i++ { if _, err := r.ReadString(); err != nil { return nil, err } + log.Debug("zset read: ", i) if t == RdbTypeZSet2 { if _, err := r.ReadDouble(); err != nil { return nil, err diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index 4c000f8..bed4b1d 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -180,8 +180,12 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth log.Debugf("routine[%v] start restoring key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) + //e.Value = []byte(string("\x0c\x1f\x1f\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\xf1\x02\xf1\x02\xd0A*\xf4`\x06\xe0\x8c.\x1e\xa0l\x01\x00\x00\xff\x06\x00\xbbl\x83\x0b\x91? Date: Tue, 27 Aug 2019 19:03:36 +0800 Subject: [PATCH 31/33] add debug log --- .circleci/config.yml | 29 ----------------------------- src/pkg/rdb/loader.go | 6 +++--- src/pkg/rdb/reader.go | 7 +++---- 3 files changed, 6 insertions(+), 36 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index d926c18..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Golang CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-go/ for more details -version: 2 -jobs: - build: - docker: - # specify the version - - image: circleci/golang:1.10 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - #### TEMPLATE_NOTE: go expects specific checkout path representing url - #### expecting it in the form of - #### /go/src/github.com/circleci/go-tool - #### /go/src/bitbucket.org/circleci/go-tool - working_directory: /go/src/github.com/alibaba/RedisShake - steps: - - checkout - - # specify any bash command here prefixed with `run: ` - - run: export GOPATH=`pwd` - - run: go get -u github.com/kardianos/govendor - - run: cd src && ls && ls vendor - - run: govendor sync -v - - run: cd .. && sh build.sh diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go index 28760f8..a591ac0 100644 --- a/src/pkg/rdb/loader.go +++ b/src/pkg/rdb/loader.go @@ -196,9 +196,9 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { } else { key = l.lastEntry.Key } - log.Debugf("l %p r %p", l, l.rdbReader) - log.Debugf("remainMember:", l.remainMember, " key:", string(key[:]), " type:", t) - log.Debugf("r.remainMember:", l.rdbReader.remainMember) + //log.Debugf("l %p r %p", l, l.rdbReader) + //log.Debug("remainMember:", l.remainMember, " key:", string(key[:]), " type:", t) + //log.Debug("r.remainMember:", l.rdbReader.remainMember) val, err := l.readObjectValue(t, l) if err != nil { return nil, err diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index 7fe4eea..a4ab75a 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -13,8 +13,7 @@ import ( "strconv" "pkg/libs/errors" - // "libs/log" - "pkg/libs/log" + ) var FromVersion int64 = 9 @@ -145,12 +144,12 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { if n, err := r.ReadLength(); err != nil { return nil, err } else { - log.Debug("zset length: ", n) + // log.Debug("zset length: ", n) for i := 0; i < int(n); i++ { if _, err := r.ReadString(); err != nil { return nil, err } - log.Debug("zset read: ", i) + // log.Debug("zset read: ", i) if t == RdbTypeZSet2 { if _, err := r.ReadDouble(); err != nil { return nil, err From c1316cabb6c1a8700410cf9d05f23149b2abe1be Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 27 Aug 2019 19:04:12 +0800 Subject: [PATCH 32/33] release v1.6.17 --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index a6159d6..cf892b7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -2019-08-12 Alibaba Cloud. +2019-08-27 Alibaba Cloud. * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. From 177a6fb59ea102db42c5da4cc0e2027d7b523b97 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 27 Aug 2019 19:06:17 +0800 Subject: [PATCH 33/33] release v1.6.17 2 --- src/redis-shake/restore.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index bed4b1d..4c000f8 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -180,12 +180,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth log.Debugf("routine[%v] start restoring key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) - //e.Value = []byte(string("\x0c\x1f\x1f\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\xf1\x02\xf1\x02\xd0A*\xf4`\x06\xe0\x8c.\x1e\xa0l\x01\x00\x00\xff\x06\x00\xbbl\x83\x0b\x91?