From 3a53adde266ca8712e7410e58255f4dab8a5ab93 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:26:03 +0800 Subject: [PATCH 1/8] update redis-go-cluster driver --- src/vendor/vendor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json index f382246..1760128 100644 --- a/src/vendor/vendor.json +++ b/src/vendor/vendor.json @@ -159,10 +159,10 @@ "revisionTime": "2019-03-04T09:57:49Z" }, { - "checksumSHA1": "IjUkzECf826IH8K1bexM/uockIk=", + "checksumSHA1": "1PrwQi6VvhLG4ovw1B7zD/afGZ4=", "path": "github.com/vinllen/redis-go-cluster", - "revision": "ffc34c0f905b65e6b71e07fd68a1d50b435c60f5", - "revisionTime": "2019-07-04T07:53:38Z" + "revision": "0799101ddfdb7d2bc9d47948f8a0cc17d23a216e", + "revisionTime": "2019-08-12T11:22:58Z" }, { "checksumSHA1": "TM3Neoy1xRAKyZYMGzKc41sDFW4=", From 4c96006d2ec4e0c4d17190e4ad50ff22519aa783 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:34:32 +0800 Subject: [PATCH 2/8] transaction syncing panic when target redis is cluster. see #145 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 68fe4a1..8fb97d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2019-08-12 Alibaba Cloud. + * VERSION: 1.6.17 + * BUGFIX: transaction syncing panic when target redis is cluster. see + #145. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. From ded322871d4faedde883b889efc60d9e9d157ad5 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:52:45 +0800 Subject: [PATCH 3/8] set sender count to 4096 if target redis type is cluster --- conf/redis-shake.conf | 2 +- src/redis-shake/main/main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 08b30dc..281578f 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -189,7 +189,7 @@ sender.size = 104857600 # sender flush buffer size of oplog number. # used in `sync`. # 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 -sender.count = 5000 +sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving # ack from target redis. diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 3ee087c..a6fa959 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,6 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } + if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { + conf.Options.SenderCount = utils.RecvChanSize + } if conf.Options.SenderDelayChannelSize == 0 { conf.Options.SenderDelayChannelSize = 32 From 89f99c839a54820f29706c4ffe207d572c808e96 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 12 Aug 2019 19:57:22 +0800 Subject: [PATCH 4/8] set max sender.count to 4096 if target redis type is cluster --- ChangeLog | 1 + conf/redis-shake.conf | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8fb97d5..646cb1a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. + * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 281578f..1214700 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -187,8 +187,9 @@ heartbeat.network_interface = # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 +# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, +# the maximum value is 4096. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving From e040724afaacc72023c8a7621e7d53d01c2747e7 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 18:31:39 +0800 Subject: [PATCH 5/8] add more debug in rdb restore --- src/redis-shake/common/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index b429299..6fb9b35 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -811,6 +811,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { params = append(params, "FREQ") params = append(params, e.Freq) } + + log.Debugf("restore key[%s] with params[%v]", e.Key, params) // fmt.Printf("key: %v, value: %v params: %v\n", string(e.Key), e.Value, params) // s, err := redigo.String(c.Do("restore", params...)) s, err := redigoCluster.String(c.Do("restore", params...)) From bece38f2ca10c1b63b4e606eb4d847bd117710c1 Mon Sep 17 00:00:00 2001 From: vinllen Date: Tue, 13 Aug 2019 19:19:47 +0800 Subject: [PATCH 6/8] adjust RecvChanSize based on 'sender.count' or 'scan.key_number' if target redis type is cluster --- ChangeLog | 3 ++- src/redis-shake/common/cluster.go | 2 +- src/redis-shake/main/main.go | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 646cb1a..ea8ffd7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,7 +2,8 @@ * VERSION: 1.6.17 * BUGFIX: transaction syncing panic when target redis is cluster. see #145. - * IMPROVE: set max `sender.count` to 4096 if target redis type is cluster. + * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` + if target redis type is cluster. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired. diff --git a/src/redis-shake/common/cluster.go b/src/redis-shake/common/cluster.go index 8bf9df5..2abb1e7 100644 --- a/src/redis-shake/common/cluster.go +++ b/src/redis-shake/common/cluster.go @@ -7,7 +7,7 @@ import ( "pkg/libs/log" ) -const( +var ( RecvChanSize = 4096 ) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index a6fa959..ea58fc1 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -396,8 +396,9 @@ func sanitizeOptions(tp string) error { // set to default when not set conf.Options.SenderCount = defaultSenderCount } - if conf.Options.TargetType == conf.RedisTypeCluster && conf.Options.SenderCount > utils.RecvChanSize { - conf.Options.SenderCount = utils.RecvChanSize + if conf.Options.TargetType == conf.RedisTypeCluster && int(conf.Options.SenderCount) > utils.RecvChanSize { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.SenderCount)) + utils.RecvChanSize = int(conf.Options.SenderCount) } if conf.Options.SenderDelayChannelSize == 0 { @@ -451,8 +452,9 @@ func sanitizeOptions(tp string) error { conf.Options.ScanSpecialCloud, conf.Options.ScanKeyFile) } - if conf.Options.ScanKeyNumber > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { - return fmt.Errorf("scan.key_number should less than [%v] when target type is cluster", utils.RecvChanSize) + if int(conf.Options.ScanKeyNumber) > utils.RecvChanSize && conf.Options.TargetType == conf.RedisTypeCluster { + log.Infof("RecvChanSize is modified from [%v] to [%v]", utils.RecvChanSize, int(conf.Options.ScanKeyNumber)) + utils.RecvChanSize = int(conf.Options.ScanKeyNumber) } //if len(conf.Options.SourceAddressList) == 1 { From 0fccb31eff9f1230270a23a818d2b21f03985c67 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:05 +0800 Subject: [PATCH 7/8] remove some useless variables in conf --- conf/redis-shake.conf | 30 +------ src/redis-shake/configure/configure.go | 108 ++++++++++++------------- 2 files changed, 57 insertions(+), 81 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 1214700..0d941a4 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -19,9 +19,6 @@ system_profile = 9310 # restful port，查看metric端口, -1表示不启用，如果是`restore`模式，只有设置为-1才会在完成RDB恢复后退出，否则会一直block。 http_profile = 9320 -# runtime.GOMAXPROCS, 0 means use cpu core number: runtime.NumCPU() -ncpu = 0 - # parallel routines number used in RDB file syncing. default is 64. # 启动多少个并发线程同步一个RDB文件。 parallel = 32 @@ -165,31 +162,15 @@ metric = true # 是否将metric打印到log中 metric.print_log = false -# heartbeat -# send heartbeat to this url -# used in `sync`. -# 心跳的url地址，redis-shake将会发送到这个地址 -#heartbeat.url = http://127.0.0.1:8000 -heartbeat.url = -# interval by seconds -# 心跳保活周期 -heartbeat.interval = 3 -# external info which will be included in heartbeat data. -# 在心跳报文中添加额外的信息 -heartbeat.external = test external -# local network card to get ip address, e.g., "lo", "eth0", "en0" -# 获取ip的网卡 -heartbeat.network_interface = - # sender information. # sender flush buffer size of byte. # used in `sync`. # 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 sender.size = 104857600 # sender flush buffer size of oplog number. -# used in `sync`. flush sender buffer when bigger than this threshold. for cluster, -# the maximum value is 4096. -# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，如果目的端是cluster，这个最大只能是4096。 +# used in `sync`. flush sender buffer when bigger than this threshold. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送，对于目的端是cluster的情况，这个值 +# 的调大将会占用部分内存。 sender.count = 4096 # delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also # stored in this delay queue. this timestamp will be used to calculate the time delay when receiving @@ -208,13 +189,11 @@ keep_alive = 0 # number of keys captured each time. default is 100. # 每次scan的个数，不配置则默认100. scan.key_number = 50 - # used in `rump`. # we support some special redis types that don't use default `scan` command like alibaba cloud and tencent cloud. # 有些版本具有特殊的格式，与普通的scan命令有所不同，我们进行了特殊的适配。目前支持腾讯云的集群版"tencent_cluster" # 和阿里云的集群版"aliyun_cluster"。 scan.special_cloud = - # used in `rump`. # we support to fetching data from given file which marks the key list. # 有些云版本，既不支持sync/psync，也不支持scan，我们支持从文件中进行读取所有key列表并进行抓取：一行一个key。 @@ -230,6 +209,3 @@ qps = 200000 # replace hash tag. # used in `sync`. replace_hash_tag = false - -# used in `restore` and `dump`. -extra = false diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index 5a8cc85..ec5343d 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -4,65 +4,65 @@ import "time" type Configuration struct { // config file variables - Id string `config:"id"` - LogFile string `config:"log.file"` - LogLevel string `config:"log.level"` - SystemProfile int `config:"system_profile"` - HttpProfile int `config:"http_profile"` + Id string `config:"id"` + LogFile string `config:"log.file"` + LogLevel string `config:"log.level"` + SystemProfile int `config:"system_profile"` + HttpProfile int `config:"http_profile"` + Parallel int `config:"parallel"` + SourceType string `config:"source.type"` + SourceAddress string `config:"source.address"` + SourcePasswordRaw string `config:"source.password_raw"` + SourcePasswordEncoding string `config:"source.password_encoding"` + SourceAuthType string `config:"source.auth_type"` + SourceTLSEnable bool `config:"source.tls_enable"` + SourceRdbInput []string `config:"source.rdb.input"` + SourceRdbParallel int `config:"source.rdb.parallel"` + SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` + TargetAddress string `config:"target.address"` + TargetPasswordRaw string `config:"target.password_raw"` + TargetPasswordEncoding string `config:"target.password_encoding"` + TargetDBString string `config:"target.db"` + TargetAuthType string `config:"target.auth_type"` + TargetType string `config:"target.type"` + TargetTLSEnable bool `config:"target.tls_enable"` + TargetRdbOutput string `config:"target.rdb.output"` + TargetVersion string `config:"target.version"` + FakeTime string `config:"fake_time"` + Rewrite bool `config:"rewrite"` + FilterDBWhitelist []string `config:"filter.db.whitelist"` + FilterDBBlacklist []string `config:"filter.db.blacklist"` + FilterKeyWhitelist []string `config:"filter.key.whitelist"` + FilterKeyBlacklist []string `config:"filter.key.blacklist"` + FilterSlot []string `config:"filter.slot"` + FilterLua bool `config:"filter.lua"` + BigKeyThreshold uint64 `config:"big_key_threshold"` + Psync bool `config:"psync"` + Metric bool `config:"metric"` + MetricPrintLog bool `config:"metric.print_log"` + SenderSize uint64 `config:"sender.size"` + SenderCount uint `config:"sender.count"` + SenderDelayChannelSize uint `config:"sender.delay_channel_size"` + KeepAlive uint `config:"keep_alive"` + PidPath string `config:"pid_path"` + ScanKeyNumber uint32 `config:"scan.key_number"` + ScanSpecialCloud string `config:"scan.special_cloud"` + ScanKeyFile string `config:"scan.key_file"` + Qps int `config:"qps"` + + /*---------------------------------------------------------*/ + // inner variables NCpu int `config:"ncpu"` - Parallel int `config:"parallel"` - SourceType string `config:"source.type"` - SourceAddress string `config:"source.address"` - SourcePasswordRaw string `config:"source.password_raw"` - SourcePasswordEncoding string `config:"source.password_encoding"` - SourceAuthType string `config:"source.auth_type"` - SourceTLSEnable bool `config:"source.tls_enable"` - SourceRdbInput []string `config:"source.rdb.input"` - SourceRdbParallel int `config:"source.rdb.parallel"` - SourceRdbSpecialCloud string `config:"source.rdb.special_cloud"` - TargetAddress string `config:"target.address"` - TargetPasswordRaw string `config:"target.password_raw"` - TargetPasswordEncoding string `config:"target.password_encoding"` - TargetDBString string `config:"target.db"` - TargetAuthType string `config:"target.auth_type"` - TargetType string `config:"target.type"` - TargetTLSEnable bool `config:"target.tls_enable"` - TargetRdbOutput string `config:"target.rdb.output"` - TargetVersion string `config:"target.version"` - FakeTime string `config:"fake_time"` - Rewrite bool `config:"rewrite"` - FilterDBWhitelist []string `config:"filter.db.whitelist"` - FilterDBBlacklist []string `config:"filter.db.blacklist"` - FilterKeyWhitelist []string `config:"filter.key.whitelist"` - FilterKeyBlacklist []string `config:"filter.key.blacklist"` - FilterSlot []string `config:"filter.slot"` - FilterLua bool `config:"filter.lua"` - BigKeyThreshold uint64 `config:"big_key_threshold"` - Psync bool `config:"psync"` - Metric bool `config:"metric"` - MetricPrintLog bool `config:"metric.print_log"` HeartbeatUrl string `config:"heartbeat.url"` HeartbeatInterval uint `config:"heartbeat.interval"` HeartbeatExternal string `config:"heartbeat.external"` HeartbeatNetworkInterface string `config:"heartbeat.network_interface"` - SenderSize uint64 `config:"sender.size"` - SenderCount uint `config:"sender.count"` - SenderDelayChannelSize uint `config:"sender.delay_channel_size"` - KeepAlive uint `config:"keep_alive"` - PidPath string `config:"pid_path"` - ScanKeyNumber uint32 `config:"scan.key_number"` - ScanSpecialCloud string `config:"scan.special_cloud"` - ScanKeyFile string `config:"scan.key_file"` - Qps int `config:"qps"` - - /*---------------------------------------------------------*/ - // inner variables - ReplaceHashTag bool `config:"replace_hash_tag"` - ExtraInfo bool `config:"extra"` - SockFileName string `config:"sock.file_name"` - SockFileSize uint `config:"sock.file_size"` - FilterKey []string `config:"filter.key"` // compatible with older versions - FilterDB string `config:"filter.db"` // compatible with older versions + ReplaceHashTag bool `config:"replace_hash_tag"` + ExtraInfo bool `config:"extra"` + SockFileName string `config:"sock.file_name"` + SockFileSize uint `config:"sock.file_size"` + FilterKey []string `config:"filter.key"` // compatible with older versions + FilterDB string `config:"filter.db"` // compatible with older versions /*---------------------------------------------------------*/ // generated variables From 5bd145b229ca3062944dd8b1c8ee2faa7c94d9cf Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 14 Aug 2019 19:21:50 +0800 Subject: [PATCH 8/8] remove some useless variables in conf2 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ea8ffd7..44629c8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,7 @@ #145. * IMPROVE: adjust RecvChanSize based on `sender.count` or `scan.key_number` if target redis type is cluster. + * IMPROVE: remove some useless variables in conf like `heartbeat`, `ncpu`. 2019-08-09 Alibaba Cloud. * VERSION: 1.6.16 * BUGFIX: big key in `rump` mode all expired.