From da00771e6bebe34886251443ba0f875ea9dff027 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 15 Jul 2019 19:17:33 +0800 Subject: [PATCH 01/10] support target.db in rump mode --- src/redis-shake/rump.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index 5ad0624..2a0bb03 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -339,6 +339,9 @@ func (dre *dbRumperExecutor) writer() { log.Debugf("dbRumper[%v] executor[%v] skip key %s for expired", dre.rumperId, dre.executorId, ele.key) continue } + if conf.Options.TargetDB != -1 { + ele.db = conf.Options.TargetDB + } log.Debugf("dbRumper[%v] executor[%v] restore[%s], length[%v]", dre.rumperId, dre.executorId, ele.key, len(ele.value)) From e2b6ecd037ed7e07353cc585e38243c525e71696 Mon Sep 17 00:00:00 2001 From: vinllen Date: Mon, 15 Jul 2019 19:17:33 +0800 Subject: [PATCH 02/10] support target.db in rump mode --- src/redis-shake/rump.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index 5ad0624..2a0bb03 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -339,6 +339,9 @@ func (dre *dbRumperExecutor) writer() { log.Debugf("dbRumper[%v] executor[%v] skip key %s for expired", dre.rumperId, dre.executorId, ele.key) continue } + if conf.Options.TargetDB != -1 { + ele.db = conf.Options.TargetDB + } log.Debugf("dbRumper[%v] executor[%v] restore[%s], length[%v]", dre.rumperId, dre.executorId, ele.key, len(ele.value)) From f68b1756c7fa5c33ae6074f365b67a18ebd9865f Mon Sep 17 00:00:00 2001 From: vinllen Date: Fri, 19 Jul 2019 16:17:13 +0800 Subject: [PATCH 03/10] polish code comment2 --- src/redis-shake/common/mix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-shake/common/mix.go b/src/redis-shake/common/mix.go index a17daec..f3637d4 100644 --- a/src/redis-shake/common/mix.go +++ b/src/redis-shake/common/mix.go @@ -52,7 +52,7 @@ func Welcome() { `______________________________ \ \ _ ______ | \ \ / \___-=O'/|O'/__| - \ redis-shake, here we go !! \_______\ / | / ) + \ RedisShake, here we go !! \_______\ / | / ) / / '/-==__ _/__|/__=-| -GM / / * \ | | / / (o) From bb96149325f7ce0cf10e4e2cc326a85c3987e080 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 10:16:19 +0800 Subject: [PATCH 04/10] add debug log in RDB syncing --- src/redis-shake/common/cluster.go | 8 ++++++++ src/redis-shake/restore.go | 2 ++ src/redis-shake/sync.go | 1 + 3 files changed, 11 insertions(+) diff --git a/src/redis-shake/common/cluster.go b/src/redis-shake/common/cluster.go index a4cbeb7..8bf9df5 100644 --- a/src/redis-shake/common/cluster.go +++ b/src/redis-shake/common/cluster.go @@ -7,6 +7,10 @@ import ( "pkg/libs/log" ) +const( + RecvChanSize = 4096 +) + /* implement redigo.Conn(https://github.com/garyburd/redigo) * Embed redis-go-cluster(https://github.com/chasex/redis-go-cluster) * The reason I create this struct is that redis-go-cluster isn't fulfill redigo.Conn @@ -24,6 +28,10 @@ type reply struct { } func NewClusterConn(clusterClient *redigoCluster.Cluster, recvChanSize int) redigo.Conn { + if recvChanSize == 0 { + recvChanSize = RecvChanSize + } + return &ClusterConn{ client: clusterClient, recvChan: make(chan reply, recvChanSize), diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index ef1fef9..6286a58 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -174,6 +174,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth continue } utils.RestoreRdbEntry(c, e) + + log.Debugf("routine[%v] restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) } } }() diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 1545695..a3e126f 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -439,6 +439,7 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type } } utils.RestoreRdbEntry(c, e) + log.Debugf("dbSyncer[%v] restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value)) } } }() From c2329a032590fd9f9c968d0fe581324de8718449 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 10:54:11 +0800 Subject: [PATCH 05/10] add target db judge when target type is cluster --- src/redis-shake/main/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index d899d5a..c20b314 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -353,6 +353,10 @@ func sanitizeOptions(tp string) error { conf.Options.TargetDB = v } + if conf.Options.TargetDB > 0 && conf.Options.TargetType == conf.RedisTypeCluster { + return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB) + } + if conf.Options.HttpProfile < 0 || conf.Options.HttpProfile > 65535 { return fmt.Errorf("HttpProfile[%v] should in [0, 65535]", conf.Options.HttpProfile) } else if conf.Options.HttpProfile == 0 { From 23293da176d631ceb6c115aeb7c1681a92b03f4e Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:06:21 +0800 Subject: [PATCH 06/10] add more debug log in RDB --- src/redis-shake/restore.go | 3 ++- src/redis-shake/sync.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index 6286a58..1c1f15e 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -173,9 +173,10 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth if filter.FilterKey(string(e.Key)) { continue } - utils.RestoreRdbEntry(c, e) log.Debugf("routine[%v] restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) + utils.RestoreRdbEntry(c, e) + log.Debugf("routine[%v] restore key[%s] ok", dr.id, e.Key) } } }() diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index a3e126f..4054a2e 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -438,8 +438,9 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type continue } } - utils.RestoreRdbEntry(c, e) log.Debugf("dbSyncer[%v] restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value)) + utils.RestoreRdbEntry(c, e) + log.Debugf("routine[%v] restore key[%s] ok", ds.id, e.Key) } } }() From c5b3885063cda7ddda8520b237fa3463ecc1c695 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:13:40 +0800 Subject: [PATCH 07/10] add more debug log in RDB2 --- src/redis-shake/restore.go | 6 +++++- src/redis-shake/sync.go | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index 1c1f15e..234f807 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -158,6 +158,9 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth dr.ignore.Incr() } else { dr.nentry.Incr() + + log.Debugf("routine[%v] try restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) + if conf.Options.TargetDB != -1 { if conf.Options.TargetDB != int(lastdb) { lastdb = uint32(conf.Options.TargetDB) @@ -174,7 +177,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth continue } - log.Debugf("routine[%v] restore key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) + log.Debugf("routine[%v] start restoring key[%s] with value length[%v]", dr.id, e.Key, len(e.Value)) + utils.RestoreRdbEntry(c, e) log.Debugf("routine[%v] restore key[%s] ok", dr.id, e.Key) } diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 4054a2e..a5a12ef 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -414,6 +414,9 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type ds.ignore.Incr() } else { ds.nentry.Incr() + + log.Debugf("dbSyncer[%v] try restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value)) + if conf.Options.TargetDB != -1 { if conf.Options.TargetDB != int(lastdb) { lastdb = uint32(conf.Options.TargetDB) @@ -438,9 +441,11 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type continue } } - log.Debugf("dbSyncer[%v] restore key[%s] with value length[%v]", ds.id, e.Key, len(e.Value)) + + log.Debugf("dbSyncer[%v] start restoring key[%s] with value length[%v]", ds.id, e.Key, len(e.Value)) + utils.RestoreRdbEntry(c, e) - log.Debugf("routine[%v] restore key[%s] ok", ds.id, e.Key) + log.Debugf("dbSyncer[%v] restore key[%s] ok", ds.id, e.Key) } } }() From c96cae1ae4d984ff25439010cb9e183f3b161ef0 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:31:39 +0800 Subject: [PATCH 08/10] support filter db to db0 when target type is cluster, seee #127 --- src/redis-shake/main/main.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index c20b314..fe40357 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -328,13 +328,6 @@ func sanitizeOptions(tp string) error { return fmt.Errorf("only one of 'filter.key.whitelist' and 'filter.key.blacklist' can be given") } - // if the target is "cluster", only allow pass db 0 - if conf.Options.TargetType == conf.RedisTypeCluster { - conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0 - conf.Options.FilterDBBlacklist = []string{} // reset blacklist - log.Info("the target redis type is cluster, only pass db0") - } - if len(conf.Options.FilterSlot) > 0 { for i, val := range conf.Options.FilterSlot { if _, err := strconv.Atoi(val); err != nil { @@ -357,6 +350,17 @@ func sanitizeOptions(tp string) error { return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB) } + // if the target is "cluster", only allow pass db 0 + if conf.Options.TargetType == conf.RedisTypeCluster { + if conf.Options.TargetDB == -1 { + conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0 + conf.Options.FilterDBBlacklist = []string{} // reset blacklist + log.Info("the target redis type is cluster, only pass db0") + } else { // targetDB == 0 + log.Info("the target redis type is cluster, all db syncing to db0") + } + } + if conf.Options.HttpProfile < 0 || conf.Options.HttpProfile > 65535 { return fmt.Errorf("HttpProfile[%v] should in [0, 65535]", conf.Options.HttpProfile) } else if conf.Options.HttpProfile == 0 { From a371cd8d2e5f784bdb11f8505100a347989afbb4 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:39:18 +0800 Subject: [PATCH 09/10] release v1.6.13 --- ChangeLog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2f996b4..90a3f6b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2019-07-24 Alibaba Cloud. + * VERSION: 1.6.13 + * IMPROVE: support `filter.db.whitelist` and `filter.db.blacklist` to let + different db syncing to db0 even when target type is cluster. see #127. + * BUGFIX: fix bug of connection url in automatic discovery in cluster. see + #124. + * IMPROVE: support `target.db` in rump mode. + * IMPROVE: add debug log in RDB syncing. 2019-07-11 Alibaba Cloud. * VERSION: 1.6.12 * IMPROVE: support filter key with whitelist and blacklist. From fe721fe316484e815515ad9c22ab8f7de90e3951 Mon Sep 17 00:00:00 2001 From: vinllen Date: Wed, 24 Jul 2019 15:41:52 +0800 Subject: [PATCH 10/10] polish code --- src/redis-shake/main/main.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index fe40357..b91c146 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -346,18 +346,17 @@ func sanitizeOptions(tp string) error { conf.Options.TargetDB = v } - if conf.Options.TargetDB > 0 && conf.Options.TargetType == conf.RedisTypeCluster { - return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB) - } - // if the target is "cluster", only allow pass db 0 if conf.Options.TargetType == conf.RedisTypeCluster { if conf.Options.TargetDB == -1 { conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0 conf.Options.FilterDBBlacklist = []string{} // reset blacklist log.Info("the target redis type is cluster, only pass db0") - } else { // targetDB == 0 + } else if conf.Options.TargetDB == 0 { log.Info("the target redis type is cluster, all db syncing to db0") + } else { + // > 0 + return fmt.Errorf("target.db[%v] should in {-1, 0} when target type is cluster", conf.Options.TargetDB) } }