From b855b5033950b15ae4aa3a39d6080567a2495d6c Mon Sep 17 00:00:00 2001 From: vinllen Date: Thu, 11 Jul 2019 14:58:45 +0800 Subject: [PATCH] release v1.6.12 --- ChangeLog | 5 + conf/redis-shake.conf | 24 +- src/redis-shake/base/runner.go | 4 - src/redis-shake/command/redis-command_test.go | 125 -------- src/redis-shake/common/filter.go | 17 -- src/redis-shake/common/utils.go | 12 +- src/redis-shake/configure/configure.go | 16 +- src/redis-shake/filter/filter.go | 111 +++++++ src/redis-shake/filter/filter_test.go | 287 ++++++++++++++++++ .../redis_command.go} | 30 +- src/redis-shake/main/main.go | 23 +- src/redis-shake/restore.go | 9 +- src/redis-shake/rump.go | 8 +- src/redis-shake/sync.go | 68 ++--- 14 files changed, 497 insertions(+), 242 deletions(-) delete mode 100644 src/redis-shake/command/redis-command_test.go delete mode 100644 src/redis-shake/common/filter.go create mode 100644 src/redis-shake/filter/filter.go create mode 100644 src/redis-shake/filter/filter_test.go rename src/redis-shake/{command/redis-command.go => filter/redis_command.go} (85%) diff --git a/ChangeLog b/ChangeLog index 6e7d25c..2f996b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2019-07-11 Alibaba Cloud. + * VERSION: 1.6.12 + * IMPROVE: support filter key with whitelist and blacklist. + * IMPROVE: support filter db with whitelist and blacklist. + * BUGFIX: fix "bypass" count in metric. 2019-07-04 Alibaba Cloud. * VERSION: 1.6.11 * BUGFIX: adapt "redis-go-cluster" driver to fix bug of big key syncing diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index fffa352..9b2fecf 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -105,15 +105,25 @@ rewrite = true # filter db or key or slot # choose these db, e.g., 5, only choose db5. defalut is all. -# used in `restore` and `sync`. -# 支持过滤db，只让指定的db通过 -filter.db = +# used in `restore`, `sync` and `rump`. +# e.g., "0;5;10" means match db0, db5 and db10. +# at most one of `filter.db.whitelist` and `filter.db.blacklist` parameters can be given. +# if the filter.db.whitelist is not empty, the given db list will be passed while others filtered. +# if the filter.db.blacklist is not empty, the given db list will be filtered while others passed. +# all dbs will be passed if no condition given. +filter.db.whitelist = +filter.db.blacklist = # filter key with prefix string. multiple keys are separated by ';'. -# e.g., a;b;c -# default is all. +# e.g., "abc;bzz" match let "abc", "abc1", "abcxxx", "bzz" and "bzzwww". # used in `restore`, `sync` and `rump`. -# 支持按前缀过滤key，只让指定前缀的key通过，分号分隔。比如指定abc，将会过滤abc, abc1, abcxxx -filter.key = +# at most one of `filter.key.whitelist` and `filter.key.blacklist` parameters can be given. +# if the filter.key.whitelist is not empty, the given keys will be passed while others filtered. +# if the filter.key.blacklist is not empty, the given keys will be filtered while others passed. +# all the namespace will be passed if no condition given. +# 支持按前缀过滤key，只让指定前缀的key通过，分号分隔。比如指定abc，将会通过abc, abc1, abcxxx +filter.key.whitelist = +# 支持按前缀过滤key，不让指定前缀的key通过，分号分隔。比如指定abc，将会阻塞abc, abc1, abcxxx +filter.key.blacklist = # filter given slot, multiple slots are separated by ';'. # e.g., 1;2;3 # used in `sync`. diff --git a/src/redis-shake/base/runner.go b/src/redis-shake/base/runner.go index 70742d1..3a3f612 100644 --- a/src/redis-shake/base/runner.go +++ b/src/redis-shake/base/runner.go @@ -2,10 +2,6 @@ package base var( Status = "null" - AcceptDB = func(db uint32) bool { - return db >= 0 && db < 1024 - } - RDBPipeSize = 1024 ) diff --git a/src/redis-shake/command/redis-command_test.go b/src/redis-shake/command/redis-command_test.go deleted file mode 100644 index adff5ba..0000000 --- a/src/redis-shake/command/redis-command_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package command - -import ( - "testing" -) - -func Test_Get_Match_Keys_Mset_Cmd(t *testing.T) { - mset_cmd := redisCommands["mset"] - /*filterkey: x - *cmd: mset kk 1 - */ - args := make([][]byte, 2) - args[0] = []byte("kk") - args[1] = []byte("1") - filterkey := make([]string, 1) - filterkey[0] = "x" - new_args, ret := GetMatchKeys(mset_cmd, args, filterkey) - - if len(new_args) != 0 || ret != false { - t.Error("mset test fail") - } - - /*filterkey: k - *cmd: mset kk 1 - */ - args = make([][]byte, 2) - args[0] = []byte("kk") - args[1] = []byte("1") - filterkey = make([]string, 1) - filterkey[0] = "k" - new_args, ret = GetMatchKeys(mset_cmd, args, filterkey) - - if len(new_args) != 2 || ret != true { - t.Error("mset test fail") - } - - /*filterkey: k - *cmd: mset kk 1 gg ll zz nn k ll - */ - args = make([][]byte, 8) - args[0] = []byte("kk") - args[1] = []byte("1") - args[2] = []byte("gg") - args[3] = []byte("ll") - args[4] = []byte("zz") - args[5] = []byte("nn") - args[6] = []byte("k") - args[7] = []byte("ll") - filterkey = make([]string, 1) - filterkey[0] = "k" - new_args, ret = GetMatchKeys(mset_cmd, args, filterkey) - - if len(new_args) != 4 || ret != true || - string(new_args[0]) != "kk" || string(new_args[1]) != "1" || - string(new_args[2]) != "k" || string(new_args[3]) != "ll" { - t.Error("mset test fail") - } -} - -func Test_Get_Match_Keys_SetXX_Cmd(t *testing.T) { - set_cmd := redisCommands["set"] - /*filterkey: x - *cmd: set kk 1 - */ - args := make([][]byte, 2) - args[0] = []byte("kk") - args[1] = []byte("1") - filterkey := make([]string, 1) - filterkey[0] = "x" - new_args, ret := GetMatchKeys(set_cmd, args, filterkey) - - if ret != false { - t.Error("set test fail", ret, len(new_args)) - } - - /*filterkey: k - *cmd: set kk 1 - */ - args = make([][]byte, 2) - args[0] = []byte("kk") - args[1] = []byte("1") - filterkey = make([]string, 1) - filterkey[0] = "k" - new_args, ret = GetMatchKeys(set_cmd, args, filterkey) - - if len(new_args) != 2 || ret != true { - t.Error("set test fail") - } - - /*filterkey: k - *cmd: setex kk 3000 lll - */ - set_cmd = redisCommands["setex"] - args = make([][]byte, 3) - args[0] = []byte("kk") - args[1] = []byte("3000") - args[2] = []byte("lll") - filterkey = make([]string, 1) - filterkey[0] = "k" - new_args, ret = GetMatchKeys(set_cmd, args, filterkey) - - if len(new_args) != 3 || ret != true || - string(new_args[0]) != "kk" || string(new_args[1]) != "3000" || - string(new_args[2]) != "lll" { - t.Error("setex test fail") - } - - /*filterkey: k - *cmd: setrange kk 3000 lll - */ - set_cmd = redisCommands["setrange"] - args = make([][]byte, 3) - args[0] = []byte("kk") - args[1] = []byte("3000") - args[2] = []byte("lll") - filterkey = make([]string, 1) - filterkey[0] = "k" - new_args, ret = GetMatchKeys(set_cmd, args, filterkey) - - if len(new_args) != 3 || ret != true || - string(new_args[0]) != "kk" || string(new_args[1]) != "3000" || - string(new_args[2]) != "lll" { - t.Error("setrange test fail") - } -} diff --git a/src/redis-shake/common/filter.go b/src/redis-shake/common/filter.go deleted file mode 100644 index 720c438..0000000 --- a/src/redis-shake/common/filter.go +++ /dev/null @@ -1,17 +0,0 @@ -package utils - -import "strings" - -// return true means not pass -func FilterCommands(cmd string, luaFilter bool) bool { - if strings.EqualFold(cmd, "opinfo") { - return true - } - - if luaFilter && (strings.EqualFold(cmd, "eval") || strings.EqualFold(cmd, "script") || - strings.EqualFold(cmd, "evalsha")) { - return true - } - - return false -} \ No newline at end of file diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index 3b06aa8..b1e0deb 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -1007,14 +1007,4 @@ var defaultDialFunction = func(addr string) (redigo.Conn, error) { return nil, err } return c, nil -} - -// HasAtLeastOnePrefix checks whether the key begins with at least one of prefixes. -func HasAtLeastOnePrefix(key string, prefixes []string) bool { - for _, prefix := range prefixes { - if strings.HasPrefix(key, prefix) { - return true - } - } - return false -} +} \ No newline at end of file diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index 3ffc31d..0ab42bb 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -33,8 +33,10 @@ type Configuration struct { RdbSpecialCloud string `config:"rdb.special_cloud"` FakeTime string `config:"fake_time"` Rewrite bool `config:"rewrite"` - FilterDB string `config:"filter.db"` - FilterKey []string `config:"filter.key"` + FilterDBWhitelist []string `config:"filter.db.whitelist"` + FilterDBBlacklist []string `config:"filter.db.blacklist"` + FilterKeyWhitelist []string `config:"filter.key.whitelist"` + FilterKeyBlacklist []string `config:"filter.key.blacklist"` FilterSlot []string `config:"filter.slot"` FilterLua bool `config:"filter.lua"` BigKeyThreshold uint64 `config:"big_key_threshold"` @@ -56,10 +58,12 @@ type Configuration struct { Qps int `config:"qps"` // inner variables - ReplaceHashTag bool `config:"replace_hash_tag"` - ExtraInfo bool `config:"extra"` - SockFileName string `config:"sock.file_name"` - SockFileSize uint `config:"sock.file_size"` + ReplaceHashTag bool `config:"replace_hash_tag"` + ExtraInfo bool `config:"extra"` + SockFileName string `config:"sock.file_name"` + SockFileSize uint `config:"sock.file_size"` + FilterKey []string `config:"filter.key"` // compatible with older versions + FilterDB string `config:"filter.db"` // compatible with older versions /*---------------------------------------------------------*/ // generated variables diff --git a/src/redis-shake/filter/filter.go b/src/redis-shake/filter/filter.go new file mode 100644 index 0000000..a7b7ef8 --- /dev/null +++ b/src/redis-shake/filter/filter.go @@ -0,0 +1,111 @@ +package filter + +import ( + "strings" + "redis-shake/configure" + "strconv" +) + +// return true means not pass +func FilterCommands(cmd string) bool { + if strings.EqualFold(cmd, "opinfo") { + return true + } + + if conf.Options.FilterLua && (strings.EqualFold(cmd, "eval") || strings.EqualFold(cmd, "script") || + strings.EqualFold(cmd, "evalsha")) { + return true + } + + return false +} + +// return true means not pass +func FilterKey(key string) bool { + if len(conf.Options.FilterKeyBlacklist) != 0 { + if hasAtLeastOnePrefix(key, conf.Options.FilterKeyBlacklist) { + return true + } + return false + } else if len(conf.Options.FilterKeyWhitelist) != 0 { + if hasAtLeastOnePrefix(key, conf.Options.FilterKeyWhitelist) { + return false + } + return true + } + return false +} + +// return true means not pass +func FilterSlot(slot int) bool { + if len(conf.Options.FilterSlot) == 0 { + return false + } + + // the slot in FilterSlot need to be passed + for _, ele := range conf.Options.FilterSlot { + slotInt, _ := strconv.Atoi(ele) + if slot == slotInt { + return false + } + } + return true +} + +// return true means not pass +func FilterDB(db int) bool { + dbString := strconv.FormatInt(int64(db), 10) + if len(conf.Options.FilterDBBlacklist) != 0 { + if matchOne(dbString, conf.Options.FilterDBBlacklist) { + return true + } + return false + } else if len(conf.Options.FilterDBWhitelist) != 0 { + if matchOne(dbString, conf.Options.FilterDBWhitelist) { + return false + } + return true + } + return false +} + +/* + * judge whether the input command with key should be filter, + * @return: + * [][]byte: the new argv which may be modified after filter. + * bool: true means pass + */ +func HandleFilterKeyWithCommand(scmd string, commandArgv [][]byte) ([][]byte, bool) { + if len(conf.Options.FilterKeyWhitelist) == 0 && len(conf.Options.FilterKeyBlacklist) == 0 { + // pass if no filter given + return commandArgv, false + } + + cmdNode, ok := RedisCommands[scmd] + if !ok || len(commandArgv) == 0 { + // pass when command not found or length of argv == 0 + return commandArgv, false + } + + newArgs, pass := getMatchKeys(cmdNode, commandArgv) + return newArgs, !pass +} + +// hasAtLeastOnePrefix checks whether the key begins with at least one of prefixes. +func hasAtLeastOnePrefix(key string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func matchOne(input string, list []string) bool { + for _, ele := range list { + if ele == input { + return true + } + } + return false +} \ No newline at end of file diff --git a/src/redis-shake/filter/filter_test.go b/src/redis-shake/filter/filter_test.go new file mode 100644 index 0000000..dce0253 --- /dev/null +++ b/src/redis-shake/filter/filter_test.go @@ -0,0 +1,287 @@ +package filter + +import ( + "testing" + "fmt" + + "redis-shake/configure" + + "github.com/stretchr/testify/assert" +) + +func TestFilterCommands(t *testing.T) { + // test FilterCommands + + var nr int + { + fmt.Printf("TestFilterCommands case %d.\n", nr) + nr++ + + assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal") + assert.Equal(t, true, FilterCommands("opinfo"), "should be equal") + assert.Equal(t, false, FilterCommands("eval"), "should be equal") + conf.Options.FilterLua = true + assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal") + assert.Equal(t, true, FilterCommands("eval"), "should be equal") + assert.Equal(t, true, FilterCommands("evalsha"), "should be equal") + assert.Equal(t, true, FilterCommands("script"), "should be equal") + + } +} + +func TestFilterKey(t *testing.T) { + // test FilterKey + + var nr int + { + fmt.Printf("TestFilterKey case %d.\n", nr) + nr++ + + assert.Equal(t, false, FilterKey("unknown-key"), "should be equal") + } + + { + fmt.Printf("TestFilterKey case %d.\n", nr) + nr++ + + conf.Options.FilterKeyBlacklist = []string{"abc", "xyz", "a"} + conf.Options.FilterKeyWhitelist = []string{} + assert.Equal(t, false, FilterKey("unknown-key"), "should be equal") + assert.Equal(t, true, FilterKey("abc"), "should be equal") + assert.Equal(t, true, FilterKey("abc111"), "should be equal") + assert.Equal(t, true, FilterKey("abcxyz"), "should be equal") + assert.Equal(t, true, FilterKey("xyz"), "should be equal") + assert.Equal(t, false, FilterKey("xy"), "should be equal") + assert.Equal(t, true, FilterKey("a"), "should be equal") + assert.Equal(t, true, FilterKey("ab"), "should be equal") + } + + { + fmt.Printf("TestFilterKey case %d.\n", nr) + nr++ + + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{"abc", "xyz", "a"} + assert.Equal(t, true, FilterKey("unknown-key"), "should be equal") + assert.Equal(t, false, FilterKey("abc"), "should be equal") + assert.Equal(t, false, FilterKey("abc111"), "should be equal") + assert.Equal(t, false, FilterKey("abcxyz"), "should be equal") + assert.Equal(t, false, FilterKey("xyz"), "should be equal") + assert.Equal(t, true, FilterKey("xy"), "should be equal") + assert.Equal(t, false, FilterKey("a"), "should be equal") + assert.Equal(t, false, FilterKey("ab"), "should be equal") + } +} + +func TestFilterSlot(t *testing.T) { + // test FilterSlot + + var nr int + { + fmt.Printf("TestFilterSlot case %d.\n", nr) + nr++ + + conf.Options.FilterSlot = []string{} + assert.Equal(t, false, FilterSlot(2), "should be equal") + assert.Equal(t, false, FilterSlot(0), "should be equal") + } + + { + fmt.Printf("TestFilterSlot case %d.\n", nr) + nr++ + + conf.Options.FilterSlot = []string{"1", "3", "5"} + assert.Equal(t, false, FilterSlot(1), "should be equal") + assert.Equal(t, true, FilterSlot(0), "should be equal") + assert.Equal(t, false, FilterSlot(5), "should be equal") + } +} + +func TestFilterDB(t *testing.T) { + // test FilterDB + + var nr int + { + fmt.Printf("TestFilterDB case %d.\n", nr) + nr++ + + conf.Options.FilterDBWhitelist = []string{} + conf.Options.FilterDBBlacklist = []string{} + assert.Equal(t, false, FilterDB(2), "should be equal") + assert.Equal(t, false, FilterDB(0), "should be equal") + } + + { + fmt.Printf("TestFilterDB case %d.\n", nr) + nr++ + + conf.Options.FilterDBWhitelist = []string{"0", "1", "5"} + conf.Options.FilterDBBlacklist = []string{} + assert.Equal(t, true, FilterDB(2), "should be equal") + assert.Equal(t, false, FilterDB(0), "should be equal") + assert.Equal(t, false, FilterDB(5), "should be equal") + + } + + { + fmt.Printf("TestFilterDB case %d.\n", nr) + nr++ + + conf.Options.FilterDBWhitelist = []string{} + conf.Options.FilterDBBlacklist = []string{"0", "1", "5"} + assert.Equal(t, false, FilterDB(2), "should be equal") + assert.Equal(t, true, FilterDB(0), "should be equal") + assert.Equal(t, true, FilterDB(5), "should be equal") + + } +} + +func TestHandleFilterKeyWithCommand(t *testing.T) { + // test HandleFilterKeyWithCommand + + var nr int + var cmd string + var args, expectArgs, ret [][]byte + var filter bool + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "set" + args = convertToByte("xyz", "1") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{"x", "y"} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, true, filter, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{"x"} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + } + + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "mset" + args = convertToByte("xyz", "1", "abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{"x"} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{"x"} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("xyz", "1") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + } + + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "msetnx" + args = convertToByte("xyz", "1", "abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{"x"} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{"x"} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("xyz", "1") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + } + + // unknown command, should pass + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "unknownCmd" + args = convertToByte("xyz", "1") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + } + + // length == 0 + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "unknownCmd" + args = convertToByte("xyz") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + } + + // del + { + fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr) + nr++ + + cmd = "del" + args = convertToByte("xyz", "abc", "ab", "zzz", "ffffffffff") + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, args, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{"x"} + conf.Options.FilterKeyWhitelist = []string{} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("abc", "ab", "zzz", "ffffffffff") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + + conf.Options.FilterKeyBlacklist = []string{} + conf.Options.FilterKeyWhitelist = []string{"x"} + ret, filter = HandleFilterKeyWithCommand(cmd, args) + expectArgs = convertToByte("xyz") + assert.Equal(t, false, filter, "should be equal") + assert.Equal(t, expectArgs, ret, "should be equal") + } +} + +func convertToByte(args... string) [][]byte { + ret := make([][]byte, 0) + for _, arg := range args { + ret = append(ret, []byte(arg)) + } + return ret +} \ No newline at end of file diff --git a/src/redis-shake/command/redis-command.go b/src/redis-shake/filter/redis_command.go similarity index 85% rename from src/redis-shake/command/redis-command.go rename to src/redis-shake/filter/redis_command.go index 3245eef..e2a69a5 100644 --- a/src/redis-shake/command/redis-command.go +++ b/src/redis-shake/filter/redis_command.go @@ -1,9 +1,5 @@ // redis command struct. -package command - -import ( - "strings" -) +package filter type getkeys_proc func(args []string) []int type redisCommand struct { @@ -86,7 +82,7 @@ var RedisCommands = map[string]redisCommand{ "pfmerge": {nil, 1, -1, 1}, } -func GetMatchKeys(redis_cmd redisCommand, args [][]byte, filterkeys []string) (new_args [][]byte, pass bool) { +func getMatchKeys(redis_cmd redisCommand, args [][]byte) (new_args [][]byte, pass bool) { lastkey := redis_cmd.lastkey - 1 keystep := redis_cmd.keystep @@ -94,34 +90,32 @@ func GetMatchKeys(redis_cmd redisCommand, args [][]byte, filterkeys []string) (n lastkey = lastkey + len(args) } - array := make([]int, len(args)) - number := 0 + array := make([]int, len(args)) // store all positions of the pass key + number := 0 // matching key number for firstkey := redis_cmd.firstkey - 1; firstkey <= lastkey; firstkey += keystep { key := string(args[firstkey]) - for i := 0; i < len(filterkeys); i++ { - if strings.HasPrefix(key, filterkeys[i]) { - array[number] = firstkey - number++ - break - } + if FilterKey(key) == false { + // pass + array[number] = firstkey + number++ } } pass = false - new_args = make([][]byte, number*redis_cmd.keystep+len(args)-lastkey-redis_cmd.keystep) + new_args = make([][]byte, number * redis_cmd.keystep + len(args) - lastkey - redis_cmd.keystep) if number > 0 { pass = true for i := 0; i < number; i++ { for j := 0; j < redis_cmd.keystep; j++ { - new_args[i*redis_cmd.keystep+j] = args[array[i]+j] + new_args[i * redis_cmd.keystep + j] = args[array[i] + j] } } } - // add alis paramters + // add alias parameters j := 0 for i := lastkey + redis_cmd.keystep; i < len(args); i++ { - new_args[number*redis_cmd.keystep+j] = args[i] + new_args[number * redis_cmd.keystep + j] = args[i] j = j + 1 } diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 1b5c649..d899d5a 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -315,20 +315,23 @@ func sanitizeOptions(tp string) error { } if conf.Options.FilterDB != "" { - if n, err := strconv.ParseInt(conf.Options.FilterDB, 10, 32); err != nil { - return fmt.Errorf("parse FilterDB failed[%v]", err) - } else { - base.AcceptDB = func(db uint32) bool { - return db == uint32(n) - } - } + conf.Options.FilterDBWhitelist = []string{conf.Options.FilterDB} + } + if len(conf.Options.FilterDBWhitelist) != 0 && len(conf.Options.FilterDBBlacklist) != 0 { + return fmt.Errorf("only one of 'filter.db.whitelist' and 'filter.db.blacklist' can be given") + } + + if len(conf.Options.FilterKey) != 0 { + conf.Options.FilterKeyWhitelist = conf.Options.FilterKey + } + if len(conf.Options.FilterKeyWhitelist) != 0 && len(conf.Options.FilterKeyBlacklist) != 0 { + return fmt.Errorf("only one of 'filter.key.whitelist' and 'filter.key.blacklist' can be given") } // if the target is "cluster", only allow pass db 0 if conf.Options.TargetType == conf.RedisTypeCluster { - base.AcceptDB = func(db uint32) bool { - return db == 0 - } + conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0 + conf.Options.FilterDBBlacklist = []string{} // reset blacklist log.Info("the target redis type is cluster, only pass db0") } diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go index 3b63c3a..ef1fef9 100644 --- a/src/redis-shake/restore.go +++ b/src/redis-shake/restore.go @@ -19,6 +19,7 @@ import ( "redis-shake/base" "redis-shake/common" "redis-shake/configure" + "redis-shake/filter" ) type CmdRestore struct { @@ -152,7 +153,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth defer c.Close() var lastdb uint32 = 0 for e := range pipe { - if !base.AcceptDB(e.DB) { + if filter.FilterDB(int(e.DB)) { + // filter db dr.ignore.Incr() } else { dr.nentry.Incr() @@ -167,7 +169,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth utils.SelectDB(c, lastdb) } } - if len(conf.Options.FilterKey) != 0 && !utils.HasAtLeastOnePrefix(string(e.Key), conf.Options.FilterKey) { + + if filter.FilterKey(string(e.Key)) { continue } utils.RestoreRdbEntry(c, e) @@ -233,7 +236,7 @@ func (dr *dbRestorer) restoreCommand(reader *bufio.Reader, target []string, auth if err != nil { log.PanicErrorf(err, "routine[%v] parse db = %s failed", dr.id, s) } - bypass = !base.AcceptDB(uint32(n)) + bypass = filter.FilterDB(n) } if bypass { dr.nbypass.Incr() diff --git a/src/redis-shake/rump.go b/src/redis-shake/rump.go index 46fa3da..5ad0624 100644 --- a/src/redis-shake/rump.go +++ b/src/redis-shake/rump.go @@ -15,6 +15,7 @@ import ( "redis-shake/scanner" "github.com/garyburd/redigo/redis" + "redis-shake/filter" ) type CmdRump struct { @@ -301,6 +302,11 @@ func (dre *dbRumperExecutor) fetcher() { log.Infof("dbRumper[%v] executor[%v] fetch db list: %v", dre.rumperId, dre.executorId, dbNumber) // iterate all db nodes for _, db := range dbNumber { + if filter.FilterDB(int(db)) { + log.Infof("dbRumper[%v] executor[%v] db[%v] filtered", dre.rumperId, dre.executorId, db) + continue + } + log.Infof("dbRumper[%v] executor[%v] fetch logical db: %v", dre.rumperId, dre.executorId, db) if err := dre.doFetch(int(db)); err != nil { log.Panic(err) @@ -320,7 +326,7 @@ func (dre *dbRumperExecutor) writer() { bucket := utils.StartQoS(conf.Options.Qps) preDb := 0 for ele := range dre.keyChan { - if len(conf.Options.FilterKey) != 0 && !utils.HasAtLeastOnePrefix(ele.key, conf.Options.FilterKey) { + if filter.FilterKey(ele.key) { continue } // QoS, limit the qps diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 24f1266..1545695 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -21,11 +21,11 @@ import ( "pkg/libs/log" "pkg/redis" "redis-shake/base" - "redis-shake/command" "redis-shake/common" "redis-shake/configure" "redis-shake/heartbeat" "redis-shake/metric" + "redis-shake/filter" ) type delayNode struct { @@ -409,7 +409,8 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type defer c.Close() var lastdb uint32 = 0 for e := range pipe { - if !base.AcceptDB(e.DB) { + if filter.FilterDB(int(e.DB)) { + // db filter ds.ignore.Incr() } else { ds.nentry.Incr() @@ -425,21 +426,19 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type } } - if len(conf.Options.FilterKey) != 0 { - if utils.HasAtLeastOnePrefix(string(e.Key), conf.Options.FilterKey) { - utils.RestoreRdbEntry(c, e) - } - } else if len(conf.Options.FilterSlot) > 0 { - for _, slot := range conf.Options.FilterSlot { - slotInt, _ := strconv.Atoi(slot) - if int(utils.KeyToSlot(string(e.Key))) == slotInt { - utils.RestoreRdbEntry(c, e) - break - } - } + if filter.FilterKey(string(e.Key)) == true { + // 1. judge if not pass filter key + ds.ignore.Incr() + continue } else { - utils.RestoreRdbEntry(c, e) + slot := int(utils.KeyToSlot(string(e.Key))) + if filter.FilterSlot(slot) == true { + // 2. judge if not pass filter slot + ds.ignore.Incr() + continue + } } + utils.RestoreRdbEntry(c, e) } } }() @@ -569,13 +568,15 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type }() go func() { - var lastdb int32 = 0 - var bypass bool = false - var isselect bool = false - - var scmd string - var argv, new_argv [][]byte - var err error + var ( + lastdb int32 = 0 + bypass = false + isselect = false + scmd string + argv, newArgv [][]byte + err error + reject bool + ) decoder := redis.NewDecoder(reader) @@ -611,9 +612,9 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type if err != nil { log.PanicErrorf(err, "dbSyncer[%v] parse db = %s failed", ds.id, s) } - bypass = !base.AcceptDB(uint32(n)) + bypass = filter.FilterDB(n) isselect = true - } else if utils.FilterCommands(scmd, conf.Options.FilterLua) { + } else if filter.FilterCommands(scmd) { ignorecmd = true } if bypass || ignorecmd { @@ -625,21 +626,8 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type } } - pass := false - if len(conf.Options.FilterKey) != 0 { - cmdNode, ok := command.RedisCommands[scmd] - if ok && len(argv) > 0 { - // log.Debugf("dbSyncer[%v] filter command[%v]", ds.id, scmd) - new_argv, pass = command.GetMatchKeys(cmdNode, argv, conf.Options.FilterKey) - } else { - pass = true - new_argv = argv - } - } else { - pass = true - new_argv = argv - } - if bypass || ignorecmd || !pass { + newArgv, reject = filter.HandleFilterKeyWithCommand(scmd, argv) + if bypass || ignorecmd || reject { ds.nbypass.Incr() metric.GetMetric(ds.id).AddBypassCmdCount(ds.id, 1) log.Debugf("dbSyncer[%v] filter command[%v]", ds.id, scmd) @@ -659,7 +647,7 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type } continue } - ds.sendBuf <- cmdDetail{Cmd: scmd, Args: new_argv} + ds.sendBuf <- cmdDetail{Cmd: scmd, Args: newArgv} } }()