From 2651e3d749718de211c6546138744c20e01eaa56 Mon Sep 17 00:00:00 2001 From: Jacky Wu Date: Sun, 26 Sep 2021 09:55:58 +0800 Subject: [PATCH] feat: Add config options for filtering commands. (#385) --- conf/redis-shake.conf | 13 +++++++++ src/redis-shake/configure/configure.go | 2 ++ src/redis-shake/filter/filter.go | 20 ++++++++++++-- src/redis-shake/filter/filter_test.go | 37 ++++++++++++++++++++------ src/redis-shake/main/sanitize.go | 3 +++ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 06c104b..c444957 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -185,6 +185,19 @@ filter.key.blacklist = # used in `sync`. # 指定过滤slot，只让指定的slot通过 filter.slot = +# filter give commands. multiple commands are separated by ';'. +# e.g., "flushall;flushdb". +# used in `sync`. +# at most one of `filter.command.whitelist` and `filter.command.blacklist` parameters can be given. +# if the filter.command.whitelist is not empty, the given commands will be passed while others filtered. +# if the filter.command.blacklist is not empty, the given commands will be filtered. +# besides, the other config caused filters also effect as usual, e.g. filter.lua = true would filter lua commands. +# all the commands, except the other config caused filtered commands, will be passed if no condition given. +# 只让指定命令通过，分号分隔 +filter.command.whitelist = +# 不让指定命令通过，分号分隔 +# 除了这些指定的命令外，其他配置选项指定的过滤命令也会照常生效，如开启 filter.lua 将会过滤 lua 相关命令 +filter.command.blacklist = # filter lua script. true means not pass. However, in redis 5.0, the lua # converts to transaction(multi+{commands}+exec) which will be passed. # 控制不让lua脚本通过，true表示不通过 diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index 87f5fcc..2eff599 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -39,6 +39,8 @@ type Configuration struct { FilterKeyWhitelist []string `config:"filter.key.whitelist"` FilterKeyBlacklist []string `config:"filter.key.blacklist"` FilterSlot []string `config:"filter.slot"` + FilterCommandWhitelist []string `config:"filter.command.whitelist"` + FilterCommandBlacklist []string `config:"filter.command.blacklist"` FilterLua bool `config:"filter.lua"` BigKeyThreshold uint64 `config:"big_key_threshold"` Metric bool `config:"metric"` diff --git a/src/redis-shake/filter/filter.go b/src/redis-shake/filter/filter.go index 7b94082..34c4883 100644 --- a/src/redis-shake/filter/filter.go +++ b/src/redis-shake/filter/filter.go @@ -1,10 +1,11 @@ package filter import ( - "github.com/alibaba/RedisShake/redis-shake/common" - "github.com/alibaba/RedisShake/redis-shake/configure" "strconv" "strings" + + utils "github.com/alibaba/RedisShake/redis-shake/common" + conf "github.com/alibaba/RedisShake/redis-shake/configure" ) var ( @@ -19,6 +20,21 @@ func FilterCommands(cmd string) bool { return true } + if len(conf.Options.FilterCommandWhitelist) != 0 { + if matchOne(cmd, conf.Options.FilterCommandWhitelist) { + return false + } + return true + } + + if len(conf.Options.FilterCommandBlacklist) != 0 { + if matchOne(cmd, conf.Options.FilterCommandBlacklist) { + return true + } + } + + // besides the blacklist, do the other filterings. + if conf.Options.FilterLua && (strings.EqualFold(cmd, "eval") || strings.EqualFold(cmd, "script") || strings.EqualFold(cmd, "evalsha")) { return true diff --git a/src/redis-shake/filter/filter_test.go b/src/redis-shake/filter/filter_test.go index eb12461..f108521 100644 --- a/src/redis-shake/filter/filter_test.go +++ b/src/redis-shake/filter/filter_test.go @@ -1,3 +1,4 @@ +//go:build (linux || darwin || windows) && integration // +build linux darwin windows // +build integration @@ -7,7 +8,7 @@ import ( "fmt" "testing" - "redis-shake/configure" + conf "github.com/alibaba/RedisShake/redis-shake/configure" "github.com/stretchr/testify/assert" ) @@ -20,15 +21,35 @@ func TestFilterCommands(t *testing.T) { fmt.Printf("TestFilterCommands case %d.\n", nr) nr++ - assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal") - assert.Equal(t, true, FilterCommands("opinfo"), "should be equal") - assert.Equal(t, false, FilterCommands("eval"), "should be equal") + fakedArgv := [][]byte{} + + assert.Equal(t, false, FilterCommands("unknown-cmd", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("opinfo", fakedArgv), "should be equal") + assert.Equal(t, false, FilterCommands("eval", fakedArgv), "should be equal") + conf.Options.FilterLua = true - assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal") - assert.Equal(t, true, FilterCommands("eval"), "should be equal") - assert.Equal(t, true, FilterCommands("evalsha"), "should be equal") - assert.Equal(t, true, FilterCommands("script"), "should be equal") + assert.Equal(t, false, FilterCommands("unknown-cmd", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("eval", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("evalsha", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("script", fakedArgv), "should be equal") + + conf.Options.FilterCommandBlacklist = []string{} + conf.Options.FilterCommandWhitelist = []string{"unknown-cmd"} + conf.Options.FilterLua = false + + assert.Equal(t, false, FilterCommands("unknown-cmd", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("eval", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("evalsha", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("script", fakedArgv), "should be equal") + + conf.Options.FilterCommandBlacklist = []string{"eval"} + conf.Options.FilterCommandWhitelist = []string{} + + assert.Equal(t, false, FilterCommands("unknown-cmd", fakedArgv), "should be equal") + assert.Equal(t, true, FilterCommands("eval", fakedArgv), "should be equal") + assert.Equal(t, false, FilterCommands("evalsha", fakedArgv), "should be equal") + assert.Equal(t, false, FilterCommands("script", fakedArgv), "should be equal") } } diff --git a/src/redis-shake/main/sanitize.go b/src/redis-shake/main/sanitize.go index d91add5..8530656 100644 --- a/src/redis-shake/main/sanitize.go +++ b/src/redis-shake/main/sanitize.go @@ -205,6 +205,9 @@ func SanitizeOptions(tp string) error { if len(conf.Options.FilterKeyWhitelist) != 0 && len(conf.Options.FilterKeyBlacklist) != 0 { return fmt.Errorf("only one of 'filter.key.whitelist' and 'filter.key.blacklist' can be given") } + if len(conf.Options.FilterCommandWhitelist) != 0 && len(conf.Options.FilterCommandBlacklist) != 0 { + return fmt.Errorf("only one of 'filter.command.whitelist' and 'filter.command.blacklist' can be given") + } if len(conf.Options.FilterSlot) > 0 { for i, val := range conf.Options.FilterSlot {