release v1.6.12

ChangeLog

@@ -1,3 +1,8 @@
+2019-07-11 Alibaba Cloud.
+	* VERSION: 1.6.12
+	* IMPROVE: support filter key with whitelist and blacklist.
+	* IMPROVE: support filter db with whitelist and blacklist.
+	* BUGFIX: fix "bypass" count in metric.
 2019-07-04 Alibaba Cloud.
 	* VERSION: 1.6.11
 	* BUGFIX: adapt "redis-go-cluster" driver to fix bug of big key syncing

conf/redis-shake.conf

@@ -105,15 +105,25 @@ rewrite = true
 
 # filter db or key or slot
 # choose these db, e.g., 5, only choose db5. defalut is all.
-# used in `restore` and `sync`.
-# 支持过滤db，只让指定的db通过
-filter.db =
+# used in `restore`, `sync` and `rump`.
+# e.g., "0;5;10" means match db0, db5 and db10.
+# at most one of `filter.db.whitelist` and `filter.db.blacklist` parameters can be given.
+# if the filter.db.whitelist is not empty, the given db list will be passed while others filtered.
+# if the filter.db.blacklist is not empty, the given db list will be filtered while others passed.
+# all dbs will be passed if no condition given.
+filter.db.whitelist =
+filter.db.blacklist =
 # filter key with prefix string. multiple keys are separated by ';'.
-# e.g., a;b;c
-# default is all.
+# e.g., "abc;bzz" match let "abc", "abc1", "abcxxx", "bzz" and "bzzwww".
 # used in `restore`, `sync` and `rump`.
-# 支持按前缀过滤key，只让指定前缀的key通过，分号分隔。比如指定abc，将会过滤abc, abc1, abcxxx
-filter.key =
+# at most one of `filter.key.whitelist` and `filter.key.blacklist` parameters can be given.
+# if the filter.key.whitelist is not empty, the given keys will be passed while others filtered.
+# if the filter.key.blacklist is not empty, the given keys will be filtered while others passed.
+# all the namespace will be passed if no condition given.
+# 支持按前缀过滤key，只让指定前缀的key通过，分号分隔。比如指定abc，将会通过abc, abc1, abcxxx
+filter.key.whitelist =
+# 支持按前缀过滤key，不让指定前缀的key通过，分号分隔。比如指定abc，将会阻塞abc, abc1, abcxxx
+filter.key.blacklist =
 # filter given slot, multiple slots are separated by ';'.
 # e.g., 1;2;3
 # used in `sync`.

src/redis-shake/base/runner.go

@@ -2,10 +2,6 @@ package base
 
 var(
 	Status = "null"
-	AcceptDB = func(db uint32) bool {
-		return db >= 0 && db < 1024
-	}
-
 	RDBPipeSize = 1024
 )
 

src/redis-shake/command/redis-command_test.go

@@ -1,125 +0,0 @@
-package command
-
-import (
-	"testing"
-)
-
-func Test_Get_Match_Keys_Mset_Cmd(t *testing.T) {
-	mset_cmd := redisCommands["mset"]
-	/*filterkey: x
-	 *cmd: mset kk 1
-	 */
-	args := make([][]byte, 2)
-	args[0] = []byte("kk")
-	args[1] = []byte("1")
-	filterkey := make([]string, 1)
-	filterkey[0] = "x"
-	new_args, ret := GetMatchKeys(mset_cmd, args, filterkey)
-
-	if len(new_args) != 0 || ret != false {
-		t.Error("mset test fail")
-	}
-
-	/*filterkey: k
-	 *cmd: mset kk 1
-	 */
-	args = make([][]byte, 2)
-	args[0] = []byte("kk")
-	args[1] = []byte("1")
-	filterkey = make([]string, 1)
-	filterkey[0] = "k"
-	new_args, ret = GetMatchKeys(mset_cmd, args, filterkey)
-
-	if len(new_args) != 2 || ret != true {
-		t.Error("mset test fail")
-	}
-
-	/*filterkey: k
-	 *cmd: mset kk 1 gg ll zz nn k ll
-	 */
-	args = make([][]byte, 8)
-	args[0] = []byte("kk")
-	args[1] = []byte("1")
-	args[2] = []byte("gg")
-	args[3] = []byte("ll")
-	args[4] = []byte("zz")
-	args[5] = []byte("nn")
-	args[6] = []byte("k")
-	args[7] = []byte("ll")
-	filterkey = make([]string, 1)
-	filterkey[0] = "k"
-	new_args, ret = GetMatchKeys(mset_cmd, args, filterkey)
-
-	if len(new_args) != 4 || ret != true ||
-		string(new_args[0]) != "kk" || string(new_args[1]) != "1" ||
-		string(new_args[2]) != "k" || string(new_args[3]) != "ll" {
-		t.Error("mset test fail")
-	}
-}
-
-func Test_Get_Match_Keys_SetXX_Cmd(t *testing.T) {
-	set_cmd := redisCommands["set"]
-	/*filterkey: x
-	 *cmd: set kk 1
-	 */
-	args := make([][]byte, 2)
-	args[0] = []byte("kk")
-	args[1] = []byte("1")
-	filterkey := make([]string, 1)
-	filterkey[0] = "x"
-	new_args, ret := GetMatchKeys(set_cmd, args, filterkey)
-
-	if ret != false {
-		t.Error("set test fail", ret, len(new_args))
-	}
-
-	/*filterkey: k
-	 *cmd: set kk 1
-	 */
-	args = make([][]byte, 2)
-	args[0] = []byte("kk")
-	args[1] = []byte("1")
-	filterkey = make([]string, 1)
-	filterkey[0] = "k"
-	new_args, ret = GetMatchKeys(set_cmd, args, filterkey)
-
-	if len(new_args) != 2 || ret != true {
-		t.Error("set test fail")
-	}
-
-	/*filterkey: k
-	 *cmd: setex kk 3000 lll
-	 */
-	set_cmd = redisCommands["setex"]
-	args = make([][]byte, 3)
-	args[0] = []byte("kk")
-	args[1] = []byte("3000")
-	args[2] = []byte("lll")
-	filterkey = make([]string, 1)
-	filterkey[0] = "k"
-	new_args, ret = GetMatchKeys(set_cmd, args, filterkey)
-
-	if len(new_args) != 3 || ret != true ||
-		string(new_args[0]) != "kk" || string(new_args[1]) != "3000" ||
-		string(new_args[2]) != "lll" {
-		t.Error("setex test fail")
-	}
-
-	/*filterkey: k
-	 *cmd: setrange kk 3000 lll
-	 */
-	set_cmd = redisCommands["setrange"]
-	args = make([][]byte, 3)
-	args[0] = []byte("kk")
-	args[1] = []byte("3000")
-	args[2] = []byte("lll")
-	filterkey = make([]string, 1)
-	filterkey[0] = "k"
-	new_args, ret = GetMatchKeys(set_cmd, args, filterkey)
-
-	if len(new_args) != 3 || ret != true ||
-		string(new_args[0]) != "kk" || string(new_args[1]) != "3000" ||
-		string(new_args[2]) != "lll" {
-		t.Error("setrange test fail")
-	}
-}

src/redis-shake/common/filter.go

@@ -1,17 +0,0 @@
-package utils
-
-import "strings"
-
-// return true means not pass
-func FilterCommands(cmd string, luaFilter bool) bool {
-	if strings.EqualFold(cmd, "opinfo") {
-		return true
-	}
-
-	if luaFilter && (strings.EqualFold(cmd, "eval") || strings.EqualFold(cmd, "script") ||
-			strings.EqualFold(cmd, "evalsha")) {
-		return true
-	}
-
-	return false
-}

src/redis-shake/common/utils.go

@@ -1007,14 +1007,4 @@ var defaultDialFunction = func(addr string) (redigo.Conn, error) {
 		return nil, err
 	}
 	return c, nil
-}
-
-// HasAtLeastOnePrefix checks whether the key begins with at least one of prefixes.
-func HasAtLeastOnePrefix(key string, prefixes []string) bool {
-	for _, prefix := range prefixes {
-		if strings.HasPrefix(key, prefix) {
-			return true
-		}
-	}
-	return false
-}
+}

src/redis-shake/configure/configure.go

@@ -33,8 +33,10 @@ type Configuration struct {
 	RdbSpecialCloud           string   `config:"rdb.special_cloud"`
 	FakeTime                  string   `config:"fake_time"`
 	Rewrite                   bool     `config:"rewrite"`
-	FilterDB                  string   `config:"filter.db"`
-	FilterKey                 []string `config:"filter.key"`
+	FilterDBWhitelist         []string `config:"filter.db.whitelist"`
+	FilterDBBlacklist         []string `config:"filter.db.blacklist"`
+	FilterKeyWhitelist        []string `config:"filter.key.whitelist"`
+	FilterKeyBlacklist        []string `config:"filter.key.blacklist"`
 	FilterSlot                []string `config:"filter.slot"`
 	FilterLua                 bool     `config:"filter.lua"`
 	BigKeyThreshold           uint64   `config:"big_key_threshold"`
@@ -56,10 +58,12 @@ type Configuration struct {
 	Qps                       int      `config:"qps"`
 
 	// inner variables
-	ReplaceHashTag bool   `config:"replace_hash_tag"`
-	ExtraInfo      bool   `config:"extra"`
-	SockFileName   string `config:"sock.file_name"`
-	SockFileSize   uint   `config:"sock.file_size"`
+	ReplaceHashTag bool     `config:"replace_hash_tag"`
+	ExtraInfo      bool     `config:"extra"`
+	SockFileName   string   `config:"sock.file_name"`
+	SockFileSize   uint     `config:"sock.file_size"`
+	FilterKey      []string `config:"filter.key"` // compatible with older versions
+	FilterDB       string   `config:"filter.db"`  // compatible with older versions
 
 	/*---------------------------------------------------------*/
 	// generated variables

src/redis-shake/filter/filter.go

@@ -0,0 +1,111 @@
+package filter
+
+import (
+	"strings"
+	"redis-shake/configure"
+	"strconv"
+)
+
+// return true means not pass
+func FilterCommands(cmd string) bool {
+	if strings.EqualFold(cmd, "opinfo") {
+		return true
+	}
+
+	if conf.Options.FilterLua && (strings.EqualFold(cmd, "eval") || strings.EqualFold(cmd, "script") ||
+			strings.EqualFold(cmd, "evalsha")) {
+		return true
+	}
+
+	return false
+}
+
+// return true means not pass
+func FilterKey(key string) bool {
+	if len(conf.Options.FilterKeyBlacklist) != 0 {
+		if hasAtLeastOnePrefix(key, conf.Options.FilterKeyBlacklist) {
+			return true
+		}
+		return false
+	} else if len(conf.Options.FilterKeyWhitelist) != 0 {
+		if hasAtLeastOnePrefix(key, conf.Options.FilterKeyWhitelist) {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// return true means not pass
+func FilterSlot(slot int) bool {
+	if len(conf.Options.FilterSlot) == 0 {
+		return false
+	}
+
+	// the slot in FilterSlot need to be passed
+	for _, ele := range conf.Options.FilterSlot {
+		slotInt, _ := strconv.Atoi(ele)
+		if slot == slotInt {
+			return false
+		}
+	}
+	return true
+}
+
+// return true means not pass
+func FilterDB(db int) bool {
+	dbString := strconv.FormatInt(int64(db), 10)
+	if len(conf.Options.FilterDBBlacklist) != 0 {
+		if matchOne(dbString, conf.Options.FilterDBBlacklist) {
+			return true
+		}
+		return false
+	} else if len(conf.Options.FilterDBWhitelist) != 0 {
+		if matchOne(dbString, conf.Options.FilterDBWhitelist) {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+/*
+ * judge whether the input command with key should be filter,
+ * @return:
+ *     [][]byte: the new argv which may be modified after filter.
+ *     bool: true means pass
+ */
+func HandleFilterKeyWithCommand(scmd string, commandArgv [][]byte) ([][]byte, bool) {
+	if len(conf.Options.FilterKeyWhitelist) == 0 && len(conf.Options.FilterKeyBlacklist) == 0 {
+		// pass if no filter given
+		return commandArgv, false
+	}
+
+	cmdNode, ok := RedisCommands[scmd]
+	if !ok || len(commandArgv) == 0 {
+		// pass when command not found or length of argv == 0
+		return commandArgv, false
+	}
+
+	newArgs, pass := getMatchKeys(cmdNode, commandArgv)
+	return newArgs, !pass
+}
+
+// hasAtLeastOnePrefix checks whether the key begins with at least one of prefixes.
+func hasAtLeastOnePrefix(key string, prefixes []string) bool {
+	for _, prefix := range prefixes {
+		if strings.HasPrefix(key, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchOne(input string, list []string) bool {
+	for _, ele := range list {
+		if ele == input {
+			return true
+		}
+	}
+	return false
+}

src/redis-shake/filter/filter_test.go

@@ -0,0 +1,287 @@
+package filter
+
+import (
+	"testing"
+	"fmt"
+
+	"redis-shake/configure"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterCommands(t *testing.T) {
+	// test FilterCommands
+
+	var nr int
+	{
+		fmt.Printf("TestFilterCommands case %d.\n", nr)
+		nr++
+
+		assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal")
+		assert.Equal(t, true, FilterCommands("opinfo"), "should be equal")
+		assert.Equal(t, false, FilterCommands("eval"), "should be equal")
+		conf.Options.FilterLua = true
+		assert.Equal(t, false, FilterCommands("unknown-cmd"), "should be equal")
+		assert.Equal(t, true, FilterCommands("eval"), "should be equal")
+		assert.Equal(t, true, FilterCommands("evalsha"), "should be equal")
+		assert.Equal(t, true, FilterCommands("script"), "should be equal")
+
+	}
+}
+
+func TestFilterKey(t *testing.T) {
+	// test FilterKey
+
+	var nr int
+	{
+		fmt.Printf("TestFilterKey case %d.\n", nr)
+		nr++
+
+		assert.Equal(t, false, FilterKey("unknown-key"), "should be equal")
+	}
+
+	{
+		fmt.Printf("TestFilterKey case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterKeyBlacklist = []string{"abc", "xyz", "a"}
+		conf.Options.FilterKeyWhitelist = []string{}
+		assert.Equal(t, false, FilterKey("unknown-key"), "should be equal")
+		assert.Equal(t, true, FilterKey("abc"), "should be equal")
+		assert.Equal(t, true, FilterKey("abc111"), "should be equal")
+		assert.Equal(t, true, FilterKey("abcxyz"), "should be equal")
+		assert.Equal(t, true, FilterKey("xyz"), "should be equal")
+		assert.Equal(t, false, FilterKey("xy"), "should be equal")
+		assert.Equal(t, true, FilterKey("a"), "should be equal")
+		assert.Equal(t, true, FilterKey("ab"), "should be equal")
+	}
+
+	{
+		fmt.Printf("TestFilterKey case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{"abc", "xyz", "a"}
+		assert.Equal(t, true, FilterKey("unknown-key"), "should be equal")
+		assert.Equal(t, false, FilterKey("abc"), "should be equal")
+		assert.Equal(t, false, FilterKey("abc111"), "should be equal")
+		assert.Equal(t, false, FilterKey("abcxyz"), "should be equal")
+		assert.Equal(t, false, FilterKey("xyz"), "should be equal")
+		assert.Equal(t, true, FilterKey("xy"), "should be equal")
+		assert.Equal(t, false, FilterKey("a"), "should be equal")
+		assert.Equal(t, false, FilterKey("ab"), "should be equal")
+	}
+}
+
+func TestFilterSlot(t *testing.T) {
+	// test FilterSlot
+
+	var nr int
+	{
+		fmt.Printf("TestFilterSlot case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterSlot = []string{}
+		assert.Equal(t, false, FilterSlot(2), "should be equal")
+		assert.Equal(t, false, FilterSlot(0), "should be equal")
+	}
+
+	{
+		fmt.Printf("TestFilterSlot case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterSlot = []string{"1", "3", "5"}
+		assert.Equal(t, false, FilterSlot(1), "should be equal")
+		assert.Equal(t, true, FilterSlot(0), "should be equal")
+		assert.Equal(t, false, FilterSlot(5), "should be equal")
+	}
+}
+
+func TestFilterDB(t *testing.T) {
+	// test FilterDB
+
+	var nr int
+	{
+		fmt.Printf("TestFilterDB case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterDBWhitelist = []string{}
+		conf.Options.FilterDBBlacklist = []string{}
+		assert.Equal(t, false, FilterDB(2), "should be equal")
+		assert.Equal(t, false, FilterDB(0), "should be equal")
+	}
+
+	{
+		fmt.Printf("TestFilterDB case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterDBWhitelist = []string{"0", "1", "5"}
+		conf.Options.FilterDBBlacklist = []string{}
+		assert.Equal(t, true, FilterDB(2), "should be equal")
+		assert.Equal(t, false, FilterDB(0), "should be equal")
+		assert.Equal(t, false, FilterDB(5), "should be equal")
+
+	}
+
+	{
+		fmt.Printf("TestFilterDB case %d.\n", nr)
+		nr++
+
+		conf.Options.FilterDBWhitelist = []string{}
+		conf.Options.FilterDBBlacklist = []string{"0", "1", "5"}
+		assert.Equal(t, false, FilterDB(2), "should be equal")
+		assert.Equal(t, true, FilterDB(0), "should be equal")
+		assert.Equal(t, true, FilterDB(5), "should be equal")
+
+	}
+}
+
+func TestHandleFilterKeyWithCommand(t *testing.T) {
+	// test HandleFilterKeyWithCommand
+
+	var nr int
+	var cmd string
+	var args, expectArgs, ret [][]byte
+	var filter bool
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "set"
+		args = convertToByte("xyz", "1")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{"x", "y"}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, true, filter, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{"x"}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+	}
+
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "mset"
+		args = convertToByte("xyz", "1", "abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{"x"}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{"x"}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("xyz", "1")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+	}
+
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "msetnx"
+		args = convertToByte("xyz", "1", "abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{"x"}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("abc", "2", "ab", "3", "zzz", "1111111111111", "ffffffffff", "90")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{"x"}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("xyz", "1")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+	}
+
+	// unknown command, should pass
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "unknownCmd"
+		args = convertToByte("xyz", "1")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+	}
+
+	// length == 0
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "unknownCmd"
+		args = convertToByte("xyz")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+	}
+
+	// del
+	{
+		fmt.Printf("TestHandleFilterKeyWithCommand case %d.\n", nr)
+		nr++
+
+		cmd = "del"
+		args = convertToByte("xyz", "abc", "ab", "zzz", "ffffffffff")
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, args, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{"x"}
+		conf.Options.FilterKeyWhitelist = []string{}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("abc", "ab", "zzz", "ffffffffff")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+
+		conf.Options.FilterKeyBlacklist = []string{}
+		conf.Options.FilterKeyWhitelist = []string{"x"}
+		ret, filter = HandleFilterKeyWithCommand(cmd, args)
+		expectArgs = convertToByte("xyz")
+		assert.Equal(t, false, filter, "should be equal")
+		assert.Equal(t, expectArgs, ret, "should be equal")
+	}
+}
+
+func convertToByte(args... string) [][]byte {
+	ret := make([][]byte, 0)
+	for _, arg := range args {
+		ret = append(ret, []byte(arg))
+	}
+	return ret
+}

src/redis-shake/filter/redis_command.go

@@ -1,9 +1,5 @@
 // redis command struct.
-package command
-
-import (
-	"strings"
-)
+package filter
 
 type getkeys_proc func(args []string) []int
 type redisCommand struct {
@@ -86,7 +82,7 @@ var RedisCommands = map[string]redisCommand{
 	"pfmerge": {nil, 1, -1, 1},
 }
 
-func GetMatchKeys(redis_cmd redisCommand, args [][]byte, filterkeys []string) (new_args [][]byte, pass bool) {
+func getMatchKeys(redis_cmd redisCommand, args [][]byte) (new_args [][]byte, pass bool) {
 	lastkey := redis_cmd.lastkey - 1
 	keystep := redis_cmd.keystep
 
@@ -94,34 +90,32 @@ func GetMatchKeys(redis_cmd redisCommand, args [][]byte, filterkeys []string) (n
 		lastkey = lastkey + len(args)
 	}
 
-	array := make([]int, len(args))
-	number := 0
+	array := make([]int, len(args)) // store all positions of the pass key
+	number := 0                     // matching key number
 	for firstkey := redis_cmd.firstkey - 1; firstkey <= lastkey; firstkey += keystep {
 		key := string(args[firstkey])
-		for i := 0; i < len(filterkeys); i++ {
-			if strings.HasPrefix(key, filterkeys[i]) {
-				array[number] = firstkey
-				number++
-				break
-			}
+		if FilterKey(key) == false {
+			// pass
+			array[number] = firstkey
+			number++
 		}
 	}
 
 	pass = false
-	new_args = make([][]byte, number*redis_cmd.keystep+len(args)-lastkey-redis_cmd.keystep)
+	new_args = make([][]byte, number * redis_cmd.keystep + len(args) - lastkey - redis_cmd.keystep)
 	if number > 0 {
 		pass = true
 		for i := 0; i < number; i++ {
 			for j := 0; j < redis_cmd.keystep; j++ {
-				new_args[i*redis_cmd.keystep+j] = args[array[i]+j]
+				new_args[i * redis_cmd.keystep + j] = args[array[i] + j]
 			}
 		}
 	}
 
-	// add alis paramters
+	// add alias parameters
 	j := 0
 	for i := lastkey + redis_cmd.keystep; i < len(args); i++ {
-		new_args[number*redis_cmd.keystep+j] = args[i]
+		new_args[number * redis_cmd.keystep + j] = args[i]
 		j = j + 1
 	}
 

src/redis-shake/main/main.go

@@ -315,20 +315,23 @@ func sanitizeOptions(tp string) error {
 	}
 
 	if conf.Options.FilterDB != "" {
-		if n, err := strconv.ParseInt(conf.Options.FilterDB, 10, 32); err != nil {
-			return fmt.Errorf("parse FilterDB failed[%v]", err)
-		} else {
-			base.AcceptDB = func(db uint32) bool {
-				return db == uint32(n)
-			}
-		}
+		conf.Options.FilterDBWhitelist = []string{conf.Options.FilterDB}
+	}
+	if len(conf.Options.FilterDBWhitelist) != 0 && len(conf.Options.FilterDBBlacklist) != 0 {
+		return fmt.Errorf("only one of 'filter.db.whitelist' and 'filter.db.blacklist' can be given")
+	}
+
+	if len(conf.Options.FilterKey) != 0 {
+		conf.Options.FilterKeyWhitelist = conf.Options.FilterKey
+	}
+	if len(conf.Options.FilterKeyWhitelist) != 0 && len(conf.Options.FilterKeyBlacklist) != 0 {
+		return fmt.Errorf("only one of 'filter.key.whitelist' and 'filter.key.blacklist' can be given")
 	}
 
 	// if the target is "cluster", only allow pass db 0
 	if conf.Options.TargetType == conf.RedisTypeCluster {
-		base.AcceptDB = func(db uint32) bool {
-			return db == 0
-		}
+		conf.Options.FilterDBWhitelist = []string{"0"} // set whitelist = 0
+		conf.Options.FilterDBBlacklist = []string{}  // reset blacklist
 		log.Info("the target redis type is cluster, only pass db0")
 	}
 

src/redis-shake/restore.go

@@ -19,6 +19,7 @@ import (
 	"redis-shake/base"
 	"redis-shake/common"
 	"redis-shake/configure"
+	"redis-shake/filter"
 )
 
 type CmdRestore struct {
@@ -152,7 +153,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth
 				defer c.Close()
 				var lastdb uint32 = 0
 				for e := range pipe {
-					if !base.AcceptDB(e.DB) {
+					if filter.FilterDB(int(e.DB)) {
+						// filter db
 						dr.ignore.Incr()
 					} else {
 						dr.nentry.Incr()
@@ -167,7 +169,8 @@ func (dr *dbRestorer) restoreRDBFile(reader *bufio.Reader, target []string, auth
 								utils.SelectDB(c, lastdb)
 							}
 						}
-						if len(conf.Options.FilterKey) != 0 && !utils.HasAtLeastOnePrefix(string(e.Key), conf.Options.FilterKey) {
+
+						if filter.FilterKey(string(e.Key)) {
 							continue
 						}
 						utils.RestoreRdbEntry(c, e)
@@ -233,7 +236,7 @@ func (dr *dbRestorer) restoreCommand(reader *bufio.Reader, target []string, auth
 					if err != nil {
 						log.PanicErrorf(err, "routine[%v] parse db = %s failed", dr.id, s)
 					}
-					bypass = !base.AcceptDB(uint32(n))
+					bypass = filter.FilterDB(n)
 				}
 				if bypass {
 					dr.nbypass.Incr()

src/redis-shake/rump.go

@@ -15,6 +15,7 @@ import (
 	"redis-shake/scanner"
 
 	"github.com/garyburd/redigo/redis"
+	"redis-shake/filter"
 )
 
 type CmdRump struct {
@@ -301,6 +302,11 @@ func (dre *dbRumperExecutor) fetcher() {
 	log.Infof("dbRumper[%v] executor[%v] fetch db list: %v", dre.rumperId, dre.executorId, dbNumber)
 	// iterate all db nodes
 	for _, db := range dbNumber {
+		if filter.FilterDB(int(db)) {
+			log.Infof("dbRumper[%v] executor[%v] db[%v] filtered", dre.rumperId, dre.executorId, db)
+			continue
+		}
+
 		log.Infof("dbRumper[%v] executor[%v] fetch logical db: %v", dre.rumperId, dre.executorId, db)
 		if err := dre.doFetch(int(db)); err != nil {
 			log.Panic(err)
@@ -320,7 +326,7 @@ func (dre *dbRumperExecutor) writer() {
 	bucket := utils.StartQoS(conf.Options.Qps)
 	preDb := 0
 	for ele := range dre.keyChan {
-		if len(conf.Options.FilterKey) != 0 && !utils.HasAtLeastOnePrefix(ele.key, conf.Options.FilterKey) {
+		if filter.FilterKey(ele.key) {
 			continue
 		}
 		// QoS, limit the qps

src/redis-shake/sync.go

@@ -21,11 +21,11 @@ import (
 	"pkg/libs/log"
 	"pkg/redis"
 	"redis-shake/base"
-	"redis-shake/command"
 	"redis-shake/common"
 	"redis-shake/configure"
 	"redis-shake/heartbeat"
 	"redis-shake/metric"
+	"redis-shake/filter"
 )
 
 type delayNode struct {
@@ -409,7 +409,8 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type
 				defer c.Close()
 				var lastdb uint32 = 0
 				for e := range pipe {
-					if !base.AcceptDB(e.DB) {
+					if filter.FilterDB(int(e.DB)) {
+						// db filter
 						ds.ignore.Incr()
 					} else {
 						ds.nentry.Incr()
@@ -425,21 +426,19 @@ func (ds *dbSyncer) syncRDBFile(reader *bufio.Reader, target []string, auth_type
 							}
 						}
 
-						if len(conf.Options.FilterKey) != 0 {
-							if utils.HasAtLeastOnePrefix(string(e.Key), conf.Options.FilterKey) {
-								utils.RestoreRdbEntry(c, e)
-							}
-						} else if len(conf.Options.FilterSlot) > 0 {
-							for _, slot := range conf.Options.FilterSlot {
-								slotInt, _ := strconv.Atoi(slot)
-								if int(utils.KeyToSlot(string(e.Key))) == slotInt {
-									utils.RestoreRdbEntry(c, e)
-									break
-								}
-							}
+						if filter.FilterKey(string(e.Key)) == true {
+							// 1. judge if not pass filter key
+							ds.ignore.Incr()
+							continue
 						} else {
-							utils.RestoreRdbEntry(c, e)
+							slot := int(utils.KeyToSlot(string(e.Key)))
+							if filter.FilterSlot(slot) == true {
+								// 2. judge if not pass filter slot
+								ds.ignore.Incr()
+								continue
+							}
 						}
+						utils.RestoreRdbEntry(c, e)
 					}
 				}
 			}()
@@ -569,13 +568,15 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type
 	}()
 
 	go func() {
-		var lastdb int32 = 0
-		var bypass bool = false
-		var isselect bool = false
-
-		var scmd string
-		var argv, new_argv [][]byte
-		var err error
+		var (
+			lastdb        int32 = 0
+			bypass              = false
+			isselect            = false
+			scmd          string
+			argv, newArgv [][]byte
+			err           error
+			reject        bool
+		)
 
 		decoder := redis.NewDecoder(reader)
 
@@ -611,9 +612,9 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type
 						if err != nil {
 							log.PanicErrorf(err, "dbSyncer[%v] parse db = %s failed", ds.id, s)
 						}
-						bypass = !base.AcceptDB(uint32(n))
+						bypass = filter.FilterDB(n)
 						isselect = true
-					} else if utils.FilterCommands(scmd, conf.Options.FilterLua) {
+					} else if filter.FilterCommands(scmd) {
 						ignorecmd = true
 					}
 					if bypass || ignorecmd {
@@ -625,21 +626,8 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type
 					}
 				}
 
-				pass := false
-				if len(conf.Options.FilterKey) != 0 {
-					cmdNode, ok := command.RedisCommands[scmd]
-					if ok && len(argv) > 0 {
-						// log.Debugf("dbSyncer[%v] filter command[%v]", ds.id, scmd)
-						new_argv, pass = command.GetMatchKeys(cmdNode, argv, conf.Options.FilterKey)
-					} else {
-						pass = true
-						new_argv = argv
-					}
-				} else {
-					pass = true
-					new_argv = argv
-				}
-				if bypass || ignorecmd || !pass {
+				newArgv, reject = filter.HandleFilterKeyWithCommand(scmd, argv)
+				if bypass || ignorecmd || reject {
 					ds.nbypass.Incr()
 					metric.GetMetric(ds.id).AddBypassCmdCount(ds.id, 1)
 					log.Debugf("dbSyncer[%v] filter command[%v]", ds.id, scmd)
@@ -659,7 +647,7 @@ func (ds *dbSyncer) syncCommand(reader *bufio.Reader, target []string, auth_type
 				}
 				continue
 			}
-			ds.sendBuf <- cmdDetail{Cmd: scmd, Args: new_argv}
+			ds.sendBuf <- cmdDetail{Cmd: scmd, Args: newArgv}
 		}
 	}()