From 5b30e7bd1f964e53415135e0ec605bfb74aaaba7 Mon Sep 17 00:00:00 2001 From: suxb201 Date: Thu, 7 Jul 2022 21:07:37 +0800 Subject: [PATCH] support restore mode --- README.md | 7 +++-- build.sh | 3 ++- cmd/redis-shake/main.go | 11 +++++++- internal/config/config.go | 10 +++++--- internal/reader/rdb_reader.go | 40 +++++++++++++++++++++++++++++ redis-shake.toml | 3 ++- restore.toml | 48 +++++++++++++++++++++++++++++++++++ test/assets/empty.toml | 3 ++- 8 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 internal/reader/rdb_reader.go create mode 100644 restore.toml diff --git a/README.md b/README.md index f7e97bd..9c92894 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,13 @@ redis-shake is a tool for Redis data migration and provides a certain degree of ## Feature -* ⚡ high efficiency +* ⚡ High efficiency * 🌲 Native Redis data structure support * 🌐 Support single instance and cluster * ✅ Tested on Redis 5.0, Redis 6.0 and Redis 7.0 * 🤗 Supports custom filtering rules using lua * 💪 Support large instance migration +* 💖 Support restore mode and sync mode ![image.png](https://s2.loli.net/2022/06/30/vU346lVBrNofKzu.png) @@ -35,11 +36,13 @@ sh build.sh ## Usage -1. Edit redis-shake.toml and modify the source and target configuration items in it. +1. Edit `redis-shake.toml` or `restore.toml` and modify the source and target configuration items in it. 2. Start redis-shake. ```shell ./bin/redis-shake redis-shake.toml +# or +./bin/redis-shake restore.toml ``` 3. Check data synchronization status. diff --git a/build.sh b/build.sh index b2f1808..980ab7a 100755 --- a/build.sh +++ b/build.sh @@ -25,12 +25,13 @@ for g in "linux" "darwin"; do done cp redis-shake.toml "$BIN_DIR" +cp restore.toml "$BIN_DIR" if [ "$1" == "dist" ]; then echo "[ DIST ]" cd bin cp -r ../filters ./ - tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./redis-shake-* ./filters + tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./restore.toml ./redis-shake-* ./filters rm -rf ./filters cd .. fi diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 5c7add4..0e2aed3 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -63,7 +63,14 @@ func main() { // create reader source := &config.Config.Source - theReader := reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS) + var theReader reader.Reader + if source.Type == "sync" { + theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS) + } else if source.Type == "restore" { + theReader = reader.NewRDBReader(source.RDBFilePath) + } else { + log.Panicf("unknown source type: %s", source.Type) + } ch := theReader.StartRead() // start sync @@ -88,4 +95,6 @@ func main() { log.Panicf("error when run lua filter. entry: %s", e.ToString()) } } + + log.Infof("finished.") } diff --git a/internal/config/config.go b/internal/config/config.go index 4ff9900..1bbdf28 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,10 +10,12 @@ import ( ) type tomlSource struct { - Address string `toml:"address"` - Username string `toml:"username"` - Password string `toml:"password"` - IsTLS bool `toml:"tls"` + Type string `toml:"type"` + Address string `toml:"address"` + Username string `toml:"username"` + Password string `toml:"password"` + IsTLS bool `toml:"tls"` + RDBFilePath string `toml:"rdb_file_path"` } type tomlTarget struct { diff --git a/internal/reader/rdb_reader.go b/internal/reader/rdb_reader.go new file mode 100644 index 0000000..31e00d7 --- /dev/null +++ b/internal/reader/rdb_reader.go @@ -0,0 +1,40 @@ +package reader + +import ( + "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb" + "path/filepath" +) + +type rdbReader struct { + path string + ch chan *entry.Entry +} + +func NewRDBReader(path string) Reader { + log.Infof("NewRDBReader: path=[%s]", path) + absolutePath, err := filepath.Abs(path) + if err != nil { + log.Panicf("NewRDBReader: filepath.Abs error: %s", err.Error()) + } + log.Infof("NewRDBReader: absolute path=[%s]", absolutePath) + r := new(rdbReader) + r.path = absolutePath + return r +} + +func (r *rdbReader) StartRead() chan *entry.Entry { + r.ch = make(chan *entry.Entry, 1024) + + go func() { + // start parse rdb + log.Infof("start send RDB. path=[%s]", r.path) + rdbLoader := rdb.NewLoader(r.path, r.ch) + _ = rdbLoader.ParseRDB() + log.Infof("send RDB finished. path=[%s]", r.path) + close(r.ch) + }() + + return r.ch +} diff --git a/redis-shake.toml b/redis-shake.toml index 810f3d4..43b65df 100644 --- a/redis-shake.toml +++ b/redis-shake.toml @@ -1,4 +1,5 @@ -[source] # standalone +[source] +type = "sync" # sync or restore address = "127.0.0.1:6379" username = "" # keep empty if not using ACL password = "" # keep empty if no authentication is required diff --git a/restore.toml b/restore.toml new file mode 100644 index 0000000..81aaa5f --- /dev/null +++ b/restore.toml @@ -0,0 +1,48 @@ +[source] +type = "restore" # sync, restore +# Path to the dump.rdb file. Absolute path or relative path. Note +# that relative paths are relative to the dir directory. +rdb_file_path = "dump.rdb" + +[target] +type = "standalone" # standalone or cluster +# When the target is a cluster, write the address of one of the nodes. +# redis-shake will obtain other nodes through the `cluster nodes` command. +address = "127.0.0.1:6379" +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false + +[advanced] +dir = "data" + +# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores +ncpu = 3 + +# pprof port, 0 means disable +pprof_port = 0 + +# log +log_file = "redis-shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# pipeline +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/test/assets/empty.toml b/test/assets/empty.toml index 810f3d4..43b65df 100644 --- a/test/assets/empty.toml +++ b/test/assets/empty.toml @@ -1,4 +1,5 @@ -[source] # standalone +[source] +type = "sync" # sync or restore address = "127.0.0.1:6379" username = "" # keep empty if not using ACL password = "" # keep empty if no authentication is required