support restore mode

README.md

@@ -8,12 +8,13 @@ redis-shake is a tool for Redis data migration and provides a certain degree of
 
 ## Feature
 
-* ⚡ high efficiency
+* ⚡ High efficiency
 * 🌲 Native Redis data structure support
 * 🌐 Support single instance and cluster
 * ✅ Tested on Redis 5.0, Redis 6.0 and Redis 7.0
 * 🤗 Supports custom filtering rules using lua
 * 💪 Support large instance migration
+* 💖 Support restore mode and sync mode
 
 ![image.png](https://s2.loli.net/2022/06/30/vU346lVBrNofKzu.png)
 
@@ -35,11 +36,13 @@ sh build.sh
 
 ## Usage
 
-1. Edit redis-shake.toml and modify the source and target configuration items in it.
+1. Edit `redis-shake.toml` or `restore.toml` and modify the source and target configuration items in it.
 2. Start redis-shake.
 
 ```shell
 ./bin/redis-shake redis-shake.toml
+# or
+./bin/redis-shake restore.toml
 ```
 
 3. Check data synchronization status.

build.sh

@@ -25,12 +25,13 @@ for g in "linux" "darwin"; do
 done
 
 cp redis-shake.toml "$BIN_DIR"
+cp restore.toml "$BIN_DIR"
 
 if [ "$1" == "dist" ]; then
   echo "[ DIST ]"
   cd bin
   cp -r ../filters ./
-  tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./redis-shake-* ./filters
+  tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./restore.toml ./redis-shake-* ./filters
   rm -rf ./filters
   cd ..
 fi

cmd/redis-shake/main.go

@@ -63,7 +63,14 @@ func main() {
 
 	// create reader
 	source := &config.Config.Source
-	theReader := reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS)
+	var theReader reader.Reader
+	if source.Type == "sync" {
+		theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS)
+	} else if source.Type == "restore" {
+		theReader = reader.NewRDBReader(source.RDBFilePath)
+	} else {
+		log.Panicf("unknown source type: %s", source.Type)
+	}
 	ch := theReader.StartRead()
 
 	// start sync
@@ -88,4 +95,6 @@ func main() {
 			log.Panicf("error when run lua filter. entry: %s", e.ToString())
 		}
 	}
+
+	log.Infof("finished.")
 }

internal/config/config.go

@@ -10,10 +10,12 @@ import (
 )
 
 type tomlSource struct {
-	Address  string `toml:"address"`
-	Username string `toml:"username"`
-	Password string `toml:"password"`
-	IsTLS    bool   `toml:"tls"`
+	Type        string `toml:"type"`
+	Address     string `toml:"address"`
+	Username    string `toml:"username"`
+	Password    string `toml:"password"`
+	IsTLS       bool   `toml:"tls"`
+	RDBFilePath string `toml:"rdb_file_path"`
 }
 
 type tomlTarget struct {

internal/reader/rdb_reader.go

@@ -0,0 +1,40 @@
+package reader
+
+import (
+	"github.com/alibaba/RedisShake/internal/entry"
+	"github.com/alibaba/RedisShake/internal/log"
+	"github.com/alibaba/RedisShake/internal/rdb"
+	"path/filepath"
+)
+
+type rdbReader struct {
+	path string
+	ch   chan *entry.Entry
+}
+
+func NewRDBReader(path string) Reader {
+	log.Infof("NewRDBReader: path=[%s]", path)
+	absolutePath, err := filepath.Abs(path)
+	if err != nil {
+		log.Panicf("NewRDBReader: filepath.Abs error: %s", err.Error())
+	}
+	log.Infof("NewRDBReader: absolute path=[%s]", absolutePath)
+	r := new(rdbReader)
+	r.path = absolutePath
+	return r
+}
+
+func (r *rdbReader) StartRead() chan *entry.Entry {
+	r.ch = make(chan *entry.Entry, 1024)
+
+	go func() {
+		// start parse rdb
+		log.Infof("start send RDB. path=[%s]", r.path)
+		rdbLoader := rdb.NewLoader(r.path, r.ch)
+		_ = rdbLoader.ParseRDB()
+		log.Infof("send RDB finished. path=[%s]", r.path)
+		close(r.ch)
+	}()
+
+	return r.ch
+}

redis-shake.toml

@@ -1,4 +1,5 @@
-[source] # standalone
+[source]
+type = "sync" # sync or restore
 address = "127.0.0.1:6379"
 username = "" # keep empty if not using ACL
 password = "" # keep empty if no authentication is required

restore.toml

@@ -0,0 +1,48 @@
+[source]
+type = "restore" # sync, restore
+# Path to the dump.rdb file. Absolute path or relative path. Note
+# that relative paths are relative to the dir directory.
+rdb_file_path = "dump.rdb"
+
+[target]
+type = "standalone" # standalone or cluster
+# When the target is a cluster, write the address of one of the nodes.
+# redis-shake will obtain other nodes through the `cluster nodes` command.
+address = "127.0.0.1:6379"
+username = "" # keep empty if not using ACL
+password = "" # keep empty if no authentication is required
+tls = false
+
+[advanced]
+dir = "data"
+
+# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
+ncpu = 3
+
+# pprof port, 0 means disable
+pprof_port = 0
+
+# log
+log_file = "redis-shake.log"
+log_level = "info" # debug, info or warn
+log_interval = 5 # in seconds
+
+# redis-shake gets key and value from rdb file, and uses RESTORE command to
+# create the key in target redis. Redis RESTORE will return a "Target key name
+# is busy" error when key already exists. You can use this configuration item
+# to change the default behavior of restore:
+# panic:   redis-shake will stop when meet "Target key name is busy" error.
+# rewrite: redis-shake will replace the key with new value.
+# ignore:  redis-shake will skip restore the key when meet "Target key name is busy" error.
+rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip
+
+# pipeline
+pipeline_count_limit = 1024
+
+# Client query buffers accumulate new commands. They are limited to a fixed
+# amount by default. This amount is normally 1gb.
+target_redis_client_max_querybuf_len = 1024_000_000
+
+# In the Redis protocol, bulk requests, that are, elements representing single
+# strings, are normally limited to 512 mb.
+target_redis_proto_max_bulk_len = 512_000_000

test/assets/empty.toml

@@ -1,4 +1,5 @@
-[source] # standalone
+[source]
+type = "sync" # sync or restore
 address = "127.0.0.1:6379"
 username = "" # keep empty if not using ACL
 password = "" # keep empty if no authentication is required