support restore mode

v4
suxb201 2 years ago
parent 4df84993f7
commit 5b30e7bd1f
  1. 7
      README.md
  2. 3
      build.sh
  3. 11
      cmd/redis-shake/main.go
  4. 10
      internal/config/config.go
  5. 40
      internal/reader/rdb_reader.go
  6. 3
      redis-shake.toml
  7. 48
      restore.toml
  8. 3
      test/assets/empty.toml

@ -8,12 +8,13 @@ redis-shake is a tool for Redis data migration and provides a certain degree of
## Feature
* ⚡ high efficiency
* ⚡ High efficiency
* 🌲 Native Redis data structure support
* 🌐 Support single instance and cluster
* ✅ Tested on Redis 5.0, Redis 6.0 and Redis 7.0
* 🤗 Supports custom filtering rules using lua
* 💪 Support large instance migration
* 💖 Support restore mode and sync mode
![image.png](https://s2.loli.net/2022/06/30/vU346lVBrNofKzu.png)
@ -35,11 +36,13 @@ sh build.sh
## Usage
1. Edit redis-shake.toml and modify the source and target configuration items in it.
1. Edit `redis-shake.toml` or `restore.toml` and modify the source and target configuration items in it.
2. Start redis-shake.
```shell
./bin/redis-shake redis-shake.toml
# or
./bin/redis-shake restore.toml
```
3. Check data synchronization status.

@ -25,12 +25,13 @@ for g in "linux" "darwin"; do
done
cp redis-shake.toml "$BIN_DIR"
cp restore.toml "$BIN_DIR"
if [ "$1" == "dist" ]; then
echo "[ DIST ]"
cd bin
cp -r ../filters ./
tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./redis-shake-* ./filters
tar -czvf ./redis-shake.tar.gz ./redis-shake.toml ./restore.toml ./redis-shake-* ./filters
rm -rf ./filters
cd ..
fi

@ -63,7 +63,14 @@ func main() {
// create reader
source := &config.Config.Source
theReader := reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS)
var theReader reader.Reader
if source.Type == "sync" {
theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS)
} else if source.Type == "restore" {
theReader = reader.NewRDBReader(source.RDBFilePath)
} else {
log.Panicf("unknown source type: %s", source.Type)
}
ch := theReader.StartRead()
// start sync
@ -88,4 +95,6 @@ func main() {
log.Panicf("error when run lua filter. entry: %s", e.ToString())
}
}
log.Infof("finished.")
}

@ -10,10 +10,12 @@ import (
)
type tomlSource struct {
Address string `toml:"address"`
Username string `toml:"username"`
Password string `toml:"password"`
IsTLS bool `toml:"tls"`
Type string `toml:"type"`
Address string `toml:"address"`
Username string `toml:"username"`
Password string `toml:"password"`
IsTLS bool `toml:"tls"`
RDBFilePath string `toml:"rdb_file_path"`
}
type tomlTarget struct {

@ -0,0 +1,40 @@
package reader
import (
"github.com/alibaba/RedisShake/internal/entry"
"github.com/alibaba/RedisShake/internal/log"
"github.com/alibaba/RedisShake/internal/rdb"
"path/filepath"
)
type rdbReader struct {
path string
ch chan *entry.Entry
}
func NewRDBReader(path string) Reader {
log.Infof("NewRDBReader: path=[%s]", path)
absolutePath, err := filepath.Abs(path)
if err != nil {
log.Panicf("NewRDBReader: filepath.Abs error: %s", err.Error())
}
log.Infof("NewRDBReader: absolute path=[%s]", absolutePath)
r := new(rdbReader)
r.path = absolutePath
return r
}
func (r *rdbReader) StartRead() chan *entry.Entry {
r.ch = make(chan *entry.Entry, 1024)
go func() {
// start parse rdb
log.Infof("start send RDB. path=[%s]", r.path)
rdbLoader := rdb.NewLoader(r.path, r.ch)
_ = rdbLoader.ParseRDB()
log.Infof("send RDB finished. path=[%s]", r.path)
close(r.ch)
}()
return r.ch
}

@ -1,4 +1,5 @@
[source] # standalone
[source]
type = "sync" # sync or restore
address = "127.0.0.1:6379"
username = "" # keep empty if not using ACL
password = "" # keep empty if no authentication is required

@ -0,0 +1,48 @@
[source]
type = "restore" # sync, restore
# Path to the dump.rdb file. Absolute path or relative path. Note
# that relative paths are relative to the dir directory.
rdb_file_path = "dump.rdb"
[target]
type = "standalone" # standalone or cluster
# When the target is a cluster, write the address of one of the nodes.
# redis-shake will obtain other nodes through the `cluster nodes` command.
address = "127.0.0.1:6379"
username = "" # keep empty if not using ACL
password = "" # keep empty if no authentication is required
tls = false
[advanced]
dir = "data"
# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
ncpu = 3
# pprof port, 0 means disable
pprof_port = 0
# log
log_file = "redis-shake.log"
log_level = "info" # debug, info or warn
log_interval = 5 # in seconds
# redis-shake gets key and value from rdb file, and uses RESTORE command to
# create the key in target redis. Redis RESTORE will return a "Target key name
# is busy" error when key already exists. You can use this configuration item
# to change the default behavior of restore:
# panic: redis-shake will stop when meet "Target key name is busy" error.
# rewrite: redis-shake will replace the key with new value.
# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error.
rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip
# pipeline
pipeline_count_limit = 1024
# Client query buffers accumulate new commands. They are limited to a fixed
# amount by default. This amount is normally 1gb.
target_redis_client_max_querybuf_len = 1024_000_000
# In the Redis protocol, bulk requests, that are, elements representing single
# strings, are normally limited to 512 mb.
target_redis_proto_max_bulk_len = 512_000_000

@ -1,4 +1,5 @@
[source] # standalone
[source]
type = "sync" # sync or restore
address = "127.0.0.1:6379"
username = "" # keep empty if not using ACL
password = "" # keep empty if no authentication is required

Loading…
Cancel
Save