Merge pull request #11 from alibaba/inner_push_github

Inner push GitHub

.gitignore

@@ -10,8 +10,13 @@ logs
 *.yml
 *.pid
 *.tar.gz
+*.log
 tags
 result.db.*
+bin/*
+!/bin/redis-shake.*
+conf/*
+!conf/redis-shake.conf
 
 dump.data
 runtime.trace

ChangeLog

@@ -1,6 +1,9 @@
+2019-03-27 Alibaba Cloud.
+	* version: 1.2.1
+	* IMPROVE: support syncing lua script in RDB syncing.
 2019-03-11 Alibaba Cloud.
 	* version: 1.2.0
-	* redis-shake: support 5.0.
+	* IMPROVE: support 5.0.
 2019-02-21 Alibaba Cloud.
 	* version: 1.0.0
 	* redis-shake: initial release.

README.md

@@ -1,4 +1,4 @@
-Redis-shake is mainly used to synchronize data from one redis database to another.<br>
+RedisShake is mainly used to synchronize data from one redis database to another.<br>
 Thanks to the Douyu's WSD team for the support. <br>
 
 * [中文文档](https://yq.aliyun.com/articles/691794)
@@ -10,7 +10,7 @@ Redis-shake has made some improvements based on [redis-port](https://github.com/
 
 # Main Functions
 ---
-The type can be one of the following:<br>
+The type can be one of the followings:<br>
 
 * **decode**: Decode dumped payload to human readable format (hex-encoding).
 * **restore**: Restore RDB file to target redis.
@@ -21,7 +21,7 @@ Please check out the `conf/redis-shake.conf` to see the detailed parameters desc
 
 # Verification
 ---
-User can use [redis-full-check](https://github.com/aliyun/redis-full-check) to verify correctness.<br>
+User can use [RedisFullCheck](https://github.com/alibaba/RedisFullCheck) to verify correctness.<br>
 
 # Metric
 ---
@@ -63,8 +63,9 @@ Or you can build redis-shake yourself according to the following steps:
 ---
 We also provide some tools for synchronization in Shake series.<br>
 
-* [mongo-shake](https://github.com/aliyun/mongo-shake): mongodb data synchronization tool.
-* [redis-shake](https://github.com/aliyun/redis-shake): redis data synchronization tool.
-* [redis-full-check](https://github.com/aliyun/redis-full-check): redis data synchronization verification tool.
+* [MongoShake](https://github.com/aliyun/MongoShake): mongodb data synchronization tool.
+* [RedisShake](https://github.com/aliyun/RedisShake): redis data synchronization tool.
+* [RedisFullCheck](https://github.com/aliyun/RedisFullCheck): redis data synchronization verification tool.
 
 Plus, we have a WeChat group so that users can join and discuss, but the group user number is limited. So please add my WeChat number: `vinllen_xingge` first, and I will add you to this group.<br>
+

src/pkg/rdb/loader.go

@@ -121,10 +121,18 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) {
 			t = rtype
 		}
 		switch t {
-		case rdbFlagAUX:
+		case RdbFlagAUX:
 			aux_key, _ := l.ReadString()
 			aux_value, _ := l.ReadString()
 			log.Info("Aux information key:", string(aux_key), " value:", string(aux_value))
+			if string(aux_key) == "lua" {
+				// we should handle the lua script
+				entry.DB = l.db
+				entry.Key = aux_key
+				entry.Type = t
+				entry.Value = aux_value
+				return entry, nil
+			}
 		case rdbFlagResizeDB:
 			db_size, _ := l.ReadLength()
 			expire_size, _ := l.ReadLength()

src/pkg/rdb/reader.go

@@ -36,7 +36,7 @@ const (
 	RDBTypeStreamListPacks = 15 // stream
 
 	rdbFlagOnlyValue = 0xf9
-	rdbFlagAUX       = 0xfa
+	RdbFlagAUX       = 0xfa
 	rdbFlagResizeDB  = 0xfb
 	rdbFlagExpiryMS  = 0xfc
 	rdbFlagExpiry    = 0xfd
@@ -98,7 +98,7 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) {
 	switch t {
 	default:
 		return nil, errors.Errorf("unknown object-type %02x", t)
-	case rdbFlagAUX:
+	case RdbFlagAUX:
 		fallthrough
 	case rdbFlagResizeDB:
 		fallthrough

src/redis-shake/common/common.go

@@ -6,8 +6,8 @@ import(
 )
 
 const(
-	// GolangSecurityTime = "2006-01-02T15:04:05Z"
-	GolangSecurityTime = "2006-01-02 15:04:05"
+	GolangSecurityTime = "2006-01-02T15:04:05Z"
+	// GolangSecurityTime = "2006-01-02 15:04:05"
 	ReaderBufferSize = bytesize.MB * 32
 	WriterBufferSize = bytesize.MB * 8
 )
@@ -15,4 +15,5 @@ const(
 var(
 	Version = "$"
 	LogRotater *logRotate.Logger
+	StartTime string
 )

src/redis-shake/common/utils.go

@@ -679,6 +679,15 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
 		return
 	}
 
+	// load lua script
+	if e.Type == rdb.RdbFlagAUX && string(e.Key) == "lua" {
+		_, err := c.Do("script", "load", e.Value)
+		if err != nil {
+			log.Panicf(err.Error())
+		}
+		return
+	}
+
 	// TODO, need to judge big key
 	if e.Type != rdb.RDBTypeStreamListPacks &&
 			(uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) {

src/redis-shake/main/main.go

@@ -81,6 +81,7 @@ func main() {
 	initFreeOS()
 	nimo.Profiling(int(conf.Options.SystemProfile))
 	utils.Welcome()
+	utils.StartTime = fmt.Sprintf("%v", time.Now().Format(utils.GolangSecurityTime))
 
 	if err = utils.WritePidById(conf.Options.Id); err != nil {
 		crash(fmt.Sprintf("write pid failed. %v", err), -5)

src/redis-shake/metric/variables.go

@@ -3,9 +3,11 @@ package metric
 import(
 	"fmt"
 	"redis-shake/base"
+	"redis-shake/common"
 )
 
 type MetricRest struct {
+	StartTime            interface{}
 	PullCmdCount         interface{}
 	PullCmdCountTotal    interface{}
 	BypassCmdCount       interface{}
@@ -39,6 +41,7 @@ func NewMetricRest() *MetricRest {
 	sourceDbOffset := detailedInfo[3]
 
 	return &MetricRest{
+		StartTime:            utils.StartTime,
 		PullCmdCount:         MetricVar.GetPullCmdCount(),
 		PullCmdCountTotal:    MetricVar.GetPullCmdCountTotal(),
 		BypassCmdCount:       MetricVar.GetBypassCmdCount(),