From 6bbdb15f6750437e6c488665bb2a4ba5ddd0aefd Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 16:44:39 +0800 Subject: [PATCH 01/27] initial release --- .gitignore | 26 + ChangeLog | 3 + MIT-LICENSE.txt | 21 + Makefile | 11 + README.md | 55 ++ build.sh | 40 + conf/redis-shake.conf | 116 +++ src/pkg/libs/assert/assert.go | 20 + src/pkg/libs/atomic2/atomic64.go | 49 + src/pkg/libs/atomic2/bool.go | 32 + src/pkg/libs/bytesize/bytesize.go | 85 ++ src/pkg/libs/bytesize/bytesize_test.go | 46 + src/pkg/libs/cupcake/rdb/LICENCE | 21 + src/pkg/libs/cupcake/rdb/README.md | 17 + src/pkg/libs/cupcake/rdb/crc64/crc64.go | 64 ++ src/pkg/libs/cupcake/rdb/decoder.go | 860 +++++++++++++++++ src/pkg/libs/cupcake/rdb/decoder_test.go | 347 +++++++ src/pkg/libs/cupcake/rdb/encoder.go | 130 +++ src/pkg/libs/cupcake/rdb/encoder_test.go | 43 + src/pkg/libs/cupcake/rdb/examples/diff.go | 65 ++ .../cupcake/rdb/nopdecoder/nop_decoder.go | 24 + src/pkg/libs/cupcake/rdb/slice_buffer.go | 67 ++ src/pkg/libs/errors/errors.go | 90 ++ src/pkg/libs/errors/list.go | 60 ++ src/pkg/libs/fmt2/strconv.go | 159 ++++ src/pkg/libs/io/backlog/backlog.go | 225 +++++ src/pkg/libs/io/backlog/backlog_test.go | 106 +++ src/pkg/libs/io/backlog/buff.go | 68 ++ src/pkg/libs/io/backlog/file.go | 76 ++ src/pkg/libs/io/pipe/buff.go | 82 ++ src/pkg/libs/io/pipe/file.go | 90 ++ src/pkg/libs/io/pipe/pipe.go | 217 +++++ src/pkg/libs/io/pipe/pipe_test.go | 391 ++++++++ src/pkg/libs/io/pipe/pipeio.go | 63 ++ src/pkg/libs/log/log.go | 608 ++++++++++++ src/pkg/libs/oplog/cmd.go | 810 ++++++++++++++++ src/pkg/libs/oplog/infooplog.go | 87 ++ src/pkg/libs/oplog/oplog.go | 274 ++++++ src/pkg/libs/oplog/parseinfo.go | 36 + src/pkg/libs/stats/iocount.go | 62 ++ src/pkg/libs/trace/trace.go | 88 ++ src/pkg/rdb/decoder.go | 156 ++++ src/pkg/rdb/decoder_test.go | 190 ++++ src/pkg/rdb/digest/crc64.go | 106 +++ src/pkg/rdb/encoder.go | 170 ++++ src/pkg/rdb/encoder_test.go | 293 ++++++ src/pkg/rdb/loader.go | 198 ++++ src/pkg/rdb/loader_test.go | 360 ++++++++ src/pkg/rdb/reader.go | 521 +++++++++++ src/pkg/rdb/slice_buffer.go | 67 ++ src/pkg/redis/decoder.go | 192 ++++ src/pkg/redis/decoder_test.go | 112 +++ src/pkg/redis/encoder.go | 167 ++++ src/pkg/redis/encoder_test.go | 68 ++ src/pkg/redis/handler.go | 117 +++ src/pkg/redis/resp.go | 185 ++++ src/pkg/redis/server.go | 39 + src/pkg/redis/server_test.go | 68 ++ src/redis-shake/base/runner.go | 14 + src/redis-shake/command/redis-command.go | 128 +++ src/redis-shake/command/redis-command_test.go | 125 +++ src/redis-shake/common/common.go | 18 + src/redis-shake/common/crc16.go | 88 ++ src/redis-shake/common/http.go | 13 + src/redis-shake/common/mix.go | 72 ++ src/redis-shake/common/slot.go | 19 + src/redis-shake/common/utils.go | 864 ++++++++++++++++++ src/redis-shake/configure/configure.go | 56 ++ src/redis-shake/decode.go | 241 +++++ src/redis-shake/dump.go | 120 +++ src/redis-shake/heartbeat/heartbeat.go | 85 ++ src/redis-shake/main/main.go | 317 +++++++ src/redis-shake/metric/metric.go | 227 +++++ src/redis-shake/metric/variables.go | 63 ++ src/redis-shake/reader/reader.go | 61 ++ src/redis-shake/restful/restful.go | 19 + src/redis-shake/restore.go | 206 +++++ src/redis-shake/sync.go | 590 ++++++++++++ wandoujia_license.txt | 21 + 79 files changed, 12090 insertions(+) create mode 100644 .gitignore create mode 100644 ChangeLog create mode 100644 MIT-LICENSE.txt create mode 100644 Makefile create mode 100644 README.md create mode 100755 build.sh create mode 100644 conf/redis-shake.conf create mode 100644 src/pkg/libs/assert/assert.go create mode 100644 src/pkg/libs/atomic2/atomic64.go create mode 100644 src/pkg/libs/atomic2/bool.go create mode 100644 src/pkg/libs/bytesize/bytesize.go create mode 100644 src/pkg/libs/bytesize/bytesize_test.go create mode 100644 src/pkg/libs/cupcake/rdb/LICENCE create mode 100644 src/pkg/libs/cupcake/rdb/README.md create mode 100644 src/pkg/libs/cupcake/rdb/crc64/crc64.go create mode 100644 src/pkg/libs/cupcake/rdb/decoder.go create mode 100644 src/pkg/libs/cupcake/rdb/decoder_test.go create mode 100644 src/pkg/libs/cupcake/rdb/encoder.go create mode 100644 src/pkg/libs/cupcake/rdb/encoder_test.go create mode 100644 src/pkg/libs/cupcake/rdb/examples/diff.go create mode 100644 src/pkg/libs/cupcake/rdb/nopdecoder/nop_decoder.go create mode 100644 src/pkg/libs/cupcake/rdb/slice_buffer.go create mode 100644 src/pkg/libs/errors/errors.go create mode 100644 src/pkg/libs/errors/list.go create mode 100644 src/pkg/libs/fmt2/strconv.go create mode 100644 src/pkg/libs/io/backlog/backlog.go create mode 100644 src/pkg/libs/io/backlog/backlog_test.go create mode 100644 src/pkg/libs/io/backlog/buff.go create mode 100644 src/pkg/libs/io/backlog/file.go create mode 100644 src/pkg/libs/io/pipe/buff.go create mode 100644 src/pkg/libs/io/pipe/file.go create mode 100644 src/pkg/libs/io/pipe/pipe.go create mode 100644 src/pkg/libs/io/pipe/pipe_test.go create mode 100644 src/pkg/libs/io/pipe/pipeio.go create mode 100644 src/pkg/libs/log/log.go create mode 100644 src/pkg/libs/oplog/cmd.go create mode 100644 src/pkg/libs/oplog/infooplog.go create mode 100644 src/pkg/libs/oplog/oplog.go create mode 100644 src/pkg/libs/oplog/parseinfo.go create mode 100644 src/pkg/libs/stats/iocount.go create mode 100644 src/pkg/libs/trace/trace.go create mode 100644 src/pkg/rdb/decoder.go create mode 100644 src/pkg/rdb/decoder_test.go create mode 100644 src/pkg/rdb/digest/crc64.go create mode 100644 src/pkg/rdb/encoder.go create mode 100644 src/pkg/rdb/encoder_test.go create mode 100644 src/pkg/rdb/loader.go create mode 100644 src/pkg/rdb/loader_test.go create mode 100644 src/pkg/rdb/reader.go create mode 100644 src/pkg/rdb/slice_buffer.go create mode 100644 src/pkg/redis/decoder.go create mode 100644 src/pkg/redis/decoder_test.go create mode 100644 src/pkg/redis/encoder.go create mode 100644 src/pkg/redis/encoder_test.go create mode 100644 src/pkg/redis/handler.go create mode 100644 src/pkg/redis/resp.go create mode 100644 src/pkg/redis/server.go create mode 100644 src/pkg/redis/server_test.go create mode 100644 src/redis-shake/base/runner.go create mode 100644 src/redis-shake/command/redis-command.go create mode 100644 src/redis-shake/command/redis-command_test.go create mode 100644 src/redis-shake/common/common.go create mode 100644 src/redis-shake/common/crc16.go create mode 100644 src/redis-shake/common/http.go create mode 100644 src/redis-shake/common/mix.go create mode 100644 src/redis-shake/common/slot.go create mode 100644 src/redis-shake/common/utils.go create mode 100644 src/redis-shake/configure/configure.go create mode 100644 src/redis-shake/decode.go create mode 100644 src/redis-shake/dump.go create mode 100644 src/redis-shake/heartbeat/heartbeat.go create mode 100644 src/redis-shake/main/main.go create mode 100644 src/redis-shake/metric/metric.go create mode 100644 src/redis-shake/metric/variables.go create mode 100644 src/redis-shake/reader/reader.go create mode 100644 src/redis-shake/restful/restful.go create mode 100644 src/redis-shake/restore.go create mode 100644 src/redis-shake/sync.go create mode 100644 wandoujia_license.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dbc5657 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +*.log +*.rdb +*.sw[ap] +*.out +*.aof +*.dump +*.log.* +.idea +bin + +.DS_Store + +/Godeps/_workspace +/cmd/version.go +.htaccess + +tags + +logs +.cache/ +diagnostic/ +*.pid +src/vendor/ +!vendor.json + +*.tar.gz diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..6400947 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,3 @@ +2019-02-21 Alibaba Cloud. + * version: 1.0.0 + * mongo-shake: initial release. diff --git a/MIT-LICENSE.txt b/MIT-LICENSE.txt new file mode 100644 index 0000000..c0ca991 --- /dev/null +++ b/MIT-LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 CodisLabs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9fedcd6 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +all: + ./build.sh + +clean: + rm -rf bin + rm -rf *.pprof + rm -rf *.output + rm -rf logs + rm -rf diagnostic/ + rm -rf *.pid + diff --git a/README.md b/README.md new file mode 100644 index 0000000..c64e484 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +Redis-shake is mainly used to synchronize data from one redis databases to another.
+ +# Redis-Shake +--- +Redis-shake is developed and maintained by NoSQL Team in Alibaba-Cloud Database department.
+Redis-shake has made some improvements based on [redis-port](https://github.com/CodisLabs/redis-port), including bug fixes, performance improvements and feature enhancements.
+ +# Main Functions +--- +The type can be one of the following:
+ +* **decode**: Decode dumped payload to human readable format (hex-encoding). +* **restore**: Restore RDB file to target redis. +* **dump**: Dump RDB file from souce redis. +* **sync**: Sync data from source redis to target redis. + +Please check out the `conf/redis-shake.conf` to see the detailed parameters description.
+ +# Verification +--- +User can use [redis-full-check](https://github.com/aliyun/redis-full-check) to verify correctness.
+ +# Metric +--- +Redis-shake offers metrics through restful api and log file.
+ +* restful api: `curl 127.0.0.1:9320/metric`. +* log: the metric info will be printed in the log periodically if enable. + +# Code branch rules +--- +Version rules: a.b.c. + +* a: major version +* b: minor version. even number means stable version. +* c: bugfix version + +| branch name | rules | +| - | :- | +| master | master branch, do not allowed push code. store the latest stable version. develop branch will merge into this branch once new version created.| +| develop | develop branch. all the bellowing branches fork from this. | +| feature-\* | new feature branch. forked from develop branch and then merge back after finish developing, testing, and code review. | +| bugfix-\* | bugfix branch. forked from develop branch and then merge back after finish developing, testing, and code review. | +| improve-\* | improvement branch. forked from develop branch and then merge back after finish developing, testing, and code review. | + +Tag rules:
+Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0.2-20180628" + +# Usage +--- +* git clone https://github.com/aliyun/redis-shake.git +* cd redis-shake/src/vendor +* GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies +* cd ../../ && ./build.sh +* ./bin/collector -type=$(type_must_be_sync,_dump,_restore_or_decode) -conf=conf/redis-shake.conf #please note: user must modify collector.conf first to match needs. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..f20416e --- /dev/null +++ b/build.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -o errexit + +# older version Git don't support --short ! +if [ -d ".git" ];then + #branch=`git symbolic-ref --short -q HEAD` + branch=$(git symbolic-ref -q HEAD | awk -F'/' '{print $3;}') + cid=$(git rev-parse HEAD) +else + branch="unknown" + cid="0.0" +fi +branch=$branch","$cid + +# make sure we're in the directory where the script lives +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +GOPATH=$(pwd) +export GOPATH + +info="redis-shake/common.Version=$branch" +# golang version +goversion=$(go version | awk -F' ' '{print $3;}') +info=$info","$goversion +bigVersion=$(echo $goversion | awk -F'[o.]' '{print $2}') +midVersion=$(echo $goversion | awk -F'[o.]' '{print $3}') +if [ $bigVersion -lt "1" -o $bigVersion -eq "1" -a $midVersion -lt "6" ]; then + echo "go version[$goversion] must >= 1.6" + exit 1 +fi + +t=$(date "+%Y-%m-%d_%H:%M:%S") +info=$info","$t + +echo "[ BUILD RELEASE ]" +run_builder='go build -v' +$run_builder -ldflags "-X $info" -o "bin/redis-shake" "./src/redis-shake/main/main.go" +echo "build successfully!" diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf new file mode 100644 index 0000000..92bca58 --- /dev/null +++ b/conf/redis-shake.conf @@ -0,0 +1,116 @@ +# this is the configuration of redis-shake. + +# id +id = redis-shake +# log file +log_file = + +# pprof port +system_profile = 9310 +# restful port +http_profile = 9320 + +# runtime.GOMAXPROCS, 0 means use cpu core number: runtime.NumCPU() +ncpu = 0 + +# parallel routines number used in RDB file syncing. +parallel = 4 + +# input RDB file. read from stdin, default is stdin ('/dev/stdin'). +# used in `decode` and `restore`. +input_rdb = local_dump + +# output RDB file. default is stdout ('/dev/stdout'). +# used in `decode` and `dump`. +output_rdb = local_dump + +# source redis configuration. +# used in `dump` and `sync`. +# ip:port +source.address = 10.101.72.137:20441 +# password. +source.password_raw = kLNIl691OZctWST +# auth type, don't modify it +source.auth_type = auth +# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) +source.version = 6 + +# target redis configuration. used in `restore` and `sync`. +# used in `restore` and `sync`. +# ip:port +target.address = 10.101.72.137:20551 +# password. +target.password_raw = kLNIl691OZctWST +# auth type, don't modify it +target.auth_type = auth +# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) +target.version = 6 +# all the data will come into this db. < 0 means disable. +# used in `restore` and `sync`. +target.db = -1 + +# use for expire key, set the time gap when source and target timestamp are not the same. +fake_time = + +# force rewrite when destination restore has the key +# used in `restore` and `sync`. +rewrite = true + +# filter db or key or slot +# choose these db, e.g., 5, only choose db5. defalut is all. +# used in `restore` and `sync`. +filter.db = +# filter key with prefix string. multiple keys are separated by ';'. +# e.g., a;b;c +# default is all. +# used in `restore` and `sync`. +filter.key = +# filter given slot, multiple slots are separated by ';'. +# e.g., 1;2;3 +# used in `sync`. +filter.slot = + +# big key threshold, default is 500 * 1024 * 1024. The field of big key will be split in processing. +big_key_threshold = 524288000 + +# use psync command. +# used in `sync`. +psync = false + +# enable metric +# used in `sync`. +metric = true +# print in log +metric.print_log = true + +# heartbeat +# send heartbeat to this url +# used in `sync`. +heartbeat.url = http://127.0.0.1:8000 +# interval by seconds +heartbeat.interval = 3 +# external info which will be included in heartbeat data. +heartbeat.external = test external +# local network card to get ip address, e.g., "lo", "eth0", "en0" +heartbeat.network_interface = + +# sender information. +# sender flush buffer size of byte. +# used in `sync`. +sender.size = 104857600 +# sender flush buffer size of oplog number. +# used in `sync`. +sender.count = 5000 +# delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also stored in this delay queue. this timestamp will be used to calculate the time delay when receive ack from target redis. +# used in `sync`. +sender.delay_channel_size = 65535 + +# ----------------splitter---------------- +# belowing variables are useless for current opensource version so don't set. + +# replace hash tag. +# used in `sync`. +replace_hash_tag = false + +# used in `restore` and `dump`. +extra = false diff --git a/src/pkg/libs/assert/assert.go b/src/pkg/libs/assert/assert.go new file mode 100644 index 0000000..59f045a --- /dev/null +++ b/src/pkg/libs/assert/assert.go @@ -0,0 +1,20 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package assert + +import "pkg/libs/log" + +func Must(b bool) { + if b { + return + } + log.Panic("assertion failed") +} + +func MustNoError(err error) { + if err == nil { + return + } + log.PanicError(err, "error happens, assertion failed") +} diff --git a/src/pkg/libs/atomic2/atomic64.go b/src/pkg/libs/atomic2/atomic64.go new file mode 100644 index 0000000..3ea851a --- /dev/null +++ b/src/pkg/libs/atomic2/atomic64.go @@ -0,0 +1,49 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package atomic2 + +import ( + "strconv" + "sync/atomic" +) + +type Int64 struct { + v int64 +} + +func (a *Int64) Get() int64 { + return atomic.LoadInt64(&a.v) +} + +func (a *Int64) Set(v int64) { + atomic.StoreInt64(&a.v, v) +} + +func (a *Int64) CompareAndSwap(o, n int64) bool { + return atomic.CompareAndSwapInt64(&a.v, o, n) +} + +func (a *Int64) Swap(v int64) int64 { + return atomic.SwapInt64(&a.v, v) +} + +func (a *Int64) Add(v int64) int64 { + return atomic.AddInt64(&a.v, v) +} + +func (a *Int64) Sub(v int64) int64 { + return a.Add(-v) +} + +func (a *Int64) Incr() int64 { + return a.Add(1) +} + +func (a *Int64) Decr() int64 { + return a.Add(-1) +} + +func (a *Int64) String() string { + return strconv.FormatInt(a.Get(), 10) +} diff --git a/src/pkg/libs/atomic2/bool.go b/src/pkg/libs/atomic2/bool.go new file mode 100644 index 0000000..170902a --- /dev/null +++ b/src/pkg/libs/atomic2/bool.go @@ -0,0 +1,32 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package atomic2 + +type Bool struct { + c Int64 +} + +func (b *Bool) Get() bool { + return b.c.Get() != 0 +} + +func (b *Bool) toInt64(v bool) int64 { + if v { + return 1 + } else { + return 0 + } +} + +func (b *Bool) Set(v bool) { + b.c.Set(b.toInt64(v)) +} + +func (b *Bool) CompareAndSwap(o, n bool) bool { + return b.c.CompareAndSwap(b.toInt64(o), b.toInt64(n)) +} + +func (b *Bool) Swap(v bool) bool { + return b.c.Swap(b.toInt64(v)) != 0 +} diff --git a/src/pkg/libs/bytesize/bytesize.go b/src/pkg/libs/bytesize/bytesize.go new file mode 100644 index 0000000..1263ac6 --- /dev/null +++ b/src/pkg/libs/bytesize/bytesize.go @@ -0,0 +1,85 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package bytesize + +import ( + "regexp" + "strconv" + "strings" + + "pkg/libs/errors" + "pkg/libs/log" +) + +const ( + B = 1 << (10 * iota) + KB + MB + GB + TB + PB +) + +var ( + BytesizeRegexp = regexp.MustCompile(`(?i)^\s*(\-?[\d\.]+)\s*([KMGTP]?B|[BKMGTP]|)\s*$`) + digitsRegexp = regexp.MustCompile(`^\-?\d+$`) +) + +var ( + ErrBadBytesize = errors.New("invalid byte size") + ErrBadBytesizeUnit = errors.New("invalid byte size unit") +) + +func Parse(s string) (int64, error) { + if !BytesizeRegexp.MatchString(s) { + return 0, errors.Trace(ErrBadBytesize) + } + + subs := BytesizeRegexp.FindStringSubmatch(s) + if len(subs) != 3 { + return 0, errors.Trace(ErrBadBytesize) + } + + size := int64(0) + switch strings.ToUpper(string(subs[2])) { + case "B", "": + size = 1 + case "KB", "K": + size = KB + case "MB", "M": + size = MB + case "GB", "G": + size = GB + case "TB", "T": + size = TB + case "PB", "P": + size = PB + default: + return 0, errors.Trace(ErrBadBytesizeUnit) + } + + text := subs[1] + if digitsRegexp.MatchString(text) { + n, err := strconv.ParseInt(text, 10, 64) + if err != nil { + return 0, errors.Trace(ErrBadBytesize) + } + size *= n + } else { + n, err := strconv.ParseFloat(text, 64) + if err != nil { + return 0, errors.Trace(ErrBadBytesize) + } + size = int64(float64(size) * n) + } + return size, nil +} + +func MustParse(s string) int64 { + v, err := Parse(s) + if err != nil { + log.PanicError(err, "parse bytesize failed") + } + return v +} diff --git a/src/pkg/libs/bytesize/bytesize_test.go b/src/pkg/libs/bytesize/bytesize_test.go new file mode 100644 index 0000000..2871cde --- /dev/null +++ b/src/pkg/libs/bytesize/bytesize_test.go @@ -0,0 +1,46 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package bytesize + +import ( + "testing" + + "pkg/libs/assert" + "pkg/libs/errors" +) + +func TestBytesize(t *testing.T) { + assert.Must(MustParse("1") == 1) + assert.Must(MustParse("1B") == 1) + assert.Must(MustParse("1K") == KB) + assert.Must(MustParse("1M") == MB) + assert.Must(MustParse("1G") == GB) + assert.Must(MustParse("1T") == TB) + assert.Must(MustParse("1P") == PB) + + assert.Must(MustParse(" -1") == -1) + assert.Must(MustParse(" -1 b") == -1) + assert.Must(MustParse(" -1 kb ") == -1*KB) + assert.Must(MustParse(" -1 mb ") == -1*MB) + assert.Must(MustParse(" -1 gb ") == -1*GB) + assert.Must(MustParse(" -1 tb ") == -1*TB) + assert.Must(MustParse(" -1 pb ") == -1*PB) + + assert.Must(MustParse(" 1.5") == 1) + assert.Must(MustParse(" 1.5 kb ") == 1.5*KB) + assert.Must(MustParse(" 1.5 mb ") == 1.5*MB) + assert.Must(MustParse(" 1.5 gb ") == 1.5*GB) + assert.Must(MustParse(" 1.5 tb ") == 1.5*TB) + assert.Must(MustParse(" 1.5 pb ") == 1.5*PB) +} + +func TestBytesizeError(t *testing.T) { + var err error + _, err = Parse("--1") + assert.Must(errors.Equal(err, ErrBadBytesize)) + _, err = Parse("hello world") + assert.Must(errors.Equal(err, ErrBadBytesize)) + _, err = Parse("123.132.32") + assert.Must(errors.Equal(err, ErrBadBytesize)) +} diff --git a/src/pkg/libs/cupcake/rdb/LICENCE b/src/pkg/libs/cupcake/rdb/LICENCE new file mode 100644 index 0000000..5025790 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/LICENCE @@ -0,0 +1,21 @@ +Copyright (c) 2012 Jonathan Rudenberg +Copyright (c) 2012 Sripathi Krishnan + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/pkg/libs/cupcake/rdb/README.md b/src/pkg/libs/cupcake/rdb/README.md new file mode 100644 index 0000000..5c19212 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/README.md @@ -0,0 +1,17 @@ +# rdb [![Build Status](https://travis-ci.org/cupcake/rdb.png?branch=master)](https://travis-ci.org/cupcake/rdb) + +rdb is a Go package that implements parsing and encoding of the +[Redis](http://redis.io) [RDB file +format](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_File_Format.textile). + +This package was heavily inspired by +[redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) by +[Sripathi Krishnan](https://github.com/sripathikrishnan). + +[**Documentation**](http://godoc.org/github.com/cupcake/rdb) + +## Installation + +``` +go get github.com/cupcake/rdb +``` diff --git a/src/pkg/libs/cupcake/rdb/crc64/crc64.go b/src/pkg/libs/cupcake/rdb/crc64/crc64.go new file mode 100644 index 0000000..54fed9c --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/crc64/crc64.go @@ -0,0 +1,64 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package crc64 implements the Jones coefficients with an init value of 0. +package crc64 + +import "hash" + +// Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. +// +// Specification of this CRC64 variant follows: +// Name: crc-64-jones +// Width: 64 bits +// Poly: 0xad93d23594c935a9 +// Reflected In: True +// Xor_In: 0xffffffffffffffff +// Reflected_Out: True +// Xor_Out: 0x0 + +var table = [256]uint64{0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} + +func crc64(crc uint64, b []byte) uint64 { + for _, v := range b { + crc = table[byte(crc)^v] ^ (crc >> 8) + } + return crc +} + +func Digest(b []byte) uint64 { + return crc64(0, b) +} + +type digest struct { + crc uint64 +} + +func New() hash.Hash64 { + return &digest{} +} + +func (h *digest) Write(p []byte) (int, error) { + h.crc = crc64(h.crc, p) + return len(p), nil +} + +// Encode in little endian +func (d *digest) Sum(in []byte) []byte { + s := d.Sum64() + in = append(in, byte(s)) + in = append(in, byte(s>>8)) + in = append(in, byte(s>>16)) + in = append(in, byte(s>>24)) + in = append(in, byte(s>>32)) + in = append(in, byte(s>>40)) + in = append(in, byte(s>>48)) + in = append(in, byte(s>>56)) + return in +} + +func (d *digest) Sum64() uint64 { return d.crc } +func (d *digest) BlockSize() int { return 1 } +func (d *digest) Size() int { return 8 } +func (d *digest) Reset() { d.crc = 0 } diff --git a/src/pkg/libs/cupcake/rdb/decoder.go b/src/pkg/libs/cupcake/rdb/decoder.go new file mode 100644 index 0000000..ac715be --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/decoder.go @@ -0,0 +1,860 @@ +// Package rdb implements parsing and encoding of the Redis RDB file format. +package rdb + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "strconv" + + "github.com/cupcake/rdb/crc64" +) + +// A Decoder must be implemented to parse a RDB file. +type Decoder interface { + // StartRDB is called when parsing of a valid RDB file starts. + StartRDB() + // StartDatabase is called when database n starts. + // Once a database starts, another database will not start until EndDatabase is called. + StartDatabase(n int) + // AUX field + Aux(key, value []byte) + // ResizeDB hint + ResizeDatabase(dbSize, expiresSize uint32) + // Set is called once for each string key. + Set(key, value []byte, expiry int64) + // StartHash is called at the beginning of a hash. + // Hset will be called exactly length times before EndHash. + StartHash(key []byte, length, expiry int64) + // Hset is called once for each field=value pair in a hash. + Hset(key, field, value []byte) + // EndHash is called when there are no more fields in a hash. + EndHash(key []byte) + // StartSet is called at the beginning of a set. + // Sadd will be called exactly cardinality times before EndSet. + StartSet(key []byte, cardinality, expiry int64) + // Sadd is called once for each member of a set. + Sadd(key, member []byte) + // EndSet is called when there are no more fields in a set. + EndSet(key []byte) + // StartList is called at the beginning of a list. + // Rpush will be called exactly length times before EndList. + // If length of the list is not known, then length is -1 + StartList(key []byte, length, expiry int64) + // Rpush is called once for each value in a list. + Rpush(key, value []byte) + // EndList is called when there are no more values in a list. + EndList(key []byte) + // StartZSet is called at the beginning of a sorted set. + // Zadd will be called exactly cardinality times before EndZSet. + StartZSet(key []byte, cardinality, expiry int64) + // Zadd is called once for each member of a sorted set. + Zadd(key []byte, score float64, member []byte) + // EndZSet is called when there are no more members in a sorted set. + EndZSet(key []byte) + // EndDatabase is called at the end of a database. + EndDatabase(n int) + // EndRDB is called when parsing of the RDB file is complete. + EndRDB() +} + +// Decode parses a RDB file from r and calls the decode hooks on d. +func Decode(r io.Reader, d Decoder) error { + decoder := &decode{d, make([]byte, 8), bufio.NewReader(r)} + return decoder.decode() +} + +// Decode a byte slice from the Redis DUMP command. The dump does not contain the +// database, key or expiry, so they must be included in the function call (but +// can be zero values). +func DecodeDump(dump []byte, db int, key []byte, expiry int64, d Decoder) error { + err := verifyDump(dump) + if err != nil { + return err + } + + decoder := &decode{d, make([]byte, 8), bytes.NewReader(dump[1:])} + decoder.event.StartRDB() + decoder.event.StartDatabase(db) + + err = decoder.readObject(key, ValueType(dump[0]), expiry) + + decoder.event.EndDatabase(db) + decoder.event.EndRDB() + return err +} + +type byteReader interface { + io.Reader + io.ByteReader +} + +type decode struct { + event Decoder + intBuf []byte + r byteReader +} + +type ValueType byte + +const ( + TypeString ValueType = 0 + TypeList ValueType = 1 + TypeSet ValueType = 2 + TypeZSet ValueType = 3 + TypeHash ValueType = 4 + TypeZSet2 ValueType = 5 + TypeModule ValueType = 6 + + TypeHashZipmap ValueType = 9 + TypeListZiplist ValueType = 10 + TypeSetIntset ValueType = 11 + TypeZSetZiplist ValueType = 12 + TypeHashZiplist ValueType = 13 + TypeListQuicklist ValueType = 14 +) + +const ( + rdb6bitLen = 0 + rdb14bitLen = 1 + rdb32bitLen = 0x80 + rdb64bitLen = 0x81 + rdbEncVal = 3 + + rdbFlagAux = 0xfa + rdbFlagResizeDB = 0xfb + rdbFlagExpiryMS = 0xfc + rdbFlagExpiry = 0xfd + rdbFlagSelectDB = 0xfe + rdbFlagEOF = 0xff + + rdbEncInt8 = 0 + rdbEncInt16 = 1 + rdbEncInt32 = 2 + rdbEncLZF = 3 + + rdbZiplist6bitlenString = 0 + rdbZiplist14bitlenString = 1 + rdbZiplist32bitlenString = 2 + + rdbZiplistInt16 = 0xc0 + rdbZiplistInt32 = 0xd0 + rdbZiplistInt64 = 0xe0 + rdbZiplistInt24 = 0xf0 + rdbZiplistInt8 = 0xfe + rdbZiplistInt4 = 15 +) + +func (d *decode) decode() error { + err := d.checkHeader() + if err != nil { + return err + } + + d.event.StartRDB() + + var db uint32 + var expiry int64 + firstDB := true + for { + objType, err := d.r.ReadByte() + if err != nil { + return err + } + switch objType { + case rdbFlagAux: + auxKey, err := d.readString() + if err != nil { + return err + } + auxVal, err := d.readString() + if err != nil { + return err + } + d.event.Aux(auxKey, auxVal) + case rdbFlagResizeDB: + dbSize, _, err := d.readLength() + if err != nil { + return err + } + expiresSize, _, err := d.readLength() + if err != nil { + return err + } + d.event.ResizeDatabase(dbSize, expiresSize) + case rdbFlagExpiryMS: + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return err + } + expiry = int64(binary.LittleEndian.Uint64(d.intBuf)) + case rdbFlagExpiry: + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return err + } + expiry = int64(binary.LittleEndian.Uint32(d.intBuf)) * 1000 + case rdbFlagSelectDB: + if !firstDB { + d.event.EndDatabase(int(db)) + } + db, _, err = d.readLength() + if err != nil { + return err + } + d.event.StartDatabase(int(db)) + case rdbFlagEOF: + d.event.EndDatabase(int(db)) + d.event.EndRDB() + return nil + default: + key, err := d.readString() + if err != nil { + return err + } + err = d.readObject(key, ValueType(objType), expiry) + if err != nil { + return err + } + expiry = 0 + } + } + + panic("not reached") +} + +func (d *decode) readObject(key []byte, typ ValueType, expiry int64) error { + switch typ { + case TypeString: + value, err := d.readString() + if err != nil { + return err + } + d.event.Set(key, value, expiry) + case TypeList: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartList(key, int64(length), expiry) + for i := uint32(0); i < length; i++ { + value, err := d.readString() + if err != nil { + return err + } + d.event.Rpush(key, value) + } + d.event.EndList(key) + case TypeListQuicklist: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartList(key, int64(-1), expiry) + for i := uint32(0); i < length; i++ { + d.readZiplist(key, 0, false) + } + d.event.EndList(key) + case TypeSet: + cardinality, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + member, err := d.readString() + if err != nil { + return err + } + d.event.Sadd(key, member) + } + d.event.EndSet(key) + case TypeZSet, TypeZSet2: + cardinality, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartZSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + member, err := d.readString() + if err != nil { + return err + } + var score float64 + if typ == TypeZSet2 { + score, err = d.readDouble64() + if err != nil { + return err + } + } else { + score, err = d.readFloat64() + if err != nil { + return err + } + } + d.event.Zadd(key, score, member) + } + d.event.EndZSet(key) + case TypeHash: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartHash(key, int64(length), expiry) + for i := uint32(0); i < length; i++ { + field, err := d.readString() + if err != nil { + return err + } + value, err := d.readString() + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + case TypeHashZipmap: + return d.readZipmap(key, expiry) + case TypeListZiplist: + return d.readZiplist(key, expiry, true) + case TypeSetIntset: + return d.readIntset(key, expiry) + case TypeZSetZiplist: + return d.readZiplistZset(key, expiry) + case TypeHashZiplist: + return d.readZiplistHash(key, expiry) + case TypeModule: + //need add function to parse module type + return fmt.Errorf("rdb unable to read Redis Modules RDB objects (key %s)", key) + default: + return fmt.Errorf("rdb: unknown object type %d for key %s", typ, key) + } + return nil +} + +func (d *decode) readZipmap(key []byte, expiry int64) error { + var length int + zipmap, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(zipmap) + lenByte, err := buf.ReadByte() + if err != nil { + return err + } + if lenByte >= 254 { // we need to count the items manually + length, err = countZipmapItems(buf) + length /= 2 + if err != nil { + return err + } + } else { + length = int(lenByte) + } + d.event.StartHash(key, int64(length), expiry) + for i := 0; i < length; i++ { + field, err := readZipmapItem(buf, false) + if err != nil { + return err + } + value, err := readZipmapItem(buf, true) + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + return nil +} + +func readZipmapItem(buf *sliceBuffer, readFree bool) ([]byte, error) { + length, free, err := readZipmapItemLength(buf, readFree) + if err != nil { + return nil, err + } + if length == -1 { + return nil, nil + } + value, err := buf.Slice(length) + if err != nil { + return nil, err + } + _, err = buf.Seek(int64(free), 1) + return value, err +} + +func countZipmapItems(buf *sliceBuffer) (int, error) { + n := 0 + for { + strLen, free, err := readZipmapItemLength(buf, n%2 != 0) + if err != nil { + return 0, err + } + if strLen == -1 { + break + } + _, err = buf.Seek(int64(strLen)+int64(free), 1) + if err != nil { + return 0, err + } + n++ + } + _, err := buf.Seek(0, 0) + return n, err +} + +func readZipmapItemLength(buf *sliceBuffer, readFree bool) (int, int, error) { + b, err := buf.ReadByte() + if err != nil { + return 0, 0, err + } + switch b { + case 253: + s, err := buf.Slice(5) + if err != nil { + return 0, 0, err + } + return int(binary.BigEndian.Uint32(s)), int(s[4]), nil + case 254: + return 0, 0, fmt.Errorf("rdb: invalid zipmap item length") + case 255: + return -1, 0, nil + } + var free byte + if readFree { + free, err = buf.ReadByte() + } + return int(b), int(free), err +} + +func (d *decode) readZiplist(key []byte, expiry int64, addListEvents bool) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + length, err := readZiplistLength(buf) + if err != nil { + return err + } + if addListEvents { + d.event.StartList(key, length, expiry) + } + for i := int64(0); i < length; i++ { + entry, err := readZiplistEntry(buf) + if err != nil { + return err + } + d.event.Rpush(key, entry) + } + if addListEvents { + d.event.EndList(key) + } + return nil +} + +func (d *decode) readZiplistZset(key []byte, expiry int64) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + cardinality, err := readZiplistLength(buf) + if err != nil { + return err + } + cardinality /= 2 + d.event.StartZSet(key, cardinality, expiry) + for i := int64(0); i < cardinality; i++ { + member, err := readZiplistEntry(buf) + if err != nil { + return err + } + scoreBytes, err := readZiplistEntry(buf) + if err != nil { + return err + } + score, err := strconv.ParseFloat(string(scoreBytes), 64) + if err != nil { + return err + } + d.event.Zadd(key, score, member) + } + d.event.EndZSet(key) + return nil +} + +func (d *decode) readZiplistHash(key []byte, expiry int64) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + length, err := readZiplistLength(buf) + if err != nil { + return err + } + length /= 2 + d.event.StartHash(key, length, expiry) + for i := int64(0); i < length; i++ { + field, err := readZiplistEntry(buf) + if err != nil { + return err + } + value, err := readZiplistEntry(buf) + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + return nil +} + +func readZiplistLength(buf *sliceBuffer) (int64, error) { + buf.Seek(8, 0) // skip the zlbytes and zltail + lenBytes, err := buf.Slice(2) + if err != nil { + return 0, err + } + return int64(binary.LittleEndian.Uint16(lenBytes)), nil +} + +func readZiplistEntry(buf *sliceBuffer) ([]byte, error) { + prevLen, err := buf.ReadByte() + if err != nil { + return nil, err + } + if prevLen == 254 { + buf.Seek(4, 1) // skip the 4-byte prevlen + } + + header, err := buf.ReadByte() + if err != nil { + return nil, err + } + switch { + case header>>6 == rdbZiplist6bitlenString: + return buf.Slice(int(header & 0x3f)) + case header>>6 == rdbZiplist14bitlenString: + b, err := buf.ReadByte() + if err != nil { + return nil, err + } + return buf.Slice((int(header&0x3f) << 8) | int(b)) + case header>>6 == rdbZiplist32bitlenString: + lenBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return buf.Slice(int(binary.BigEndian.Uint32(lenBytes))) + case header == rdbZiplistInt16: + intBytes, err := buf.Slice(2) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10)), nil + case header == rdbZiplistInt32: + intBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10)), nil + case header == rdbZiplistInt64: + intBytes, err := buf.Slice(8) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(intBytes)), 10)), nil + case header == rdbZiplistInt24: + intBytes := make([]byte, 4) + _, err := buf.Read(intBytes[1:]) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))>>8), 10)), nil + case header == rdbZiplistInt8: + b, err := buf.ReadByte() + return []byte(strconv.FormatInt(int64(int8(b)), 10)), err + case header>>4 == rdbZiplistInt4: + return []byte(strconv.FormatInt(int64(header&0x0f)-1, 10)), nil + } + + return nil, fmt.Errorf("rdb: unknown ziplist header byte: %d", header) +} + +func (d *decode) readIntset(key []byte, expiry int64) error { + intset, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(intset) + intSizeBytes, err := buf.Slice(4) + if err != nil { + return err + } + intSize := binary.LittleEndian.Uint32(intSizeBytes) + + if intSize != 2 && intSize != 4 && intSize != 8 { + return fmt.Errorf("rdb: unknown intset encoding: %d", intSize) + } + + lenBytes, err := buf.Slice(4) + if err != nil { + return err + } + cardinality := binary.LittleEndian.Uint32(lenBytes) + + d.event.StartSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + intBytes, err := buf.Slice(int(intSize)) + if err != nil { + return err + } + var intString string + switch intSize { + case 2: + intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) + case 4: + intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) + case 8: + intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) + } + d.event.Sadd(key, []byte(intString)) + } + d.event.EndSet(key) + return nil +} + +func (d *decode) checkHeader() error { + header := make([]byte, 9) + _, err := io.ReadFull(d.r, header) + if err != nil { + return err + } + + if !bytes.Equal(header[:5], []byte("REDIS")) { + return fmt.Errorf("rdb: invalid file format") + } + + version, _ := strconv.ParseInt(string(header[5:]), 10, 64) + if version < 1 || version > 8 { + return fmt.Errorf("rdb: invalid RDB version number %d", version) + } + + return nil +} + +func (d *decode) readString() ([]byte, error) { + length, encoded, err := d.readLength() + if err != nil { + return nil, err + } + if encoded { + switch length { + case rdbEncInt8: + i, err := d.readUint8() + return []byte(strconv.FormatInt(int64(int8(i)), 10)), err + case rdbEncInt16: + i, err := d.readUint16() + return []byte(strconv.FormatInt(int64(int16(i)), 10)), err + case rdbEncInt32: + i, err := d.readUint32() + return []byte(strconv.FormatInt(int64(int32(i)), 10)), err + case rdbEncLZF: + clen, _, err := d.readLength() + if err != nil { + return nil, err + } + ulen, _, err := d.readLength() + if err != nil { + return nil, err + } + compressed := make([]byte, clen) + _, err = io.ReadFull(d.r, compressed) + if err != nil { + return nil, err + } + decompressed := lzfDecompress(compressed, int(ulen)) + if len(decompressed) != int(ulen) { + return nil, fmt.Errorf("decompressed string length %d didn't match expected length %d", len(decompressed), ulen) + } + return decompressed, nil + } + } + + str := make([]byte, length) + _, err = io.ReadFull(d.r, str) + return str, err +} + +func (d *decode) readUint8() (uint8, error) { + b, err := d.r.ReadByte() + return uint8(b), err +} + +func (d *decode) readUint16() (uint16, error) { + _, err := io.ReadFull(d.r, d.intBuf[:2]) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(d.intBuf), nil +} + +func (d *decode) readUint32() (uint32, error) { + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(d.intBuf), nil +} + +func (d *decode) readUint64Big() (uint64, error) { + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint64(d.intBuf), nil +} + +func (d *decode) readDouble64() (float64, error) { + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return 0, err + } + bits := binary.LittleEndian.Uint64(d.intBuf) + return float64(math.Float64frombits(bits)), nil +} + +func (d *decode) readUint64() (uint64, error) { + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(d.intBuf), nil +} + +func (d *decode) readUint32Big() (uint32, error) { + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint32(d.intBuf), nil +} + +// Doubles are saved as strings prefixed by an unsigned +// 8 bit integer specifying the length of the representation. +// This 8 bit integer has special values in order to specify the following +// conditions: +// 253: not a number +// 254: + inf +// 255: - inf +func (d *decode) readFloat64() (float64, error) { + length, err := d.readUint8() + if err != nil { + return 0, err + } + switch length { + case 253: + return math.NaN(), nil + case 254: + return math.Inf(0), nil + case 255: + return math.Inf(-1), nil + default: + floatBytes := make([]byte, length) + _, err := io.ReadFull(d.r, floatBytes) + if err != nil { + return 0, err + } + f, err := strconv.ParseFloat(string(floatBytes), 64) + return f, err + } + + panic("not reached") +} + +func (d *decode) readLength() (uint32, bool, error) { + b, err := d.r.ReadByte() + if err != nil { + return 0, false, err + } + // The first two bits of the first byte are used to indicate the length encoding type + switch (b & 0xc0) >> 6 { + case rdb6bitLen: + // When the first two bits are 00, the next 6 bits are the length. + return uint32(b & 0x3f), false, nil + case rdb14bitLen: + // When the first two bits are 01, the next 14 bits are the length. + bb, err := d.r.ReadByte() + if err != nil { + return 0, false, err + } + return (uint32(b&0x3f) << 8) | uint32(bb), false, nil + case rdbEncVal: + // When the first two bits are 11, the next object is encoded. + // The next 6 bits indicate the encoding type. + return uint32(b & 0x3f), true, nil + default: + // When the first two bits are 10, the next 6 bits are discarded. + // The next 4 bytes are the length. + if b == rdb64bitLen { + length, err := d.readUint64Big() + return uint32(length), false, err + } else { + length, err := d.readUint32Big() + return length, false, err + } + } + + panic("not reached") +} + +func verifyDump(d []byte) error { + if len(d) < 10 { + return fmt.Errorf("rdb: invalid dump length") + } + version := binary.LittleEndian.Uint16(d[len(d)-10:]) + if version != uint16(Version) { + return fmt.Errorf("rdb: invalid version %d, expecting %d", version, Version) + } + + if binary.LittleEndian.Uint64(d[len(d)-8:]) != crc64.Digest(d[:len(d)-8]) { + return fmt.Errorf("rdb: invalid CRC checksum") + } + + return nil +} + +func lzfDecompress(in []byte, outlen int) []byte { + out := make([]byte, outlen) + for i, o := 0, 0; i < len(in); { + ctrl := int(in[i]) + i++ + if ctrl < 32 { + for x := 0; x <= ctrl; x++ { + out[o] = in[i] + i++ + o++ + } + } else { + length := ctrl >> 5 + if length == 7 { + length = length + int(in[i]) + i++ + } + ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 + i++ + for x := 0; x <= length+1; x++ { + out[o] = out[ref] + ref++ + o++ + } + } + } + return out +} diff --git a/src/pkg/libs/cupcake/rdb/decoder_test.go b/src/pkg/libs/cupcake/rdb/decoder_test.go new file mode 100644 index 0000000..6dd0e23 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/decoder_test.go @@ -0,0 +1,347 @@ +package rdb_test + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/cupcake/rdb" + . "gopkg.in/check.v1" +) + +// Hook gocheck into the gotest runner. +func Test(t *testing.T) { TestingT(t) } + +type DecoderSuite struct{} + +var _ = Suite(&DecoderSuite{}) + +func (s *DecoderSuite) TestEmptyRDB(c *C) { + r := decodeRDB("empty_database") + c.Assert(r.started, Equals, 1) + c.Assert(r.ended, Equals, 1) + c.Assert(len(r.dbs), Equals, 0) +} + +func (s *DecoderSuite) TestMultipleDatabases(c *C) { + r := decodeRDB("multiple_databases") + c.Assert(len(r.dbs), Equals, 2) + _, ok := r.dbs[1] + c.Assert(ok, Equals, false) + c.Assert(r.dbs[0]["key_in_zeroth_database"], Equals, "zero") + c.Assert(r.dbs[2]["key_in_second_database"], Equals, "second") +} + +func (s *DecoderSuite) TestExpiry(c *C) { + r := decodeRDB("keys_with_expiry") + c.Assert(r.expiries[0]["expires_ms_precision"], Equals, int64(1671963072573)) +} + +func (s *DecoderSuite) TestMixedExpiry(c *C) { + r := decodeRDB("keys_with_mixed_expiry") + c.Assert(r.expiries[0]["key01"], Not(Equals), int64(0)) + c.Assert(r.expiries[0]["key04"], Not(Equals), int64(0)) + + c.Assert(r.expiries[0]["key02"], Equals, int64(0)) + c.Assert(r.expiries[0]["key03"], Equals, int64(0)) +} + +func (s *DecoderSuite) TestIntegerKeys(c *C) { + r := decodeRDB("integer_keys") + c.Assert(r.dbs[0]["125"], Equals, "Positive 8 bit integer") + c.Assert(r.dbs[0]["43947"], Equals, "Positive 16 bit integer") + c.Assert(r.dbs[0]["183358245"], Equals, "Positive 32 bit integer") + c.Assert(r.dbs[0]["-123"], Equals, "Negative 8 bit integer") + c.Assert(r.dbs[0]["-29477"], Equals, "Negative 16 bit integer") + c.Assert(r.dbs[0]["-183358245"], Equals, "Negative 32 bit integer") +} + +func (s *DecoderSuite) TestStringKeyWithCompression(c *C) { + r := decodeRDB("easily_compressible_string_key") + c.Assert(r.dbs[0][strings.Repeat("a", 200)], Equals, "Key that redis should compress easily") +} + +func (s *DecoderSuite) TestZipmapWithCompression(c *C) { + r := decodeRDB("zipmap_that_compresses_easily") + zm := r.dbs[0]["zipmap_compresses_easily"].(map[string]string) + c.Assert(zm["a"], Equals, "aa") + c.Assert(zm["aa"], Equals, "aaaa") + c.Assert(zm["aaaaa"], Equals, "aaaaaaaaaaaaaa") +} + +func (s *DecoderSuite) TestZipmap(c *C) { + r := decodeRDB("zipmap_that_doesnt_compress") + zm := r.dbs[0]["zimap_doesnt_compress"].(map[string]string) + c.Assert(zm["MKD1G6"], Equals, "2") + c.Assert(zm["YNNXK"], Equals, "F7TI") +} + +func (s *DecoderSuite) TestZipmapWitBigValues(c *C) { + r := decodeRDB("zipmap_with_big_values") + zm := r.dbs[0]["zipmap_with_big_values"].(map[string]string) + c.Assert(len(zm["253bytes"]), Equals, 253) + c.Assert(len(zm["254bytes"]), Equals, 254) + c.Assert(len(zm["255bytes"]), Equals, 255) + c.Assert(len(zm["300bytes"]), Equals, 300) + c.Assert(len(zm["20kbytes"]), Equals, 20000) +} + +func (s *DecoderSuite) TestHashZiplist(c *C) { + r := decodeRDB("hash_as_ziplist") + zm := r.dbs[0]["zipmap_compresses_easily"].(map[string]string) + c.Assert(zm["a"], Equals, "aa") + c.Assert(zm["aa"], Equals, "aaaa") + c.Assert(zm["aaaaa"], Equals, "aaaaaaaaaaaaaa") +} + +func (s *DecoderSuite) TestDictionary(c *C) { + r := decodeRDB("dictionary") + d := r.dbs[0]["force_dictionary"].(map[string]string) + c.Assert(len(d), Equals, 1000) + c.Assert(d["ZMU5WEJDG7KU89AOG5LJT6K7HMNB3DEI43M6EYTJ83VRJ6XNXQ"], Equals, "T63SOS8DQJF0Q0VJEZ0D1IQFCYTIPSBOUIAI9SB0OV57MQR1FI") + c.Assert(d["UHS5ESW4HLK8XOGTM39IK1SJEUGVV9WOPK6JYA5QBZSJU84491"], Equals, "6VULTCV52FXJ8MGVSFTZVAGK2JXZMGQ5F8OVJI0X6GEDDR27RZ") +} + +func (s *DecoderSuite) TestZiplistWithCompression(c *C) { + r := decodeRDB("ziplist_that_compresses_easily") + for i, length := range []int{6, 12, 18, 24, 30, 36} { + c.Assert(r.dbs[0]["ziplist_compresses_easily"].([]string)[i], Equals, strings.Repeat("a", length)) + } +} + +func (s *DecoderSuite) TestZiplist(c *C) { + r := decodeRDB("ziplist_that_doesnt_compress") + l := r.dbs[0]["ziplist_doesnt_compress"].([]string) + c.Assert(l[0], Equals, "aj2410") + c.Assert(l[1], Equals, "cc953a17a8e096e76a44169ad3f9ac87c5f8248a403274416179aa9fbd852344") +} + +func (s *DecoderSuite) TestZiplistWithInts(c *C) { + r := decodeRDB("ziplist_with_integers") + expected := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "-2", "13", "25", "-61", "63", "16380", "-16000", "65535", "-65523", "4194304", "9223372036854775807"} + for i, x := range expected { + c.Assert(r.dbs[0]["ziplist_with_integers"].([]string)[i], Equals, x) + } +} + +func (s *DecoderSuite) TestIntSet16(c *C) { + r := decodeRDB("intset_16") + for i, x := range []string{"32764", "32765", "32766"} { + c.Assert(r.dbs[0]["intset_16"].([]string)[i], Equals, x) + } +} + +func (s *DecoderSuite) TestIntSet32(c *C) { + r := decodeRDB("intset_32") + for i, x := range []string{"2147418108", "2147418109", "2147418110"} { + c.Assert(r.dbs[0]["intset_32"].([]string)[i], Equals, x) + } +} + +func (s *DecoderSuite) TestIntSet64(c *C) { + r := decodeRDB("intset_64") + for i, x := range []string{"9223090557583032316", "9223090557583032317", "9223090557583032318"} { + c.Assert(r.dbs[0]["intset_64"].([]string)[i], Equals, x) + } +} + +func (s *DecoderSuite) TestSet(c *C) { + r := decodeRDB("regular_set") + for i, x := range []string{"beta", "delta", "alpha", "phi", "gamma", "kappa"} { + c.Assert(r.dbs[0]["regular_set"].([]string)[i], Equals, x) + } +} + +func (s *DecoderSuite) TestZSetZiplist(c *C) { + r := decodeRDB("sorted_set_as_ziplist") + z := r.dbs[0]["sorted_set_as_ziplist"].(map[string]float64) + c.Assert(z["8b6ba6718a786daefa69438148361901"], Equals, float64(1)) + c.Assert(z["cb7a24bb7528f934b841b34c3a73e0c7"], Equals, float64(2.37)) + c.Assert(z["523af537946b79c4f8369ed39ba78605"], Equals, float64(3.423)) +} + +func (s *DecoderSuite) TestRDBv5(c *C) { + r := decodeRDB("rdb_version_5_with_checksum") + c.Assert(r.dbs[0]["abcd"], Equals, "efgh") + c.Assert(r.dbs[0]["foo"], Equals, "bar") + c.Assert(r.dbs[0]["bar"], Equals, "baz") + c.Assert(r.dbs[0]["abcdef"], Equals, "abcdef") + c.Assert(r.dbs[0]["longerstring"], Equals, "thisisalongerstring.idontknowwhatitmeans") +} + +func (s *DecoderSuite) TestRDBv7(c *C) { + r := decodeRDB("rdb_v7_list_quicklist") + c.Assert(r.aux["redis-ver"], Equals, "3.2.0") + c.Assert(r.dbSize[0], Equals, uint32(1)) + c.Assert(r.expiresSize[0], Equals, uint32(0)) + z := r.dbs[0]["foo"].([]string) + c.Assert(z[0], Equals, "bar") + c.Assert(z[1], Equals, "baz") + c.Assert(z[2], Equals, "boo") +} + +func (s *DecoderSuite) TestDumpDecoder(c *C) { + r := &FakeRedis{} + err := rdb.DecodeDump([]byte("\u0000\xC0\n\u0006\u0000\xF8r?\xC5\xFB\xFB_("), 1, []byte("test"), 123, r) + if err != nil { + c.Error(err) + } + c.Assert(r.dbs[1]["test"], Equals, "10") +} + +func decodeRDB(name string) *FakeRedis { + r := &FakeRedis{} + f, err := os.Open("fixtures/" + name + ".rdb") + if err != nil { + panic(err) + } + err = rdb.Decode(f, r) + if err != nil { + panic(err) + } + return r +} + +type FakeRedis struct { + dbs map[int]map[string]interface{} + lengths map[int]map[string]int + expiries map[int]map[string]int64 + dbSize map[int]uint32 + expiresSize map[int]uint32 + + cdb int + started int + ended int + + aux map[string]string +} + +func (r *FakeRedis) setExpiry(key []byte, expiry int64) { + r.expiries[r.cdb][string(key)] = expiry +} + +func (r *FakeRedis) setLength(key []byte, length int64) { + r.lengths[r.cdb][string(key)] = int(length) +} + +func (r *FakeRedis) getLength(key []byte) int { + return int(r.lengths[r.cdb][string(key)]) +} + +func (r *FakeRedis) db() map[string]interface{} { + return r.dbs[r.cdb] +} + +func (r *FakeRedis) StartRDB() { + r.started++ + r.dbs = make(map[int]map[string]interface{}) + r.expiries = make(map[int]map[string]int64) + r.lengths = make(map[int]map[string]int) + r.aux = make(map[string]string) + r.dbSize = make(map[int]uint32) + r.expiresSize = make(map[int]uint32) +} + +func (r *FakeRedis) StartDatabase(n int) { + r.dbs[n] = make(map[string]interface{}) + r.expiries[n] = make(map[string]int64) + r.lengths[n] = make(map[string]int) + r.cdb = n +} + +func (r *FakeRedis) Set(key, value []byte, expiry int64) { + r.setExpiry(key, expiry) + r.db()[string(key)] = string(value) +} + +func (r *FakeRedis) StartHash(key []byte, length, expiry int64) { + r.setExpiry(key, expiry) + r.setLength(key, length) + r.db()[string(key)] = make(map[string]string) +} + +func (r *FakeRedis) Hset(key, field, value []byte) { + r.db()[string(key)].(map[string]string)[string(field)] = string(value) +} + +func (r *FakeRedis) EndHash(key []byte) { + actual := len(r.db()[string(key)].(map[string]string)) + if actual != r.getLength(key) { + panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) + } +} + +func (r *FakeRedis) StartSet(key []byte, cardinality, expiry int64) { + r.setExpiry(key, expiry) + r.setLength(key, cardinality) + r.db()[string(key)] = make([]string, 0, cardinality) +} + +func (r *FakeRedis) Sadd(key, member []byte) { + r.db()[string(key)] = append(r.db()[string(key)].([]string), string(member)) +} + +func (r *FakeRedis) EndSet(key []byte) { + actual := len(r.db()[string(key)].([]string)) + if actual != r.getLength(key) { + panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) + } +} + +func (r *FakeRedis) StartList(key []byte, length, expiry int64) { + r.setExpiry(key, expiry) + r.setLength(key, length) + cap := length + if length < 0 { + cap = 1 + } + r.db()[string(key)] = make([]string, 0, cap) +} + +func (r *FakeRedis) Rpush(key, value []byte) { + r.db()[string(key)] = append(r.db()[string(key)].([]string), string(value)) +} + +func (r *FakeRedis) EndList(key []byte) { + actual := len(r.db()[string(key)].([]string)) + if actual != r.getLength(key) && r.getLength(key) >= 0 { + panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) + } +} + +func (r *FakeRedis) StartZSet(key []byte, cardinality, expiry int64) { + r.setExpiry(key, expiry) + r.setLength(key, cardinality) + r.db()[string(key)] = make(map[string]float64) +} + +func (r *FakeRedis) Zadd(key []byte, score float64, member []byte) { + r.db()[string(key)].(map[string]float64)[string(member)] = score +} + +func (r *FakeRedis) EndZSet(key []byte) { + actual := len(r.db()[string(key)].(map[string]float64)) + if actual != r.getLength(key) { + panic(fmt.Sprintf("wrong length for key %s got %d, expected %d", key, actual, r.getLength(key))) + } +} + +func (r *FakeRedis) EndDatabase(n int) { + if n != r.cdb { + panic(fmt.Sprintf("database end called with %d, expected %d", n, r.cdb)) + } +} + +func (r *FakeRedis) EndRDB() { + r.ended++ +} + +func (r *FakeRedis) Aux(key, value []byte) { + r.aux[string(key)] = string(value) +} + +func (r *FakeRedis) ResizeDatabase(dbSize, expiresSize uint32) { + r.dbSize[r.cdb] = dbSize + r.expiresSize[r.cdb] = expiresSize +} diff --git a/src/pkg/libs/cupcake/rdb/encoder.go b/src/pkg/libs/cupcake/rdb/encoder.go new file mode 100644 index 0000000..eac5d56 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/encoder.go @@ -0,0 +1,130 @@ +package rdb + +import ( + "encoding/binary" + "fmt" + "hash" + "io" + "math" + "strconv" + + "github.com/cupcake/rdb/crc64" +) + +const Version = 6 + +type Encoder struct { + w io.Writer + crc hash.Hash +} + +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{crc: crc64.New()} + e.w = io.MultiWriter(w, e.crc) + return e +} + +func (e *Encoder) EncodeHeader() error { + _, err := fmt.Fprintf(e.w, "REDIS%04d", Version) + return err +} + +func (e *Encoder) EncodeFooter() error { + e.w.Write([]byte{rdbFlagEOF}) + _, err := e.w.Write(e.crc.Sum(nil)) + return err +} + +func (e *Encoder) EncodeDumpFooter() error { + binary.Write(e.w, binary.LittleEndian, uint16(Version)) + _, err := e.w.Write(e.crc.Sum(nil)) + return err +} + +func (e *Encoder) EncodeDatabase(n int) error { + e.w.Write([]byte{rdbFlagSelectDB}) + return e.EncodeLength(uint32(n)) +} + +func (e *Encoder) EncodeExpiry(expiry uint64) error { + b := make([]byte, 9) + b[0] = rdbFlagExpiryMS + binary.LittleEndian.PutUint64(b[1:], expiry) + _, err := e.w.Write(b) + return err +} + +func (e *Encoder) EncodeType(v ValueType) error { + _, err := e.w.Write([]byte{byte(v)}) + return err +} + +func (e *Encoder) EncodeString(s []byte) error { + written, err := e.encodeIntString(s) + if written { + return err + } + e.EncodeLength(uint32(len(s))) + _, err = e.w.Write(s) + return err +} + +func (e *Encoder) EncodeLength(l uint32) (err error) { + switch { + case l < 1<<6: + _, err = e.w.Write([]byte{byte(l)}) + case l < 1<<14: + _, err = e.w.Write([]byte{byte(l>>8) | rdb14bitLen<<6, byte(l)}) + default: + b := make([]byte, 5) + b[0] = rdb32bitLen + binary.BigEndian.PutUint32(b[1:], l) + _, err = e.w.Write(b) + } + return +} + +func (e *Encoder) EncodeFloat(f float64) (err error) { + switch { + case math.IsNaN(f): + _, err = e.w.Write([]byte{253}) + case math.IsInf(f, 1): + _, err = e.w.Write([]byte{254}) + case math.IsInf(f, -1): + _, err = e.w.Write([]byte{255}) + default: + b := []byte(strconv.FormatFloat(f, 'g', 17, 64)) + e.w.Write([]byte{byte(len(b))}) + _, err = e.w.Write(b) + } + return +} + +func (e *Encoder) encodeIntString(b []byte) (written bool, err error) { + s := string(b) + i, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return + } + // if the stringified parsed int isn't exactly the same, we can't encode it as an int + if s != strconv.FormatInt(i, 10) { + return + } + switch { + case i >= math.MinInt8 && i <= math.MaxInt8: + _, err = e.w.Write([]byte{rdbEncVal << 6, byte(int8(i))}) + case i >= math.MinInt16 && i <= math.MaxInt16: + b := make([]byte, 3) + b[0] = rdbEncVal<<6 | rdbEncInt16 + binary.LittleEndian.PutUint16(b[1:], uint16(int16(i))) + _, err = e.w.Write(b) + case i >= math.MinInt32 && i <= math.MaxInt32: + b := make([]byte, 5) + b[0] = rdbEncVal<<6 | rdbEncInt32 + binary.LittleEndian.PutUint32(b[1:], uint32(int32(i))) + _, err = e.w.Write(b) + default: + return + } + return true, err +} diff --git a/src/pkg/libs/cupcake/rdb/encoder_test.go b/src/pkg/libs/cupcake/rdb/encoder_test.go new file mode 100644 index 0000000..93cd765 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/encoder_test.go @@ -0,0 +1,43 @@ +package rdb_test + +import ( + "bytes" + "encoding/base64" + + "github.com/cupcake/rdb" + . "gopkg.in/check.v1" +) + +type EncoderSuite struct{} + +var _ = Suite(&EncoderSuite{}) + +var stringEncodingTests = []struct { + str string + res string +}{ + {"0", "AMAABgAOrc/4DQU/mw=="}, + {"127", "AMB/BgCbWIOxpwH5hw=="}, + {"-128", "AMCABgAPi1rt2llnSg=="}, + {"128", "AMGAAAYAfZfbNeWad/Y="}, + {"-129", "AMF//wYAgY3qqKHVuBM="}, + {"32767", "AMH/fwYA37dfWuKh6bg="}, + {"-32768", "AMEAgAYAI61ux6buJl0="}, + {"-32768", "AMEAgAYAI61ux6buJl0="}, + {"2147483647", "AML///9/BgC6mY0eFXuRMg=="}, + {"-2147483648", "AMIAAACABgBRou++xgC9FA=="}, + {"a", "AAFhBgApE4cbemNBJw=="}, +} + +func (e *EncoderSuite) TestStringEncoding(c *C) { + buf := &bytes.Buffer{} + for _, t := range stringEncodingTests { + e := rdb.NewEncoder(buf) + e.EncodeType(rdb.TypeString) + e.EncodeString([]byte(t.str)) + e.EncodeDumpFooter() + expected, _ := base64.StdEncoding.DecodeString(t.res) + c.Assert(buf.Bytes(), DeepEquals, expected, Commentf("%s - expected: %x, actual: %x", t.str, expected, buf.Bytes())) + buf.Reset() + } +} diff --git a/src/pkg/libs/cupcake/rdb/examples/diff.go b/src/pkg/libs/cupcake/rdb/examples/diff.go new file mode 100644 index 0000000..4760769 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/examples/diff.go @@ -0,0 +1,65 @@ +// This is a very basic example of a program that implements rdb.decoder and +// outputs a human readable diffable dump of the rdb file. +package main + +import ( + "fmt" + "os" + + "github.com/cupcake/rdb" + "github.com/cupcake/rdb/nopdecoder" +) + +type decoder struct { + db int + i int + nopdecoder.NopDecoder +} + +func (p *decoder) StartDatabase(n int) { + p.db = n +} + +func (p *decoder) Set(key, value []byte, expiry int64) { + fmt.Printf("db=%d %q -> %q\n", p.db, key, value) +} + +func (p *decoder) Hset(key, field, value []byte) { + fmt.Printf("db=%d %q . %q -> %q\n", p.db, key, field, value) +} + +func (p *decoder) Sadd(key, member []byte) { + fmt.Printf("db=%d %q { %q }\n", p.db, key, member) +} + +func (p *decoder) StartList(key []byte, length, expiry int64) { + p.i = 0 +} + +func (p *decoder) Rpush(key, value []byte) { + fmt.Printf("db=%d %q[%d] -> %q\n", p.db, key, p.i, value) + p.i++ +} + +func (p *decoder) StartZSet(key []byte, cardinality, expiry int64) { + p.i = 0 +} + +func (p *decoder) Zadd(key []byte, score float64, member []byte) { + fmt.Printf("db=%d %q[%d] -> {%q, score=%g}\n", p.db, key, p.i, member, score) + p.i++ +} + +func maybeFatal(err error) { + if err != nil { + fmt.Printf("Fatal error: %s\n", err) + os.Exit(1) + } +} + +func main() { + f, err := os.Open(os.Args[1]) + maybeFatal(err) + err = rdb.Decode(f, &decoder{}) + maybeFatal(err) +} diff --git a/src/pkg/libs/cupcake/rdb/nopdecoder/nop_decoder.go b/src/pkg/libs/cupcake/rdb/nopdecoder/nop_decoder.go new file mode 100644 index 0000000..de93a69 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/nopdecoder/nop_decoder.go @@ -0,0 +1,24 @@ +package nopdecoder + +// NopDecoder may be embedded in a real Decoder to avoid implementing methods. +type NopDecoder struct{} + +func (d NopDecoder) StartRDB() {} +func (d NopDecoder) StartDatabase(n int) {} +func (d NopDecoder) Aux(key, value []byte) {} +func (d NopDecoder) ResizeDatabase(dbSize, expiresSize uint32) {} +func (d NopDecoder) EndDatabase(n int) {} +func (d NopDecoder) EndRDB() {} +func (d NopDecoder) Set(key, value []byte, expiry int64) {} +func (d NopDecoder) StartHash(key []byte, length, expiry int64) {} +func (d NopDecoder) Hset(key, field, value []byte) {} +func (d NopDecoder) EndHash(key []byte) {} +func (d NopDecoder) StartSet(key []byte, cardinality, expiry int64) {} +func (d NopDecoder) Sadd(key, member []byte) {} +func (d NopDecoder) EndSet(key []byte) {} +func (d NopDecoder) StartList(key []byte, length, expiry int64) {} +func (d NopDecoder) Rpush(key, value []byte) {} +func (d NopDecoder) EndList(key []byte) {} +func (d NopDecoder) StartZSet(key []byte, cardinality, expiry int64) {} +func (d NopDecoder) Zadd(key []byte, score float64, member []byte) {} +func (d NopDecoder) EndZSet(key []byte) {} diff --git a/src/pkg/libs/cupcake/rdb/slice_buffer.go b/src/pkg/libs/cupcake/rdb/slice_buffer.go new file mode 100644 index 0000000..b3e12a0 --- /dev/null +++ b/src/pkg/libs/cupcake/rdb/slice_buffer.go @@ -0,0 +1,67 @@ +package rdb + +import ( + "errors" + "io" +) + +type sliceBuffer struct { + s []byte + i int +} + +func newSliceBuffer(s []byte) *sliceBuffer { + return &sliceBuffer{s, 0} +} + +func (s *sliceBuffer) Slice(n int) ([]byte, error) { + if s.i+n > len(s.s) { + return nil, io.EOF + } + b := s.s[s.i : s.i+n] + s.i += n + return b, nil +} + +func (s *sliceBuffer) ReadByte() (byte, error) { + if s.i >= len(s.s) { + return 0, io.EOF + } + b := s.s[s.i] + s.i++ + return b, nil +} + +func (s *sliceBuffer) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if s.i >= len(s.s) { + return 0, io.EOF + } + n := copy(b, s.s[s.i:]) + s.i += n + return n, nil +} + +func (s *sliceBuffer) Seek(offset int64, whence int) (int64, error) { + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(s.i) + offset + case 2: + abs = int64(len(s.s)) + offset + default: + return 0, errors.New("invalid whence") + } + if abs < 0 { + return 0, errors.New("negative position") + } + if abs >= 1<<31 { + return 0, errors.New("position out of range") + } + s.i = int(abs) + return abs, nil +} diff --git a/src/pkg/libs/errors/errors.go b/src/pkg/libs/errors/errors.go new file mode 100644 index 0000000..bffc5ef --- /dev/null +++ b/src/pkg/libs/errors/errors.go @@ -0,0 +1,90 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package errors + +import ( + "errors" + "fmt" + + "pkg/libs/trace" +) + +var TraceEnabled = true + +type TracedError struct { + Stack trace.Stack + Cause error +} + +func (e *TracedError) Error() string { + return e.Cause.Error() +} + +func New(s string) error { + return errors.New(s) +} + +func Trace(err error) error { + if err == nil || !TraceEnabled { + return err + } + _, ok := err.(*TracedError) + if ok { + return err + } + return &TracedError{ + Stack: trace.TraceN(1, 32), + Cause: err, + } +} + +func Errorf(format string, v ...interface{}) error { + err := fmt.Errorf(format, v...) + if !TraceEnabled { + return err + } + return &TracedError{ + Stack: trace.TraceN(1, 32), + Cause: err, + } +} + +func Stack(err error) trace.Stack { + if err == nil { + return nil + } + e, ok := err.(*TracedError) + if ok { + return e.Stack + } + return nil +} + +func Cause(err error) error { + for err != nil { + e, ok := err.(*TracedError) + if ok { + err = e.Cause + } else { + return err + } + } + return nil +} + +func Equal(err1, err2 error) bool { + e1 := Cause(err1) + e2 := Cause(err2) + if e1 == e2 { + return true + } + if e1 == nil || e2 == nil { + return e1 == e2 + } + return e1.Error() == e2.Error() +} + +func NotEqual(err1, err2 error) bool { + return !Equal(err1, err2) +} diff --git a/src/pkg/libs/errors/list.go b/src/pkg/libs/errors/list.go new file mode 100644 index 0000000..40c4449 --- /dev/null +++ b/src/pkg/libs/errors/list.go @@ -0,0 +1,60 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package errors + +import ( + "container/list" + "errors" + "sync" +) + +var ErrAnonError = errors.New("anonymous error") + +type ErrorList struct { + mu sync.Mutex + el list.List +} + +func (q *ErrorList) First() error { + q.mu.Lock() + defer q.mu.Unlock() + if e := q.el.Front(); e != nil { + return e.Value.(error) + } + return nil +} + +func (q *ErrorList) Errors() []error { + q.mu.Lock() + defer q.mu.Unlock() + if n := q.el.Len(); n != 0 { + array := make([]error, 0, n) + for e := q.el.Front(); e != nil; e = e.Next() { + array = append(array, e.Value.(error)) + } + return array + } + return nil +} + +func (q *ErrorList) Len() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.el.Len() +} + +func (q *ErrorList) PushBack(err error) { + if err == nil { + err = Trace(ErrAnonError) + } + q.mu.Lock() + q.el.PushBack(err) + q.mu.Unlock() +} + +func (q *ErrorList) Reset() { + q.mu.Lock() + q.el.Init() + q.mu.Unlock() +} diff --git a/src/pkg/libs/fmt2/strconv.go b/src/pkg/libs/fmt2/strconv.go new file mode 100644 index 0000000..07af524 --- /dev/null +++ b/src/pkg/libs/fmt2/strconv.go @@ -0,0 +1,159 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package fmt2 + +import ( + "math" + "reflect" + "strconv" + + "pkg/libs/errors" +) + +func Num64(i interface{}) (n64 interface{}, ok bool) { + switch x := i.(type) { + case int: + n64 = int64(x) + case int8: + n64 = int64(x) + case int16: + n64 = int64(x) + case int32: + n64 = int64(x) + case int64: + n64 = int64(x) + case uint: + n64 = uint64(x) + case uint8: + n64 = uint64(x) + case uint16: + n64 = uint64(x) + case uint32: + n64 = uint64(x) + case uint64: + n64 = uint64(x) + case float32: + n64 = float64(x) + case float64: + n64 = float64(x) + default: + return i, false + } + return n64, true +} + +func ParseFloat64(i interface{}) (float64, error) { + if v, ok := Num64(i); ok { + switch x := v.(type) { + case int64: + return float64(x), nil + case uint64: + return float64(x), nil + case float64: + switch { + case math.IsNaN(x): + return 0, errors.Errorf("parse nan float64") + case math.IsInf(x, 0): + return 0, errors.Errorf("parse inf float64") + } + return float64(x), nil + default: + return 0, errors.Errorf("parse float64 from unknown num64") + } + } else { + var s string + switch x := i.(type) { + case nil: + return 0, errors.Errorf("parse float64 from nil") + case string: + s = x + case []byte: + s = string(x) + default: + return 0, errors.Errorf("parse float64 from <%s>", reflect.TypeOf(i)) + } + f, err := strconv.ParseFloat(s, 64) + return f, errors.Trace(err) + } +} + +func ParseInt64(i interface{}) (int64, error) { + if v, ok := Num64(i); ok { + switch x := v.(type) { + case int64: + return int64(x), nil + case uint64: + if x > math.MaxInt64 { + return 0, errors.Errorf("parse int64 from uint64, overflow") + } + return int64(x), nil + case float64: + switch { + case math.IsNaN(x): + return 0, errors.Errorf("parse int64 from nan float64") + case math.IsInf(x, 0): + return 0, errors.Errorf("parse int64 from inf float64") + case math.Abs(x-float64(int64(x))) > 1e-9: + return 0, errors.Errorf("parse int64 from inv float64") + } + return int64(x), nil + default: + return 0, errors.Errorf("parse int64 from unknown num64") + } + } else { + var s string + switch x := i.(type) { + case nil: + return 0, errors.Errorf("parse int64 from nil") + case string: + s = x + case []byte: + s = string(x) + default: + return 0, errors.Errorf("parse int64 from <%s>", reflect.TypeOf(i)) + } + v, err := strconv.ParseInt(s, 10, 64) + return v, errors.Trace(err) + } +} + +func ParseUint64(i interface{}) (uint64, error) { + if v, ok := Num64(i); ok { + switch x := v.(type) { + case int64: + if x < 0 { + return 0, errors.Errorf("parse uint64 from int64, overflow") + } + return uint64(x), nil + case uint64: + return uint64(x), nil + case float64: + switch { + case math.IsNaN(x): + return 0, errors.Errorf("parse uint64 from nan float64") + case math.IsInf(x, 0): + return 0, errors.Errorf("parse uint64 from inf float64") + case math.Abs(x-float64(uint64(x))) > 1e-9: + return 0, errors.Errorf("parse uint64 from inv float64") + } + return uint64(x), nil + default: + return 0, errors.Errorf("parse int64 from unknown num64") + } + } else { + var s string + switch x := i.(type) { + case nil: + return 0, errors.Errorf("parse uint64 from nil") + case string: + s = x + case []byte: + s = string(x) + default: + return 0, errors.Errorf("parse uint64 from <%s>", reflect.TypeOf(i)) + } + v, err := strconv.ParseUint(s, 10, 64) + return v, errors.Trace(err) + } +} diff --git a/src/pkg/libs/io/backlog/backlog.go b/src/pkg/libs/io/backlog/backlog.go new file mode 100644 index 0000000..a8c1817 --- /dev/null +++ b/src/pkg/libs/io/backlog/backlog.go @@ -0,0 +1,225 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package backlog + +import ( + "os" + "sync" + + "pkg/libs/errors" +) + +var ( + ErrClosedBacklog = errors.New("closed backlog") + ErrInvalidOffset = errors.New("invalid offset") +) + +type buffer interface { + readSomeAt(b []byte, rpos uint64) (int, error) + writeSome(b []byte) (int, error) + + dataRange() (rpos, wpos uint64) + + close() error +} + +type Backlog struct { + wl sync.Mutex + mu sync.Mutex + + err error + + rwait *sync.Cond + store buffer +} + +func roffset(blen int, size, rpos, wpos uint64) (maxlen, offset uint64) { + maxlen = uint64(blen) + if n := wpos - rpos; n < maxlen { + maxlen = n + } + offset = rpos % size + if n := size - offset; n < maxlen { + maxlen = n + } + return +} + +func woffset(blen int, size, wpos uint64) (maxlen, offset uint64) { + maxlen = uint64(blen) + if size < maxlen { + maxlen = size + } + offset = wpos % size + if n := size - offset; n < maxlen { + maxlen = n + } + return +} + +func align(size, unit int) int { + if size < unit { + return unit + } + return (size + unit - 1) / unit * unit +} + +func newBacklog(store buffer) *Backlog { + bl := &Backlog{} + bl.rwait = sync.NewCond(&bl.mu) + bl.store = store + return bl +} + +func (bl *Backlog) ReadAt(b []byte, offset uint64) (int, error) { + for { + n, err := bl.readSomeAt(b, offset) + if err != nil || n != 0 { + return n, err + } + if len(b) == 0 { + return 0, nil + } + } +} + +func (bl *Backlog) readSomeAt(b []byte, rpos uint64) (int, error) { + bl.mu.Lock() + defer bl.mu.Unlock() + if bl.store == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + if len(b) == 0 || bl.err != nil { + return 0, bl.err + } + n, err := bl.store.readSomeAt(b, rpos) + if err != nil || n != 0 { + return n, err + } + bl.rwait.Wait() + return 0, nil +} + +func (bl *Backlog) Write(b []byte) (int, error) { + bl.wl.Lock() + defer bl.wl.Unlock() + var nn int + for { + n, err := bl.writeSome(b) + if err != nil { + return nn + n, err + } + nn, b = nn+n, b[n:] + if len(b) == 0 { + return nn, nil + } + } +} + +func (bl *Backlog) writeSome(b []byte) (int, error) { + bl.mu.Lock() + defer bl.mu.Unlock() + if bl.store == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + if len(b) == 0 || bl.err != nil { + return 0, bl.err + } + n, err := bl.store.writeSome(b) + if err != nil || n != 0 { + bl.rwait.Broadcast() + return n, err + } + return 0, nil +} + +func (bl *Backlog) Close() error { + return bl.CloseWithError(nil) +} + +func (bl *Backlog) CloseWithError(err error) error { + if err == nil { + err = errors.Trace(ErrClosedBacklog) + } + bl.mu.Lock() + defer bl.mu.Unlock() + if bl.err != nil { + bl.err = err + } + bl.rwait.Broadcast() + if bl.store != nil { + return bl.store.close() + } + return nil +} + +func (bl *Backlog) DataRange() (rpos, wpos uint64, err error) { + bl.mu.Lock() + defer bl.mu.Unlock() + if bl.store == nil { + return 0, 0, errors.Trace(ErrClosedBacklog) + } + if bl.err != nil { + return 0, 0, bl.err + } + rpos, wpos = bl.store.dataRange() + return rpos, wpos, nil +} + +func (bl *Backlog) NewReader() (*Reader, error) { + bl.mu.Lock() + defer bl.mu.Unlock() + if bl.store == nil { + return nil, errors.Trace(ErrClosedBacklog) + } + if bl.err != nil { + return nil, bl.err + } + _, wpos := bl.store.dataRange() + return &Reader{bl: bl, seek: wpos}, nil +} + +type Reader struct { + bl *Backlog + seek uint64 +} + +func (r *Reader) Read(b []byte) (int, error) { + n, err := r.bl.ReadAt(b, r.seek) + r.seek += uint64(n) + return n, err +} + +func (r *Reader) DataRange() (rpos, wpos uint64, err error) { + return r.bl.DataRange() +} + +func (r *Reader) IsValid() bool { + rpos, wpos, err := r.DataRange() + if err != nil { + return false + } + return r.seek >= rpos && r.seek <= wpos +} + +func (r *Reader) Offset() uint64 { + return r.seek +} + +func (r *Reader) SeekTo(seek uint64) bool { + r.seek = seek + return r.IsValid() +} + +func New() *Backlog { + return NewSize(BuffSizeAlign) +} + +func NewSize(buffSize int) *Backlog { + return newBacklog(newMemBuffer(buffSize)) +} + +func NewFileBacklog(fileSize int, f *os.File) *Backlog { + return newBacklog(newFileBuffer(fileSize, f)) +} diff --git a/src/pkg/libs/io/backlog/backlog_test.go b/src/pkg/libs/io/backlog/backlog_test.go new file mode 100644 index 0000000..d02c811 --- /dev/null +++ b/src/pkg/libs/io/backlog/backlog_test.go @@ -0,0 +1,106 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package backlog + +import ( + "bytes" + "io" + "math/rand" + "os" + "testing" + "time" + + "pkg/libs/assert" + "pkg/libs/errors" +) + +func openFile(fileName string) *os.File { + f, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + assert.MustNoError(err) + return f +} + +func checkWriter(bl *Backlog, b []byte) { + n, err := bl.Write(b) + assert.MustNoError(err) + assert.Must(n == len(b)) +} + +func checkReader(r io.Reader, b []byte) { + x := make([]byte, len(b)) + n, err := io.ReadFull(r, x) + assert.MustNoError(err) + assert.Must(n == len(b) && bytes.Equal(x, b)) +} + +func randSlice(n int) []byte { + b := make([]byte, n) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := 0; i < len(b); i++ { + b[i] = byte(r.Int()) + } + return b +} + +func testBacklog(t *testing.T, bl *Backlog, size int) { + input := randSlice(32) + + r1, err := bl.NewReader() + assert.MustNoError(err) + + checkWriter(bl, input) + checkReader(r1, input) + checkReader(r1, []byte{}) + + input = randSlice(size) + checkWriter(bl, input) + checkReader(r1, input) + checkWriter(bl, randSlice(size)) + + assert.Must(r1.IsValid() == true) + + r2, err := bl.NewReader() + assert.MustNoError(err) + + input = []byte{0xde, 0xad, 0xbe, 0xef} + checkWriter(bl, input) + + assert.Must(r1.IsValid() == false) + + _, err = r1.Read([]byte{0}) + assert.Must(errors.Equal(err, ErrInvalidOffset)) + + b := make([]byte, len(input)) + n, err := io.ReadFull(r2, b) + assert.MustNoError(err) + assert.Must(n == len(b) && bytes.Equal(b, input)) + + bl.Close() + + assert.Must(r1.IsValid() == false) + assert.Must(r2.IsValid() == false) + + _, err = r1.Read([]byte{0}) + assert.Must(errors.Equal(err, ErrClosedBacklog)) + + _, err = r2.Read([]byte{0}) + assert.Must(errors.Equal(err, ErrClosedBacklog)) + + _, err = bl.Write([]byte{0}) + assert.Must(errors.Equal(err, ErrClosedBacklog)) +} + +func TestBacklog1(t *testing.T) { + const size = BuffSizeAlign * 2 + bl := NewSize(size) + testBacklog(t, bl, size) +} + +func TestBacklog2(t *testing.T) { + f := openFile("/tmp/backlog.test") + defer f.Close() + const size = FileSizeAlign * 2 + bl := NewFileBacklog(size, f) + testBacklog(t, bl, size) +} diff --git a/src/pkg/libs/io/backlog/buff.go b/src/pkg/libs/io/backlog/buff.go new file mode 100644 index 0000000..d6ff55e --- /dev/null +++ b/src/pkg/libs/io/backlog/buff.go @@ -0,0 +1,68 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package backlog + +import "pkg/libs/errors" + +const ( + BuffSizeAlign = 1024 * 4 +) + +type memBuffer struct { + b []byte + size uint64 + wpos uint64 +} + +func newMemBuffer(buffSize int) *memBuffer { + n := align(buffSize, BuffSizeAlign) + if n <= 0 { + panic("invalid backlog buffer size") + } + return &memBuffer{b: make([]byte, n), size: uint64(n)} +} + +func (p *memBuffer) readSomeAt(b []byte, rpos uint64) (int, error) { + if p.b == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + if rpos > p.wpos || rpos+p.size < p.wpos { + return 0, errors.Trace(ErrInvalidOffset) + } + maxlen, offset := roffset(len(b), p.size, rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n := copy(b, p.b[offset:offset+maxlen]) + return n, nil +} + +func (p *memBuffer) writeSome(b []byte) (int, error) { + if p.b == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + maxlen, offset := woffset(len(b), p.size, p.wpos) + if maxlen == 0 { + return 0, nil + } + n := copy(p.b[offset:offset+maxlen], b) + p.wpos += uint64(n) + return n, nil +} + +func (p *memBuffer) dataRange() (rpos, wpos uint64) { + if p.b == nil { + return 0, 0 + } else { + if p.wpos >= p.size { + return p.wpos - p.size, p.wpos + } + return 0, p.wpos + } +} + +func (p *memBuffer) close() error { + p.b = nil + return nil +} diff --git a/src/pkg/libs/io/backlog/file.go b/src/pkg/libs/io/backlog/file.go new file mode 100644 index 0000000..62f3a98 --- /dev/null +++ b/src/pkg/libs/io/backlog/file.go @@ -0,0 +1,76 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package backlog + +import ( + "os" + + "pkg/libs/errors" +) + +const ( + FileSizeAlign = 1024 * 1024 * 4 +) + +type fileBuffer struct { + f *os.File + size uint64 + wpos uint64 +} + +func newFileBuffer(fileSize int, f *os.File) *fileBuffer { + n := align(fileSize, FileSizeAlign) + if n <= 0 { + panic("invalid backlog buffer size") + } + return &fileBuffer{f: f, size: uint64(n)} +} + +func (p *fileBuffer) readSomeAt(b []byte, rpos uint64) (int, error) { + if p.f == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + if rpos > p.wpos || rpos+p.size < p.wpos { + return 0, errors.Trace(ErrInvalidOffset) + } + maxlen, offset := roffset(len(b), p.size, rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n, err := p.f.ReadAt(b[:maxlen], int64(offset)) + return n, errors.Trace(err) +} + +func (p *fileBuffer) writeSome(b []byte) (int, error) { + if p.f == nil { + return 0, errors.Trace(ErrClosedBacklog) + } + maxlen, offset := woffset(len(b), p.size, p.wpos) + if maxlen == 0 { + return 0, nil + } + n, err := p.f.WriteAt(b[:maxlen], int64(offset)) + p.wpos += uint64(n) + return n, errors.Trace(err) +} + +func (p *fileBuffer) dataRange() (rpos, wpos uint64) { + if p.f == nil { + return 0, 0 + } else { + if p.wpos >= p.size { + return p.wpos - p.size, p.wpos + } + return 0, p.wpos + } +} + +func (p *fileBuffer) close() error { + if f := p.f; f != nil { + p.f = nil + defer f.Close() + return errors.Trace(f.Truncate(0)) + } + return nil +} diff --git a/src/pkg/libs/io/pipe/buff.go b/src/pkg/libs/io/pipe/buff.go new file mode 100644 index 0000000..73f86d6 --- /dev/null +++ b/src/pkg/libs/io/pipe/buff.go @@ -0,0 +1,82 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package pipe + +import ( + "io" + + "pkg/libs/errors" +) + +const ( + BuffSizeAlign = 1024 * 4 +) + +type memBuffer struct { + b []byte + size uint64 + rpos uint64 + wpos uint64 +} + +func newMemBuffer(buffSize int) *memBuffer { + n := align(buffSize, BuffSizeAlign) + if n <= 0 { + panic("invalid pipe buffer size") + } + return &memBuffer{b: make([]byte, n), size: uint64(n)} +} + +func (p *memBuffer) readSome(b []byte) (int, error) { + if p.b == nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + maxlen, offset := roffset(len(b), p.size, p.rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n := copy(b, p.b[offset:offset+maxlen]) + p.rpos += uint64(n) + if p.rpos == p.wpos { + p.rpos = 0 + p.wpos = 0 + } + return n, nil +} + +func (p *memBuffer) writeSome(b []byte) (int, error) { + if p.b == nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + maxlen, offset := woffset(len(b), p.size, p.rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n := copy(p.b[offset:offset+maxlen], b) + p.wpos += uint64(n) + return n, nil +} + +func (p *memBuffer) buffered() int { + if p.b == nil { + return 0 + } + return int(p.wpos - p.rpos) +} + +func (p *memBuffer) available() int { + if p.b == nil { + return 0 + } + return int(p.size + p.rpos - p.wpos) +} + +func (p *memBuffer) rclose() error { + p.b = nil + return nil +} + +func (p *memBuffer) wclose() error { + return nil +} diff --git a/src/pkg/libs/io/pipe/file.go b/src/pkg/libs/io/pipe/file.go new file mode 100644 index 0000000..82f2110 --- /dev/null +++ b/src/pkg/libs/io/pipe/file.go @@ -0,0 +1,90 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package pipe + +import ( + "io" + "os" + + "pkg/libs/errors" +) + +const ( + FileSizeAlign = 1024 * 1024 * 4 +) + +type fileBuffer struct { + f *os.File + size uint64 + rpos uint64 + wpos uint64 +} + +func newFileBuffer(fileSize int, f *os.File) *fileBuffer { + n := align(fileSize, FileSizeAlign) + if n <= 0 { + panic("invalid pipe buffer size") + } + return &fileBuffer{f: f, size: uint64(n)} +} + +func (p *fileBuffer) readSome(b []byte) (int, error) { + if p.f == nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + maxlen, offset := roffset(len(b), p.size, p.rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n, err := p.f.ReadAt(b[:maxlen], int64(offset)) + p.rpos += uint64(n) + if p.rpos == p.wpos { + p.rpos = 0 + p.wpos = 0 + if err == nil { + err = p.f.Truncate(0) + } + } + return n, errors.Trace(err) +} + +func (p *fileBuffer) writeSome(b []byte) (int, error) { + if p.f == nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + maxlen, offset := woffset(len(b), p.size, p.rpos, p.wpos) + if maxlen == 0 { + return 0, nil + } + n, err := p.f.WriteAt(b[:maxlen], int64(offset)) + p.wpos += uint64(n) + return n, errors.Trace(err) +} + +func (p *fileBuffer) buffered() int { + if p.f == nil { + return 0 + } + return int(p.wpos - p.rpos) +} + +func (p *fileBuffer) available() int { + if p.f == nil { + return 0 + } + return int(p.size + p.rpos - p.wpos) +} + +func (p *fileBuffer) rclose() error { + if f := p.f; f != nil { + p.f = nil + defer f.Close() + return errors.Trace(f.Truncate(0)) + } + return nil +} + +func (p *fileBuffer) wclose() error { + return nil +} diff --git a/src/pkg/libs/io/pipe/pipe.go b/src/pkg/libs/io/pipe/pipe.go new file mode 100644 index 0000000..f9fe042 --- /dev/null +++ b/src/pkg/libs/io/pipe/pipe.go @@ -0,0 +1,217 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package pipe + +import ( + "io" + "os" + "sync" + + "pkg/libs/errors" +) + +type buffer interface { + readSome(b []byte) (int, error) + writeSome(b []byte) (int, error) + + buffered() int + available() int + + rclose() error + wclose() error +} + +type pipe struct { + rl sync.Mutex + wl sync.Mutex + mu sync.Mutex + + rwait *sync.Cond + wwait *sync.Cond + + rerr error + werr error + + store buffer +} + +func roffset(blen int, size, rpos, wpos uint64) (maxlen, offset uint64) { + maxlen = uint64(blen) + if n := wpos - rpos; n < maxlen { + maxlen = n + } + offset = rpos % size + if n := size - offset; n < maxlen { + maxlen = n + } + return +} + +func woffset(blen int, size, rpos, wpos uint64) (maxlen, offset uint64) { + maxlen = uint64(blen) + if n := size + rpos - wpos; n < maxlen { + maxlen = n + } + offset = wpos % size + if n := size - offset; n < maxlen { + maxlen = n + } + return +} + +func align(size, unit int) int { + if size < unit { + return unit + } + return (size + unit - 1) / unit * unit +} + +func newPipe(store buffer) (Reader, Writer) { + p := &pipe{} + p.rwait = sync.NewCond(&p.mu) + p.wwait = sync.NewCond(&p.mu) + p.store = store + r := &reader{p} + w := &writer{p} + return r, w +} + +func (p *pipe) Read(b []byte) (int, error) { + p.rl.Lock() + defer p.rl.Unlock() + for { + n, err := p.readSome(b) + if err != nil || n != 0 { + return n, err + } + if len(b) == 0 { + return 0, nil + } + } +} + +func (p *pipe) readSome(b []byte) (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.rerr != nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + if len(b) == 0 { + if p.store.buffered() != 0 { + return 0, nil + } + return 0, p.werr + } + n, err := p.store.readSome(b) + if err != nil || n != 0 { + p.wwait.Signal() + return n, err + } + if p.werr != nil { + return 0, p.werr + } + p.rwait.Wait() + return 0, nil +} + +func (p *pipe) Write(b []byte) (int, error) { + p.wl.Lock() + defer p.wl.Unlock() + var nn int + for { + n, err := p.writeSome(b) + if err != nil { + return nn + n, err + } + nn, b = nn+n, b[n:] + if len(b) == 0 { + return nn, nil + } + } +} + +func (p *pipe) writeSome(b []byte) (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.werr != nil { + return 0, errors.Trace(io.ErrClosedPipe) + } + if p.rerr != nil { + return 0, p.rerr + } + if len(b) == 0 { + return 0, nil + } + n, err := p.store.writeSome(b) + if err != nil || n != 0 { + p.rwait.Signal() + return n, err + } + p.wwait.Wait() + return 0, nil +} + +func (p *pipe) RClose(err error) error { + if err == nil { + err = errors.Trace(io.ErrClosedPipe) + } + p.mu.Lock() + defer p.mu.Unlock() + if p.rerr == nil { + p.rerr = err + } + p.rwait.Signal() + p.wwait.Signal() + return p.store.rclose() +} + +func (p *pipe) WClose(err error) error { + if err == nil { + err = errors.Trace(io.EOF) + } + p.mu.Lock() + defer p.mu.Unlock() + if p.werr == nil { + p.werr = err + } + p.rwait.Signal() + p.wwait.Signal() + return p.store.wclose() +} + +func (p *pipe) Buffered() (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.rerr != nil { + return 0, p.rerr + } + if n := p.store.buffered(); n != 0 { + return n, nil + } + return 0, p.werr +} + +func (p *pipe) Available() (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.werr != nil { + return 0, p.werr + } + if p.rerr != nil { + return 0, p.rerr + } + return p.store.available(), nil +} + +func New() (Reader, Writer) { + return NewSize(BuffSizeAlign) +} + +func NewSize(buffSize int) (Reader, Writer) { + return newPipe(newMemBuffer(buffSize)) +} + +func NewFilePipe(fileSize int, f *os.File) (Reader, Writer) { + return newPipe(newFileBuffer(fileSize, f)) +} diff --git a/src/pkg/libs/io/pipe/pipe_test.go b/src/pkg/libs/io/pipe/pipe_test.go new file mode 100644 index 0000000..4282de4 --- /dev/null +++ b/src/pkg/libs/io/pipe/pipe_test.go @@ -0,0 +1,391 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package pipe + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "fmt" + "io" + "os" + "testing" + "time" + + "pkg/libs/assert" + "pkg/libs/errors" +) + +func openPipe(t *testing.T, fileName string) (pr Reader, pw Writer, pf *os.File) { + buffSize := 8192 + fileSize := 1024 * 1024 * 32 + if fileName == "" { + pr, pw = NewSize(buffSize) + } else { + f, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + assert.MustNoError(err) + pr, pw = NewFilePipe(fileSize, f) + pf = f + } + return +} + +func testPipe1(t *testing.T, fileName string) { + r, w, f := openPipe(t, fileName) + defer f.Close() + + s := "Hello world!!" + + go func(data []byte) { + n, err := w.Write(data) + assert.MustNoError(err) + assert.Must(n == len(data)) + assert.MustNoError(w.Close()) + }([]byte(s)) + + buf := make([]byte, 64) + n, err := io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + assert.Must(n == len(s)) + assert.Must(string(buf[:n]) == s) + assert.MustNoError(r.Close()) +} + +func TestPipe1(t *testing.T) { + testPipe1(t, "") + testPipe1(t, "/tmp/pipe.test") +} + +func testPipe2(t *testing.T, fileName string) { + r, w, f := openPipe(t, fileName) + defer f.Close() + + c := 1024 * 128 + s := "Hello world!!" + + go func() { + for i := 0; i < c; i++ { + m := fmt.Sprintf("[%d]%s ", i, s) + n, err := w.Write([]byte(m)) + assert.MustNoError(err) + assert.Must(n == len(m)) + } + assert.MustNoError(w.Close()) + }() + + time.Sleep(time.Millisecond * 10) + + buf := make([]byte, len(s)*c*2) + n, err := io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + buf = buf[:n] + for i := 0; i < c; i++ { + m := fmt.Sprintf("[%d]%s ", i, s) + assert.Must(len(buf) >= len(m)) + assert.Must(string(buf[:len(m)]) == m) + buf = buf[len(m):] + } + assert.Must(len(buf) == 0) + assert.MustNoError(r.Close()) +} + +func TestPipe2(t *testing.T) { + testPipe2(t, "") + testPipe2(t, "/tmp/pipe.test") +} + +func testPipe3(t *testing.T, fileName string) { + r, w, f := openPipe(t, fileName) + defer f.Close() + + c := make(chan int) + + size := 4096 + + go func() { + buf := make([]byte, size) + for { + n, err := r.Read(buf) + if errors.Equal(err, io.EOF) { + break + } + assert.MustNoError(err) + c <- n + } + assert.MustNoError(r.Close()) + c <- 0 + }() + + go func() { + buf := make([]byte, size) + for i := 1; i < size; i++ { + n, err := w.Write(buf[:i]) + assert.MustNoError(err) + assert.Must(n == i) + } + assert.MustNoError(w.Close()) + }() + + sum := 0 + for i := 1; i < size; i++ { + sum += i + } + for { + n := <-c + if n == 0 { + break + } + sum -= n + } + assert.Must(sum == 0) +} + +func TestPipe3(t *testing.T) { + testPipe3(t, "") + testPipe3(t, "/tmp/pipe.test") +} + +func testPipe4(t *testing.T, fileName string) { + r, w, f := openPipe(t, fileName) + defer f.Close() + + key := []byte("spinlock aes-128") + + block := aes.BlockSize + count := 1024 * 1024 * 128 / block + + go func() { + buf := make([]byte, count*block) + m, err := aes.NewCipher(key) + assert.MustNoError(err) + for i := 0; i < len(buf); i++ { + buf[i] = byte(i) + } + + e := cipher.NewCBCEncrypter(m, make([]byte, block)) + e.CryptBlocks(buf, buf) + + n, err := w.Write(buf) + assert.MustNoError(err) + assert.MustNoError(w.Close()) + assert.Must(n == len(buf)) + }() + + buf := make([]byte, count*block) + m, err := aes.NewCipher(key) + assert.MustNoError(err) + + n, err := io.ReadFull(r, buf) + assert.MustNoError(err) + assert.Must(n == len(buf)) + + e := cipher.NewCBCDecrypter(m, make([]byte, block)) + e.CryptBlocks(buf, buf) + + for i := 0; i < len(buf); i++ { + assert.Must(buf[i] == byte(i)) + } + _, err = io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + assert.MustNoError(r.Close()) +} + +func TestPipe4(t *testing.T) { + testPipe4(t, "") + testPipe4(t, "/tmp/pipe.test") +} + +type pipeTest struct { + async bool + err error + witherr bool +} + +func (p pipeTest) String() string { + return fmt.Sprintf("async=%v err=%v witherr=%v", p.async, p.err, p.witherr) +} + +var pipeTests = []pipeTest{ + {true, nil, false}, + {true, nil, true}, + {true, io.ErrShortWrite, true}, + {false, nil, false}, + {false, nil, true}, + {false, io.ErrShortWrite, true}, +} + +func delayClose(t *testing.T, closer Closer, c chan int, u pipeTest) { + time.Sleep(time.Millisecond * 10) + var err error + if u.witherr { + err = closer.CloseWithError(u.err) + } else { + err = closer.Close() + } + assert.MustNoError(err) + c <- 0 +} + +func TestPipeReadClose(t *testing.T) { + for _, u := range pipeTests { + r, w := New() + c := make(chan int, 1) + + if u.async { + go delayClose(t, w, c, u) + } else { + delayClose(t, w, c, u) + } + + buf := make([]byte, 64) + n, err := r.Read(buf) + <-c + + expect := u.err + if expect == nil { + expect = io.EOF + } + assert.Must(errors.Equal(err, expect)) + assert.Must(n == 0) + assert.MustNoError(r.Close()) + } +} + +func TestPipeReadClose2(t *testing.T) { + r, w := New() + c := make(chan int, 1) + + go delayClose(t, r, c, pipeTest{}) + + n, err := r.Read(make([]byte, 64)) + <-c + + assert.Must(errors.Equal(err, io.ErrClosedPipe)) + assert.Must(n == 0) + assert.MustNoError(w.Close()) +} + +func TestPipeWriteClose(t *testing.T) { + for _, u := range pipeTests { + r, w := New() + c := make(chan int, 1) + + if u.async { + go delayClose(t, r, c, u) + } else { + delayClose(t, r, c, u) + } + <-c + + n, err := w.Write([]byte("hello, world")) + expect := u.err + if expect == nil { + expect = io.ErrClosedPipe + } + assert.Must(errors.Equal(err, expect)) + assert.Must(n == 0) + assert.MustNoError(w.Close()) + } +} + +func TestWriteEmpty(t *testing.T) { + r, w := New() + + go func() { + n, err := w.Write([]byte{}) + assert.MustNoError(err) + assert.Must(n == 0) + assert.MustNoError(w.Close()) + }() + + time.Sleep(time.Millisecond * 10) + + buf := make([]byte, 4096) + n, err := io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + assert.Must(n == 0) + assert.MustNoError(r.Close()) +} + +func TestWriteNil(t *testing.T) { + r, w := New() + + go func() { + n, err := w.Write(nil) + assert.MustNoError(err) + assert.Must(n == 0) + assert.MustNoError(w.Close()) + }() + + time.Sleep(time.Millisecond * 10) + + buf := make([]byte, 4096) + n, err := io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + assert.Must(n == 0) + assert.MustNoError(r.Close()) +} + +func TestWriteAfterWriterClose(t *testing.T) { + r, w := New() + + s := "hello" + + errs := make(chan error) + + go func() { + n, err := w.Write([]byte(s)) + assert.MustNoError(err) + assert.Must(n == len(s)) + assert.MustNoError(w.Close()) + _, err = w.Write([]byte("world")) + errs <- err + }() + + buf := make([]byte, 4096) + n, err := io.ReadFull(r, buf) + assert.Must(errors.Equal(err, io.EOF)) + assert.Must(string(buf[:n]) == s) + + err = <-errs + assert.Must(errors.Equal(err, io.ErrClosedPipe)) + assert.MustNoError(r.Close()) +} + +func TestWriteRead(t *testing.T) { + r, w := New() + p := make(chan []byte, 1) + + go func() { + var x []byte + for { + b := make([]byte, 23) + n, err := r.Read(b) + if n != 0 { + x = append(x, b[:n]...) + } + if err != nil { + p <- x + return + } + } + }() + + b := make([]byte, 1024*1024*128) + for i := 0; i < len(b); i++ { + b[i] = byte(i) + } + n, err := w.Write(b) + assert.MustNoError(err) + assert.Must(n == len(b)) + + w.Close() + + x := <-p + assert.Must(len(x) == len(b)) + assert.Must(bytes.Equal(b, x)) + + n, err = r.Read(b) + assert.Must(err != nil && n == 0) +} diff --git a/src/pkg/libs/io/pipe/pipeio.go b/src/pkg/libs/io/pipe/pipeio.go new file mode 100644 index 0000000..586013d --- /dev/null +++ b/src/pkg/libs/io/pipe/pipeio.go @@ -0,0 +1,63 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package pipe + +import "io" + +type Closer interface { + io.Closer + CloseWithError(err error) error +} + +type Reader interface { + io.Reader + Closer + Buffered() (int, error) +} + +type reader struct { + p *pipe +} + +func (r *reader) Read(b []byte) (int, error) { + return r.p.Read(b) +} + +func (r *reader) Close() error { + return r.p.RClose(nil) +} + +func (r *reader) CloseWithError(err error) error { + return r.p.RClose(err) +} + +func (r *reader) Buffered() (int, error) { + return r.p.Buffered() +} + +type Writer interface { + io.Writer + Closer + Available() (int, error) +} + +type writer struct { + p *pipe +} + +func (w *writer) Write(b []byte) (int, error) { + return w.p.Write(b) +} + +func (w *writer) Close() error { + return w.p.WClose(nil) +} + +func (w *writer) CloseWithError(err error) error { + return w.p.WClose(err) +} + +func (w *writer) Available() (int, error) { + return w.p.Available() +} diff --git a/src/pkg/libs/log/log.go b/src/pkg/libs/log/log.go new file mode 100644 index 0000000..fd7cce7 --- /dev/null +++ b/src/pkg/libs/log/log.go @@ -0,0 +1,608 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package log + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "sync" + "sync/atomic" + + "pkg/libs/errors" + "pkg/libs/trace" +) + +const ( + Ldate = log.Ldate + Llongfile = log.Llongfile + Lmicroseconds = log.Lmicroseconds + Lshortfile = log.Lshortfile + LstdFlags = log.LstdFlags + Ltime = log.Ltime +) + +type ( + LogType int64 + LogLevel int64 +) + +const ( + TYPE_ERROR = LogType(1 << iota) + TYPE_WARN + TYPE_INFO + TYPE_DEBUG + TYPE_PANIC = LogType(^0) +) + +const ( + LEVEL_NONE = LogLevel(1< opid + OpdelOpid map[string]int64 +} + +// ParseRedisOplogInfo convert result of info oplog to map[uint64]int64( --> ). +// +// info oplog reply example: +// # Oplog +// current_opid:1 +// opapply_source_count:1 +// opapply_source_0:server_id=3171317,applied_opid=1 +// opdel_source_count:2 +// opdel_source_0:source_name=bls_channel_02,to_del_opid=1,last_update_time=1500279590 +// opdel_source_1:source_name=bls_channel_01,to_del_opid=1,last_update_time=1500279587 +func ParseRedisInfoOplog(oplogInfo []byte) (*RedisInfoOplog, error) { + p := new(RedisInfoOplog) + + var err error + var i int + var opapplySourceCount uint64 + + // "opapply_source_count:1\r\nopapply_source_0:server_id=3171317,applied_opid=1\r\n" is converted to map[string]string{"opapply_source_count": "1", "opapply_source_0": "server_id=3171317,applied_opid=1"}. + KV := ParseInfo(oplogInfo) + + // current_opid + if value, ok := KV["current_opid"]; ok { + p.CurrentOpid, err = strconv.ParseInt(value, 10, 0) + if err != nil { + return nil, err + } + } else { + p.CurrentOpid = -1 + } + + // opapply_source_count:xxx + if value, ok := KV["opapply_source_count"]; ok { + opapplySourceCount, err = strconv.ParseUint(value, 10, 0) + if err != nil { + return nil, err + } + } else { + goto return_error + } + + p.GtidSet = make(map[uint64]int64) + for i = 0; i < int(opapplySourceCount); i++ { + key := fmt.Sprintf("opapply_source_%d", i) // key "opapply_source_0" + value, ok := KV[key] // value "server_id=7031,applied_opid=9842282" + if !ok { + goto return_error + } + + subKV := ParseValue(value) + + var serverID uint64 + var appliedOpid int64 + if serverIDStr, ok := subKV["server_id"]; ok { + serverID, err = strconv.ParseUint(serverIDStr, 10, 0) + if err != nil { + return nil, err + } + } else { + goto return_error + } + if appliedOpidStr, ok := subKV["applied_opid"]; ok { + appliedOpid, err = strconv.ParseInt(appliedOpidStr, 10, 0) + if err != nil { + return nil, err + } + } else { + goto return_error + } + p.GtidSet[serverID] = appliedOpid + } + return p, nil +return_error: + return nil, fmt.Errorf("invalid opapply info:\n %s", string(oplogInfo)) +} diff --git a/src/pkg/libs/oplog/oplog.go b/src/pkg/libs/oplog/oplog.go new file mode 100644 index 0000000..044d8bf --- /dev/null +++ b/src/pkg/libs/oplog/oplog.go @@ -0,0 +1,274 @@ +package oplog + +import ( + "bytes" + "encoding/binary" + "fmt" + "strconv" + "strings" + "time" + "unsafe" +) + +type errorOplog struct { + s string +} + +func (e *errorOplog) Error() string { + return e.s +} + +/* +struct OplogHeader { + uint32_t version:8; // version of oplog + uint32_t cmd_num:4; / number of commands in one oplog, currently 2 or 3 + uint32_t cmd_flag:4; + uint32_t dbid:16; + int32_t timestamp; + int64_t server_id; + int64_t opid; + int64_t src_opid; // opid of source redis +}; +*/ +type OplogHeader struct { + Version int8 + Status uint8 + DbId int16 + Timestamp int32 + ServerId uint64 + Opid int64 + SrcOpid int64 +} + +func (p *OplogHeader) IsOPLogDelByExpire() bool { + // #define REDIS_OPLOG_DEL_BY_EVICTION_FLAG (1<<7) + // #define REDIS_OPLOG_DEL_BY_EXPIRE_FLAG (1<<6) + return p.Status&(uint8(1)<<6) != 0 +} + +func (p *OplogHeader) IsOPLogDelByEviction() bool { + // #define REDIS_OPLOG_DEL_BY_EVICTION_FLAG (1<<7) + // #define REDIS_OPLOG_DEL_BY_EXPIRE_FLAG (1<<6) + return p.Status&(uint8(1)<<7) != 0 +} + +func (p *OplogHeader) GetCmdNum() int { + return int(p.Status & uint8(0xF0)) +} + +func (p OplogHeader) String() string { + t := time.Unix(int64(p.Timestamp), 0).Local() + return fmt.Sprintf("{Version:%d, Status:%d, Dbid:%d, Timestamp:%d, Time:%s, ServerId:%d, Opid:%d, SrcOpid:%d, IsDelByExpire:%v, IsDelByEviction:%v}", + p.Version, p.Status, p.DbId, p.Timestamp, t.Format(time.RFC3339), p.ServerId, p.Opid, p.SrcOpid, p.IsOPLogDelByExpire(), p.IsOPLogDelByEviction()) +} + +const OplogHeaderSize = unsafe.Sizeof(OplogHeader{}) + +var OplogHeaderPrefix = []byte("*2\r\n$6\r\nOPINFO\r\n$32\r\n") + +type Oplog struct { + FullContent []byte + Header OplogHeader + Cmd []RedisCmd +} + +func (p *Oplog) CmdContent() []byte { + return p.FullContent[len(OplogHeaderPrefix)+32+2:] +} + +func (p *Oplog) IsOPLogDelByExpire() bool { + return p.Header.IsOPLogDelByExpire() +} + +func (p *Oplog) IsOPLogDelByEviction() bool { + return p.Header.IsOPLogDelByEviction() +} + +// parseLen parses bulk string and array lengths. +func parseLen(p []byte) (int64, error) { + if len(p) == 0 { + return -1, &errorOplog{"protocal error, malformed length"} + } + + if p[0] == '-' && len(p) == 2 && p[1] == '1' { + // handle $-1 and $-1 null replies. + return -1, nil + } + + var n int64 + for _, b := range p { + n *= 10 + if b < '0' || b > '9' { + return -1, &errorOplog{"protocal error, illegal bytes in length"} + } + n += int64(b - '0') + } + + return n, nil +} + +func parseCmd(fullcontent []byte) (*RedisCmd, []byte, error) { + if fullcontent[0] != '*' { + return nil, fullcontent, &errorOplog{fmt.Sprintf("protocal error, cmd not start with '*': %s", string(fullcontent))} + } + p := bytes.IndexByte(fullcontent, '\n') + if p == -1 || fullcontent[p-1] != '\r' { + return nil, fullcontent, &errorOplog{fmt.Sprintf("protocal error, expect line terminator: %s", string(fullcontent))} + } + + arrayLen, err := parseLen(fullcontent[1 : p-1]) + if err != nil { + return nil, fullcontent, err + } + + reply := RedisCmd{Args: make([][]byte, 0, arrayLen)} + left := fullcontent[p+1:] + for i := int64(0); i < arrayLen; i++ { + if left[0] != '$' { + return nil, fullcontent, &errorOplog{fmt.Sprintf("protocal error, expect '$': %s", string(left))} + } + endIndex := bytes.IndexByte(left, '\n') + if endIndex == -1 || left[endIndex-1] != '\r' { + return nil, fullcontent, &errorOplog{fmt.Sprintf("protocal error, expect line terminator: %s", string(left))} + } + bulkLen, err := parseLen(left[1 : endIndex-1]) + if err != nil { + return nil, fullcontent, err + } + reply.Args = append(reply.Args, left[endIndex+1:endIndex+1+int(bulkLen)]) + left = left[endIndex+1+int(bulkLen)+2:] + } + + reply.CmdCode = ParseCommandStrToCode(reply.Args[0]) + return &reply, left, nil +} + +func ParseOplog(fullcontent []byte) (*Oplog, error) { + oplog := &Oplog{FullContent: fullcontent} + left := fullcontent + for { + var redisCmd *RedisCmd + var err error + redisCmd, left, err = parseCmd(left) + if err != nil { + return nil, err + } + oplog.Cmd = append(oplog.Cmd, *redisCmd) + if len(left) == 0 { + break + } + } + + if oplog.Cmd[0].CmdCode != OPINFO { + return nil, &errorOplog{fmt.Sprintf("oplog error, first cmd is not OPINFO, but is %s", string(oplog.Cmd[0].Args[0]))} + } + + if len(oplog.Cmd[0].Args[1]) != int(OplogHeaderSize) { + return nil, &errorOplog{fmt.Sprintf("oplog error, len(oplog.Cmd[0].args[1]) %d != int(OplogHeaderSize) %d", + len(oplog.Cmd[0].Args[1]), int(OplogHeaderSize))} + } + + dest := (*(*[1<<31 - 1]byte)(unsafe.Pointer(&oplog.Header)))[:OplogHeaderSize] + copy(dest, oplog.Cmd[0].Args[1]) + return oplog, nil +} + +func ParseOplogHeader(content []byte) (*OplogHeader, error) { + if len(content) != int(OplogHeaderSize) { + return nil, &errorOplog{fmt.Sprintf("parse oplog header failed, len(content) %d != int(OplogHeaderSize) %d", + len(content), int(OplogHeaderSize))} + } + var reply OplogHeader + dest := (*(*[1<<31 - 1]byte)(unsafe.Pointer(&reply)))[:OplogHeaderSize] + copy(dest, content) + return &reply, nil +} + +type FakeOplogMaker struct { + header OplogHeader + readCmdContent []byte +} + +func NewFakeOplogMaker(serverId uint64) *FakeOplogMaker { + // 必须选SCRIPT LOAD，优点如下： + // 1. 不涉及key，不会和用户的key冲突 + // 2. 不涉及事务或lua中的eval/evalsha, 不会干扰主从版升级集群版 + // 3. 对于集群版，proxy会把"OPINFO"和"SCRIPT LOAD"命令转发到所有后端redis db，省事 + var args [3]string + args[0] = "SCRIPT" + args[1] = "LOAD" + args[2] = "return 0" + + var buf bytes.Buffer + buf.WriteString("*3\r\n") + for _, arg := range args[:] { + buf.WriteString(fmt.Sprintf("$%d\r\n%s\r\n", len(arg), arg)) + } + + return &FakeOplogMaker{ + header: OplogHeader{ + Version: 1, + Status: 2, // 8bit, [0-4] command num, [5-7] flags + DbId: 0, + Timestamp: 0, + ServerId: serverId, + Opid: 0, + SrcOpid: -1, + }, + readCmdContent: buf.Bytes(), + } +} + +func (p *FakeOplogMaker) MakeFakeOplog(opid int64) (*Oplog, error) { + p.header.Timestamp = int32(time.Now().Unix()) + p.header.Opid = opid + + var buf bytes.Buffer + buf.Write(OplogHeaderPrefix) + binary.Write(&buf, binary.LittleEndian, p.header) + buf.WriteString("\r\n") + buf.Write(p.readCmdContent) + + return ParseOplog(buf.Bytes()) +} + +// ParsePSyncFullResp parse applyinfo which is response of "psync ? -1". For example, applyinfo is "applied_info{0:100,7171317:1867040,1:100}". +func ParsePsyncFullApplyInfo(applyinfo string) (map[uint64]int64, error) { + reply := make(map[uint64]int64) + var content string + var err error + var kvs, kv []string + var key uint64 + var value int64 + + if !strings.HasPrefix(applyinfo, "applied_info{") { + goto error_return + } + if !strings.HasSuffix(applyinfo, "}") { + goto error_return + } + content = applyinfo[len("applied_info{") : len(applyinfo)-len("}")] + if len(content) != 0 { + kvs = strings.Split(content, ",") + for _, item := range kvs { + // item "0:100" + kv = strings.Split(item, ":") + if len(kv) != 2 { + goto error_return + } + key, err = strconv.ParseUint(kv[0], 10, 0) + if err != nil { + return nil, err + } + value, err = strconv.ParseInt(kv[1], 10, 0) + if err != nil { + return nil, err + } + reply[key] = value + } + } + return reply, nil + +error_return: + return nil, fmt.Errorf("invalid apply info string: %s", applyinfo) +} diff --git a/src/pkg/libs/oplog/parseinfo.go b/src/pkg/libs/oplog/parseinfo.go new file mode 100644 index 0000000..7cab686 --- /dev/null +++ b/src/pkg/libs/oplog/parseinfo.go @@ -0,0 +1,36 @@ +package oplog + +import ( + "bytes" + "strings" +) + +// ParseInfo convert result of info command to map[string]string. +// For example, "opapply_source_count:1\r\nopapply_source_0:server_id=3171317,applied_opid=1\r\n" +// is converted to map[string]string{"opapply_source_count": "1", "opapply_source_0": "server_id=3171317,applied_opid=1"}. +func ParseInfo(content []byte) map[string]string { + result := make(map[string]string, 10) + lines := bytes.Split(content, []byte("\r\n")) + for i := 0; i < len(lines); i++ { + items := bytes.SplitN(lines[i], []byte(":"), 2) + if len(items) != 2 { + continue + } + result[string(items[0])] = string(items[1]) + } + return result +} + +// ParseValue convert value of one item from info command result to map[string]string. +// For example, "server_id=3171317,applied_opid=1" is converted to map[string]string{"server_id": "3171317", "applied_opid": "1"}. +func ParseValue(content string) map[string]string { + result := make(map[string]string, 2) + items := strings.Split(content, ",") + for i := 0; i < len(items); i++ { + v := strings.SplitN(items[i], "=", 2) + if len(v) == 2 { + result[v[0]] = v[1] + } + } + return result +} diff --git a/src/pkg/libs/stats/iocount.go b/src/pkg/libs/stats/iocount.go new file mode 100644 index 0000000..d605bfc --- /dev/null +++ b/src/pkg/libs/stats/iocount.go @@ -0,0 +1,62 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package stats + +import ( + "io" + + "pkg/libs/atomic2" +) + +type CountReader struct { + p *atomic2.Int64 + r io.Reader +} + +func NewCountReader(r io.Reader, p *atomic2.Int64) *CountReader { + if p == nil { + p = &atomic2.Int64{} + } + return &CountReader{p: p, r: r} +} + +func (r *CountReader) Count() int64 { + return r.p.Get() +} + +func (r *CountReader) ResetCounter() int64 { + return r.p.Swap(0) +} + +func (r *CountReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.p.Add(int64(n)) + return n, err +} + +type CountWriter struct { + p *atomic2.Int64 + w io.Writer +} + +func NewCountWriter(w io.Writer, p *atomic2.Int64) *CountWriter { + if p == nil { + p = &atomic2.Int64{} + } + return &CountWriter{p: p, w: w} +} + +func (w *CountWriter) Count() int64 { + return w.p.Get() +} + +func (w *CountWriter) ResetCounter() int64 { + return w.p.Swap(0) +} + +func (w *CountWriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.p.Add(int64(n)) + return n, err +} diff --git a/src/pkg/libs/trace/trace.go b/src/pkg/libs/trace/trace.go new file mode 100644 index 0000000..c48314b --- /dev/null +++ b/src/pkg/libs/trace/trace.go @@ -0,0 +1,88 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package trace + +import ( + "bytes" + "fmt" + "runtime" + "strings" +) + +const ( + tab = " " +) + +type Record struct { + Name string + File string + Line int +} + +func (r *Record) String() string { + if r == nil { + return "[nil-record]" + } + return fmt.Sprintf("%s:%d %s", r.File, r.Line, r.Name) +} + +type Stack []*Record + +func Trace() Stack { + return TraceN(1, 32) +} + +func (s Stack) String() string { + return s.StringWithIndent(0) +} + +func (s Stack) StringWithIndent(indent int) string { + var b bytes.Buffer + for i, r := range s { + for j := 0; j < indent; j++ { + fmt.Fprint(&b, tab) + } + fmt.Fprintf(&b, "%-3d %s:%d\n", len(s)-i-1, r.File, r.Line) + for j := 0; j < indent; j++ { + fmt.Fprint(&b, tab) + } + fmt.Fprint(&b, tab, tab) + fmt.Fprint(&b, r.Name, "\n") + } + if len(s) != 0 { + for j := 0; j < indent; j++ { + fmt.Fprint(&b, tab) + } + fmt.Fprint(&b, tab, "... ...\n") + } + return b.String() +} + +func TraceN(skip, depth int) Stack { + s := make([]*Record, 0, depth) + for i := 0; i < depth; i++ { + r := Caller(skip + i + 1) + if r == nil { + break + } + s = append(s, r) + } + return s +} + +func Caller(skip int) *Record { + pc, file, line, ok := runtime.Caller(skip + 1) + if !ok { + return nil + } + fn := runtime.FuncForPC(pc) + if fn == nil || strings.HasPrefix(fn.Name(), "runtime.") { + return nil + } + return &Record{ + Name: fn.Name(), + File: file, + Line: line, + } +} diff --git a/src/pkg/rdb/decoder.go b/src/pkg/rdb/decoder.go new file mode 100644 index 0000000..3c398a1 --- /dev/null +++ b/src/pkg/rdb/decoder.go @@ -0,0 +1,156 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + + "pkg/libs/cupcake/rdb" + "pkg/libs/cupcake/rdb/nopdecoder" + "pkg/libs/errors" +) + +func DecodeDump(p []byte) (interface{}, error) { + d := &decoder{} + if err := rdb.DecodeDump(p, 0, nil, 0, d); err != nil { + return nil, errors.Trace(err) + } + return d.obj, d.err +} + +type decoder struct { + nopdecoder.NopDecoder + obj interface{} + err error +} + +func (d *decoder) initObject(obj interface{}) { + if d.err != nil { + return + } + if d.obj != nil { + d.err = errors.Errorf("invalid object, init again") + } else { + d.obj = obj + } +} + +func (d *decoder) Set(key, value []byte, expiry int64) { + d.initObject(String(value)) +} + +func (d *decoder) StartHash(key []byte, length, expiry int64) { + d.initObject(Hash(nil)) +} + +func (d *decoder) Hset(key, field, value []byte) { + if d.err != nil { + return + } + switch h := d.obj.(type) { + default: + d.err = errors.Errorf("invalid object, not a hashmap") + case Hash: + v := &HashElement{Field: field, Value: value} + d.obj = append(h, v) + } +} + +func (d *decoder) StartSet(key []byte, cardinality, expiry int64) { + d.initObject(Set(nil)) +} + +func (d *decoder) Sadd(key, member []byte) { + if d.err != nil { + return + } + switch s := d.obj.(type) { + default: + d.err = errors.Errorf("invalid object, not a set") + case Set: + d.obj = append(s, member) + } +} + +func (d *decoder) StartList(key []byte, length, expiry int64) { + d.initObject(List(nil)) +} + +func (d *decoder) Rpush(key, value []byte) { + if d.err != nil { + return + } + switch l := d.obj.(type) { + default: + d.err = errors.Errorf("invalid object, not a list") + case List: + d.obj = append(l, value) + } +} + +func (d *decoder) StartZSet(key []byte, cardinality, expiry int64) { + d.initObject(ZSet(nil)) +} + +func (d *decoder) Zadd(key []byte, score float64, member []byte) { + if d.err != nil { + return + } + switch z := d.obj.(type) { + default: + d.err = errors.Errorf("invalid object, not a zset") + case ZSet: + v := &ZSetElement{Member: member, Score: score} + d.obj = append(z, v) + } +} + +type String []byte +type List [][]byte +type Hash []*HashElement +type ZSet []*ZSetElement +type Set [][]byte + +type HashElement struct { + Field, Value []byte +} + +type ZSetElement struct { + Member []byte + Score float64 +} + +func (hash Hash) Len() int { + return len(hash) +} + +func (hash Hash) Swap(i, j int) { + hash[i], hash[j] = hash[j], hash[i] +} + +type HSortByField struct{ Hash } + +func (by HSortByField) Less(i, j int) bool { + return bytes.Compare(by.Hash[i].Field, by.Hash[j].Field) < 0 +} + +func (zset ZSet) Len() int { + return len(zset) +} + +func (zset ZSet) Swap(i, j int) { + zset[i], zset[j] = zset[j], zset[i] +} + +type ZSortByMember struct{ ZSet } + +func (by ZSortByMember) Less(i, j int) bool { + return bytes.Compare(by.ZSet[i].Member, by.ZSet[j].Member) < 0 +} + +type ZSortByScore struct{ ZSet } + +func (by ZSortByScore) Less(i, j int) bool { + return by.ZSet[i].Score < by.ZSet[j].Score +} diff --git a/src/pkg/rdb/decoder_test.go b/src/pkg/rdb/decoder_test.go new file mode 100644 index 0000000..8d8b417 --- /dev/null +++ b/src/pkg/rdb/decoder_test.go @@ -0,0 +1,190 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/hex" + "math" + "strconv" + "strings" + "testing" + + "pkg/libs/assert" +) + +func hexStringToObject(t *testing.T, s string) interface{} { + p, err := hex.DecodeString(strings.NewReplacer("\t", "", "\r", "", "\n", "", " ", "").Replace(s)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + assert.Must(o != nil) + return o +} + +/* +#!/bin/bash +for i in 1 255 256 65535 65536 2147483647 2147483648 4294967295 4294967296; do + ./redis-cli set string ${i} + ./redis-cli dump string +done +./redis-cli set string "hello world!!" +./redis-cli dump string +s="" +for ((i=0;i<1024;i++)); do + s="01"$s +done +./redis-cli set string $s +./redis-cli dump string +*/ +func TestDecodeString(t *testing.T) { + docheck := func(hexs string, expect string) { + val := hexStringToObject(t, hexs).(String) + assert.Must(bytes.Equal([]byte(val), []byte(expect))) + } + docheck("00c0010600b0958f3624542d6f", "1") + docheck("00c1ff0006004a42131348a52fa4", "255") + docheck("00c1000106009cb3bb1c58e36c78", "256") + docheck("00c2ffff0000060047a5299686680606", "65535") + docheck("00c200000100060056e7032772340449", "65536") + docheck("00c2ffffff7f0600ba998d1e157b9132", "2147483647") + docheck("000a323134373438333634380600715c4123b9484a7d", "2147483648") + docheck("000a3432393439363732393506009a94b642c60c15f2", "4294967295") + docheck("000a343239343936373239360600334ee148efd97ac5", "4294967296") + + docheck("000d68656c6c6f20776f726c64212106004aa70c88a8ad3601", "hello world!!") + var b bytes.Buffer + for i := 0; i < 1024; i++ { + b.Write([]byte("01")) + } + docheck("00c31f480002303130e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ba010130310600bcd6e486102c99c7", b.String()) +} + +/* +#!/bin/bash +for ((i=0;i<32;i++)); do + ./redis-cli rpush list $i +done +./redis-cli dump list +*/ +func TestDecodeListZipmap(t *testing.T) { + s := ` + 0a405e5e0000005a000000200000f102f202f302f402f502f602f702f802f902 + fa02fb02fc02fd02fe0d03fe0e03fe0f03fe1003fe1103fe1203fe1303fe1403 + fe1503fe1603fe1703fe1803fe1903fe1a03fe1b03fe1c03fe1d03fe1e03fe1f + ff060052f7f617938b332a + ` + val := hexStringToObject(t, s).(List) + assert.Must(len(val) == 32) + for i := 0; i < len(val); i++ { + assert.Must(string(val[i]) == strconv.Itoa(i)) + } +} + +/* +#!/bin/bash +for ((i=0;i<32;i++)); do + ./redis-cli rpush list $i +done +./redis-cli dump list +*/ +func TestDecodeList(t *testing.T) { + s := ` + 0120c000c001c002c003c004c005c006c007c008c009c00ac00bc00cc00dc00e + c00fc010c011c012c013c014c015c016c017c018c019c01ac01bc01cc01dc01e + c01f0600e87781cbebc997f5 + ` + val := hexStringToObject(t, s).(List) + assert.Must(len(val) == 32) + for i := 0; i < len(val); i++ { + assert.Must(string(val[i]) == strconv.Itoa(i)) + } +} + +/* +#!/bin/bash +for ((i=0;i<32;i++)); do + ./redis-cli sadd set $i +done +./redis-cli dump set +*/ +func TestDecodeSet(t *testing.T) { + s := ` + 0220c016c00dc01bc012c01ac004c014c002c017c01dc01cc013c019c01ec008 + c006c000c001c007c00fc009c01fc00ec003c00ac015c010c00bc018c011c00c + c00506007bd0a89270890016 + ` + val := hexStringToObject(t, s).(Set) + assert.Must(len(val) == 32) + set := make(map[string]bool) + for _, mem := range val { + set[string(mem)] = true + } + assert.Must(len(val) == len(set)) + for i := 0; i < 32; i++ { + _, ok := set[strconv.Itoa(i)] + assert.Must(ok) + } +} + +/* +#!/bin/bash +for ((i=0;i<32;i++)); do + let j="$i*$i" + ./redis-cli hset hash $i $j +done +./redis-cli dump hash +*/ +func TestDecodeHash(t *testing.T) { + s := ` + 0420c016c1e401c00dc1a900c01bc1d902c012c14401c01ac1a402c004c010c0 + 02c004c014c19001c017c11102c01dc14903c01cc11003c013c16901c019c171 + 02c01ec18403c008c040c006c024c000c000c001c001c007c031c009c051c00f + c1e100c01fc1c103c00ec1c400c003c009c00ac064c015c1b901c010c10001c0 + 0bc079c018c14002c011c12101c00cc19000c005c019060072320e870e10799d + ` + val := hexStringToObject(t, s).(Hash) + assert.Must(len(val) == 32) + hash := make(map[string]string) + for _, ent := range val { + hash[string(ent.Field)] = string(ent.Value) + } + assert.Must(len(val) == len(hash)) + for i := 0; i < 32; i++ { + s := strconv.Itoa(i) + assert.Must(hash[s] == strconv.Itoa(i*i)) + } +} + +/* +#!/bin/bash +for ((i=0;i<32;i++)); do + ./redis-cli zadd zset -$i $i +done +./redis-cli dump zset +*/ +func TestDecodeZSet(t *testing.T) { + s := ` + 0320c016032d3232c00d032d3133c01b032d3237c012032d3138c01a032d3236 + c004022d34c014032d3230c002022d32c017032d3233c01d032d3239c01c032d + 3238c013032d3139c019032d3235c01e032d3330c008022d38c006022d36c000 + 0130c001022d31c007022d37c009022d39c00f032d3135c01f032d3331c00e03 + 2d3134c003022d33c00a032d3130c015032d3231c010032d3136c00b032d3131 + c018032d3234c011032d3137c00c032d3132c005022d35060046177397f6688b + 16 + ` + val := hexStringToObject(t, s).(ZSet) + assert.Must(len(val) == 32) + zset := make(map[string]float64) + for _, ent := range val { + zset[string(ent.Member)] = ent.Score + } + assert.Must(len(val) == len(zset)) + for i := 0; i < 32; i++ { + s := strconv.Itoa(i) + score, ok := zset[s] + assert.Must(ok) + assert.Must(math.Abs(score+float64(i)) < 1e-10) + } +} diff --git a/src/pkg/rdb/digest/crc64.go b/src/pkg/rdb/digest/crc64.go new file mode 100644 index 0000000..eb011a7 --- /dev/null +++ b/src/pkg/rdb/digest/crc64.go @@ -0,0 +1,106 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package digest + +import ( + "encoding/binary" + "hash" +) + +var crc64_table = [256]uint64{ + 0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, + 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, + 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, + 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, + 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, + 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, + 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, + 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, + 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, + 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, + 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, + 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, + 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, + 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, + 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, + 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, + 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, + 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, + 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, + 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, + 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, + 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, + 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, + 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, + 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, + 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, + 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, + 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, + 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, + 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, + 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, + 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, + 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, + 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, + 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, + 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, + 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, + 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, + 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, + 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, + 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, + 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, + 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, + 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, + 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, + 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, + 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, + 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, + 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, + 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, + 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, + 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, + 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, + 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, + 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, + 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, + 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, + 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, + 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, + 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, + 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, + 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, + 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, + 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} + +type digest struct { + crc uint64 +} + +func (d *digest) update(p []byte) { + for _, b := range p { + d.crc = crc64_table[byte(d.crc)^b] ^ (d.crc >> 8) + } +} + +func New() hash.Hash64 { + d := &digest{} + return d +} + +func (d *digest) Write(p []byte) (int, error) { + d.update(p) + return len(p), nil +} + +func (d *digest) Sum(in []byte) []byte { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, d.crc) + return append(in, buf...) +} + +func (d *digest) Sum64() uint64 { return d.crc } +func (d *digest) BlockSize() int { return 1 } +func (d *digest) Size() int { return 8 } +func (d *digest) Reset() { d.crc = 0 } diff --git a/src/pkg/rdb/encoder.go b/src/pkg/rdb/encoder.go new file mode 100644 index 0000000..b1ddf15 --- /dev/null +++ b/src/pkg/rdb/encoder.go @@ -0,0 +1,170 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "io" + + "github.com/cupcake/rdb" + "pkg/libs/errors" +) + +type objectEncoder interface { + encodeType(enc *rdb.Encoder) error + encodeValue(enc *rdb.Encoder) error +} + +func (o String) encodeType(enc *rdb.Encoder) error { + t := rdb.ValueType(RdbTypeString) + return errors.Trace(enc.EncodeType(t)) +} + +func (o String) encodeValue(enc *rdb.Encoder) error { + if err := enc.EncodeString([]byte(o)); err != nil { + return errors.Trace(err) + } + return nil +} + +func (o Hash) encodeType(enc *rdb.Encoder) error { + t := rdb.ValueType(RdbTypeHash) + return errors.Trace(enc.EncodeType(t)) +} + +func (o Hash) encodeValue(enc *rdb.Encoder) error { + if err := enc.EncodeLength(uint32(len(o))); err != nil { + return errors.Trace(err) + } + for _, e := range o { + if err := enc.EncodeString(e.Field); err != nil { + return errors.Trace(err) + } + if err := enc.EncodeString(e.Value); err != nil { + return errors.Trace(err) + } + } + return nil +} + +func (o List) encodeType(enc *rdb.Encoder) error { + t := rdb.ValueType(RdbTypeList) + return errors.Trace(enc.EncodeType(t)) +} + +func (o List) encodeValue(enc *rdb.Encoder) error { + if err := enc.EncodeLength(uint32(len(o))); err != nil { + return errors.Trace(err) + } + for _, e := range o { + if err := enc.EncodeString(e); err != nil { + return errors.Trace(err) + } + } + return nil +} + +func (o ZSet) encodeType(enc *rdb.Encoder) error { + t := rdb.ValueType(RdbTypeZSet) + return errors.Trace(enc.EncodeType(t)) +} + +func (o ZSet) encodeValue(enc *rdb.Encoder) error { + if err := enc.EncodeLength(uint32(len(o))); err != nil { + return errors.Trace(err) + } + for _, e := range o { + if err := enc.EncodeString(e.Member); err != nil { + return errors.Trace(err) + } + if err := enc.EncodeFloat(e.Score); err != nil { + return errors.Trace(err) + } + } + return nil +} + +func (o Set) encodeType(enc *rdb.Encoder) error { + t := rdb.ValueType(RdbTypeSet) + return errors.Trace(enc.EncodeType(t)) +} + +func (o Set) encodeValue(enc *rdb.Encoder) error { + if err := enc.EncodeLength(uint32(len(o))); err != nil { + return errors.Trace(err) + } + for _, e := range o { + if err := enc.EncodeString(e); err != nil { + return errors.Trace(err) + } + } + return nil +} + +func EncodeDump(obj interface{}) ([]byte, error) { + o, ok := obj.(objectEncoder) + if !ok { + return nil, errors.Errorf("unsupported object type") + } + var b bytes.Buffer + enc := rdb.NewEncoder(&b) + if err := o.encodeType(enc); err != nil { + return nil, err + } + if err := o.encodeValue(enc); err != nil { + return nil, err + } + if err := enc.EncodeDumpFooter(); err != nil { + return nil, errors.Trace(err) + } + return b.Bytes(), nil +} + +type Encoder struct { + enc *rdb.Encoder + db int64 +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + enc: rdb.NewEncoder(w), + db: -1, + } +} + +func (e *Encoder) EncodeHeader() error { + return errors.Trace(e.enc.EncodeHeader()) +} + +func (e *Encoder) EncodeFooter() error { + return errors.Trace(e.enc.EncodeFooter()) +} + +func (e *Encoder) EncodeObject(db uint32, key []byte, expireat uint64, obj interface{}) error { + o, ok := obj.(objectEncoder) + if !ok { + return errors.Errorf("unsupported object type") + } + if e.db == -1 || uint32(e.db) != db { + e.db = int64(db) + if err := e.enc.EncodeDatabase(int(db)); err != nil { + return errors.Trace(err) + } + } + if expireat != 0 { + if err := e.enc.EncodeExpiry(expireat); err != nil { + return errors.Trace(err) + } + } + if err := o.encodeType(e.enc); err != nil { + return err + } + if err := e.enc.EncodeString(key); err != nil { + return errors.Trace(err) + } + if err := o.encodeValue(e.enc); err != nil { + return err + } + return nil +} diff --git a/src/pkg/rdb/encoder_test.go b/src/pkg/rdb/encoder_test.go new file mode 100644 index 0000000..fd31546 --- /dev/null +++ b/src/pkg/rdb/encoder_test.go @@ -0,0 +1,293 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "fmt" + "math" + "math/rand" + "strconv" + "testing" + + "pkg/libs/assert" + "pkg/libs/atomic2" + "pkg/libs/stats" +) + +func toString(text string) String { + return String([]byte(text)) +} + +func checkString(t *testing.T, o interface{}, text string) { + x, ok := o.(String) + assert.Must(ok) + assert.Must(string(x) == text) +} + +func TestEncodeString(t *testing.T) { + docheck := func(text string) { + p, err := EncodeDump(toString(text)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + checkString(t, o, text) + } + docheck("hello world!!") + docheck("2147483648") + docheck("4294967296") + docheck("") + var b bytes.Buffer + for i := 0; i < 1024; i++ { + b.Write([]byte("01")) + } + docheck(b.String()) +} + +func toList(list ...string) List { + o := List{} + for _, e := range list { + o = append(o, []byte(e)) + } + return o +} + +func checkList(t *testing.T, o interface{}, list []string) { + x, ok := o.(List) + assert.Must(ok) + assert.Must(len(x) == len(list)) + for i, e := range x { + assert.Must(string(e) == list[i]) + } +} + +func TestEncodeList(t *testing.T) { + docheck := func(list ...string) { + p, err := EncodeDump(toList(list...)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + checkList(t, o, list) + } + docheck("") + docheck("", "a", "b", "c", "d", "e") + list := []string{} + for i := 0; i < 65536; i++ { + list = append(list, strconv.Itoa(i)) + } + docheck(list...) +} + +func toHash(m map[string]string) Hash { + o := Hash{} + for k, v := range m { + o = append(o, &HashElement{Field: []byte(k), Value: []byte(v)}) + } + return o +} + +func checkHash(t *testing.T, o interface{}, m map[string]string) { + x, ok := o.(Hash) + assert.Must(ok) + assert.Must(len(x) == len(m)) + for _, e := range x { + assert.Must(m[string(e.Field)] == string(e.Value)) + } +} + +func TestEncodeHash(t *testing.T) { + docheck := func(m map[string]string) { + p, err := EncodeDump(toHash(m)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + checkHash(t, o, m) + } + docheck(map[string]string{"": ""}) + docheck(map[string]string{"": "", "a": "", "b": "a", "c": "b", "d": "c"}) + hash := make(map[string]string) + for i := 0; i < 65536; i++ { + hash[strconv.Itoa(i)] = strconv.Itoa(i + 1) + } + docheck(hash) +} + +func toZSet(m map[string]float64) ZSet { + o := ZSet{} + for k, v := range m { + o = append(o, &ZSetElement{Member: []byte(k), Score: v}) + } + return o +} + +func checkZSet(t *testing.T, o interface{}, m map[string]float64) { + x, ok := o.(ZSet) + assert.Must(ok) + assert.Must(len(x) == len(m)) + for _, e := range x { + v := m[string(e.Member)] + switch { + case math.IsInf(v, 1): + assert.Must(math.IsInf(e.Score, 1)) + case math.IsInf(v, -1): + assert.Must(math.IsInf(e.Score, -1)) + case math.IsNaN(v): + assert.Must(math.IsNaN(e.Score)) + default: + assert.Must(math.Abs(e.Score-v) < 1e-10) + } + } +} + +func TestEncodeZSet(t *testing.T) { + docheck := func(m map[string]float64) { + p, err := EncodeDump(toZSet(m)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + checkZSet(t, o, m) + } + docheck(map[string]float64{"": 0}) + zset := make(map[string]float64) + for i := -65535; i < 65536; i++ { + zset[strconv.Itoa(i)] = float64(i) + } + docheck(zset) + zset["inf"] = math.Inf(1) + zset["-inf"] = math.Inf(-1) + zset["nan"] = math.NaN() + docheck(zset) +} + +func toSet(set ...string) Set { + o := Set{} + for _, e := range set { + o = append(o, []byte(e)) + } + return o +} + +func checkSet(t *testing.T, o interface{}, set []string) { + x, ok := o.(Set) + assert.Must(ok) + assert.Must(len(x) == len(set)) + for i, e := range x { + assert.Must(string(e) == set[i]) + } +} + +func TestEncodeSet(t *testing.T) { + docheck := func(set ...string) { + p, err := EncodeDump(toSet(set...)) + assert.MustNoError(err) + o, err := DecodeDump(p) + assert.MustNoError(err) + checkSet(t, o, set) + } + docheck("") + docheck("", "a", "b", "c") + set := []string{} + for i := 0; i < 65536; i++ { + set = append(set, strconv.Itoa(i)) + } + docheck(set...) +} + +func TestEncodeRdb(t *testing.T) { + objs := make([]struct { + db uint32 + expireat uint64 + key []byte + obj interface{} + typ string + }, 128) + var b bytes.Buffer + enc := NewEncoder(&b) + assert.MustNoError(enc.EncodeHeader()) + for i := 0; i < len(objs); i++ { + db := uint32(i + 32) + expireat := uint64(i) + key := []byte(strconv.Itoa(i)) + var obj interface{} + var typ string + switch i % 5 { + case 0: + s := strconv.Itoa(i) + obj = s + typ = "string" + assert.MustNoError(enc.EncodeObject(db, key, expireat, toString(s))) + case 1: + list := []string{} + for j := 0; j < 32; j++ { + list = append(list, fmt.Sprintf("l%d_%d", i, rand.Int())) + } + obj = list + typ = "list" + assert.MustNoError(enc.EncodeObject(db, key, expireat, toList(list...))) + case 2: + hash := make(map[string]string) + for j := 0; j < 32; j++ { + hash[strconv.Itoa(j)] = fmt.Sprintf("h%d_%d", i, rand.Int()) + } + obj = hash + typ = "hash" + assert.MustNoError(enc.EncodeObject(db, key, expireat, toHash(hash))) + case 3: + zset := make(map[string]float64) + for j := 0; j < 32; j++ { + zset[strconv.Itoa(j)] = rand.Float64() + } + obj = zset + typ = "zset" + assert.MustNoError(enc.EncodeObject(db, key, expireat, toZSet(zset))) + case 4: + set := []string{} + for j := 0; j < 32; j++ { + set = append(set, fmt.Sprintf("s%d_%d", i, rand.Int())) + } + obj = set + typ = "set" + assert.MustNoError(enc.EncodeObject(db, key, expireat, toSet(set...))) + } + objs[i].db = db + objs[i].expireat = expireat + objs[i].key = key + objs[i].obj = obj + objs[i].typ = typ + } + assert.MustNoError(enc.EncodeFooter()) + rdb := b.Bytes() + var c atomic2.Int64 + l := NewLoader(stats.NewCountReader(bytes.NewReader(rdb), &c)) + assert.MustNoError(l.Header()) + var i int = 0 + for { + e, err := l.NextBinEntry() + assert.MustNoError(err) + if e == nil { + break + } + assert.Must(objs[i].db == e.DB) + assert.Must(objs[i].expireat == e.ExpireAt) + assert.Must(bytes.Equal(objs[i].key, e.Key)) + o, err := DecodeDump(e.Value) + assert.MustNoError(err) + switch objs[i].typ { + case "string": + checkString(t, o, objs[i].obj.(string)) + case "list": + checkList(t, o, objs[i].obj.([]string)) + case "hash": + checkHash(t, o, objs[i].obj.(map[string]string)) + case "zset": + checkZSet(t, o, objs[i].obj.(map[string]float64)) + case "set": + checkSet(t, o, objs[i].obj.([]string)) + } + i++ + } + assert.Must(i == len(objs)) + assert.MustNoError(l.Footer()) + assert.Must(c.Get() == int64(len(rdb))) +} diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go new file mode 100644 index 0000000..b174463 --- /dev/null +++ b/src/pkg/rdb/loader.go @@ -0,0 +1,198 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "hash" + "io" + "strconv" + + "pkg/libs/errors" + "pkg/libs/log" + "pkg/rdb/digest" +) + +type Loader struct { + *rdbReader + crc hash.Hash64 + db uint32 + lastEntry *BinEntry +} + +func NewLoader(r io.Reader) *Loader { + l := &Loader{} + l.crc = digest.New() + l.rdbReader = NewRdbReader(io.TeeReader(r, l.crc)) + return l +} + +func (l *Loader) Header() error { + header := make([]byte, 9) + if err := l.readFull(header); err != nil { + return err + } + if !bytes.Equal(header[:5], []byte("REDIS")) { + return errors.Errorf("verify magic string, invalid file format") + } + if version, err := strconv.ParseInt(string(header[5:]), 10, 64); err != nil { + return errors.Trace(err) + } else if version <= 0 || version > FromVersion { + return errors.Errorf("verify version, invalid RDB version number %d, %d", version, FromVersion) + } + return nil +} + +func (l *Loader) Footer() error { + crc1 := l.crc.Sum64() + if crc2, err := l.readUint64(); err != nil { + return err + } else if crc1 != crc2 { + return errors.Errorf("checksum validation failed") + } + return nil +} + +type BinEntry struct { + DB uint32 + Key []byte + Type byte + Value []byte + ExpireAt uint64 + RealMemberCount uint32 + NeedReadLen byte +} + +func (e *BinEntry) ObjEntry() (*ObjEntry, error) { + x, err := DecodeDump(e.Value) + if err != nil { + return nil, err + } + return &ObjEntry{ + DB: e.DB, + Key: e.Key, + Type: e.Type, + Value: x, + ExpireAt: e.ExpireAt, + RealMemberCount: e.RealMemberCount, + NeedReadLen: e.NeedReadLen, + }, nil +} + +type ObjEntry struct { + DB uint32 + Key []byte + Type byte + Value interface{} + ExpireAt uint64 + RealMemberCount uint32 + NeedReadLen byte +} + +func (e *ObjEntry) BinEntry() (*BinEntry, error) { + p, err := EncodeDump(e.Value) + if err != nil { + return nil, err + } + return &BinEntry{ + DB: e.DB, + Key: e.Key, + Type: e.Type, + Value: p, + ExpireAt: e.ExpireAt, + RealMemberCount: e.RealMemberCount, + NeedReadLen: e.NeedReadLen, + }, nil +} + +func (l *Loader) NextBinEntry() (*BinEntry, error) { + var entry = &BinEntry{} + for { + var t byte + if l.remainMember != 0 { + t = l.lastEntry.Type + } else { + rtype, err := l.ReadByte() + if err != nil { + return nil, err + } + t = rtype + } + switch t { + case rdbFlagAUX: + aux_key, _ := l.ReadString() + aux_value, _ := l.ReadString() + log.Info("Aux information key:", string(aux_key), " value:", string(aux_value)) + case rdbFlagResizeDB: + db_size, _ := l.ReadLength() + expire_size, _ := l.ReadLength() + log.Info("db_size:", db_size, " expire_size:", expire_size) + case rdbFlagExpiryMS: + ttlms, err := l.readUint64() + if err != nil { + return nil, err + } + entry.ExpireAt = ttlms + case rdbFlagExpiry: + ttls, err := l.readUint32() + if err != nil { + return nil, err + } + entry.ExpireAt = uint64(ttls) * 1000 + case rdbFlagSelectDB: + dbnum, err := l.ReadLength() + if err != nil { + return nil, err + } + l.db = dbnum + case rdbFlagEOF: + return nil, nil + case rdbFlagOnlyValue: + fallthrough + default: + var key []byte + if l.remainMember == 0 { + rkey, err := l.ReadString() + if err != nil { + return nil, err + } + key = rkey + entry.NeedReadLen = 1 + } else { + key = l.lastEntry.Key + } + // log.Infof("l %p r %p", l, l.rdbReader) + // log.Info("remainMember:", l.remainMember, " key:", string(key[:]), " type:", t) + // log.Info("r.remainMember:", l.rdbReader.remainMember) + val, err := l.readObjectValue(t, l) + if err != nil { + return nil, err + } + entry.DB = l.db + entry.Key = key + entry.Type = t + entry.Value = createValueDump(t, val) + // entry.RealMemberCount = l.lastReadCount + if l.lastReadCount == l.totMemberCount { + entry.RealMemberCount = 0 + } else { + entry.RealMemberCount = l.lastReadCount + } + l.lastEntry = entry + return entry, nil + } + } +} + +func createValueDump(t byte, val []byte) []byte { + var b bytes.Buffer + c := digest.New() + w := io.MultiWriter(&b, c) + w.Write([]byte{t}) + w.Write(val) + binary.Write(w, binary.LittleEndian, uint16(ToVersion)) + binary.Write(w, binary.LittleEndian, c.Sum64()) + return b.Bytes() +} diff --git a/src/pkg/rdb/loader_test.go b/src/pkg/rdb/loader_test.go new file mode 100644 index 0000000..95248ce --- /dev/null +++ b/src/pkg/rdb/loader_test.go @@ -0,0 +1,360 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/hex" + "fmt" + "math" + "strconv" + "strings" + "testing" + + "pkg/libs/assert" +) + +func DecodeHexRdb(t *testing.T, s string, n int) map[string]*BinEntry { + p, err := hex.DecodeString(strings.NewReplacer("\t", "", "\r", "", "\n", "", " ", "").Replace(s)) + assert.MustNoError(err) + r := bytes.NewReader(p) + l := NewLoader(r) + assert.MustNoError(l.Header()) + entries := make(map[string]*BinEntry) + var i int = 0 + for { + e, err := l.NextBinEntry() + assert.MustNoError(err) + if e == nil { + break + } + assert.Must(e.DB == 0) + entries[string(e.Key)] = e + i++ + } + assert.MustNoError(l.Footer()) + assert.Must(r.Len() == 0) + assert.Must(len(entries) == i && i == n) + return entries +} + +func getobj(t *testing.T, entries map[string]*BinEntry, key string) (*BinEntry, interface{}) { + e := entries[key] + assert.Must(e != nil) + o, err := DecodeDump(e.Value) + assert.MustNoError(err) + return e, o +} + +/* +#!/bin/bash +./redis-cli flushall +for i in 1 255 256 65535 65536 2147483647 2147483648 4294967295 4294967296 -2147483648; do + ./redis-cli set string_${i} ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadIntString(t *testing.T) { + s := ` + 524544495330303036fe00000a737472696e675f323535c1ff00000873747269 + 6e675f31c0010011737472696e675f343239343936373239360a343239343936 + 373239360011737472696e675f343239343936373239350a3432393439363732 + 39350012737472696e675f2d32313437343833363438c200000080000c737472 + 696e675f3635353335c2ffff00000011737472696e675f323134373438333634 + 380a32313437343833363438000c737472696e675f3635353336c20000010000 + 0a737472696e675f323536c100010011737472696e675f323134373438333634 + 37c2ffffff7fffe49d9f131fb5c3b5 + ` + values := []int{1, 255, 256, 65535, 65536, 2147483647, 2147483648, 4294967295, 4294967296, -2147483648} + entries := DecodeHexRdb(t, s, len(values)) + for _, value := range values { + key := fmt.Sprintf("string_%d", value) + _, obj := getobj(t, entries, key) + val := obj.(String) + assert.Must(bytes.Equal([]byte(val), []byte(strconv.Itoa(value)))) + } +} + +/* +#!/bin/bash +./redis-cli flushall +./redis-cli set string_ttls string_ttls +./redis-cli expireat string_ttls 1500000000 +./redis-cli set string_ttlms string_ttlms +./redis-cli pexpireat string_ttlms 1500000000000 +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadStringTTL(t *testing.T) { + s := ` + 524544495330303036fe00fc0098f73e5d010000000c737472696e675f74746c + 6d730c737472696e675f74746c6d73fc0098f73e5d010000000b737472696e67 + 5f74746c730b737472696e675f74746c73ffd15acd935a3fe949 + ` + expireat := uint64(1500000000000) + entries := DecodeHexRdb(t, s, 2) + keys := []string{"string_ttls", "string_ttlms"} + for _, key := range keys { + e, obj := getobj(t, entries, key) + val := obj.(String) + assert.Must(bytes.Equal([]byte(val), []byte(key))) + assert.Must(e.ExpireAt == expireat) + } +} + +/* +#!/bin/bash +s="01" +for ((i=0;i<15;i++)); do + s=$s$s +done +./redis-cli flushall +./redis-cli set string_long $s +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadLongString(t *testing.T) { + s := ` + 524544495330303036fe00000b737472696e675f6c6f6e67c342f28000010000 + 02303130e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e03201013031ffdfdb02bd6d5da5e6 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "string_long") + val := []byte(obj.(String)) + for i := 0; i < (1 << 15); i++ { + var c uint8 = '0' + if i%2 != 0 { + c = '1' + } + assert.Must(val[i] == c) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<256;i++)); do + ./redis-cli rpush list_lzf 0 + ./redis-cli rpush list_lzf 1 +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadListZipmap(t *testing.T) { + s := ` + 524544495330303036fe000a086c6973745f6c7a66c31f440b040b0400000820 + 0306000200f102f202e0ff03e1ff07e1ff07e1d90701f2ffff6a1c2d51c02301 + 16 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "list_lzf") + val := obj.(List) + assert.Must(len(val) == 512) + for i := 0; i < 256; i++ { + var s string = "0" + if i%2 != 0 { + s = "1" + } + assert.Must(string(val[i]) == s) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<32;i++)); do + ./redis-cli rpush list ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadList(t *testing.T) { + s := ` + 524544495330303036fe0001046c69737420c000c001c002c003c004c005c006 + c007c008c009c00ac00bc00cc00dc00ec00fc010c011c012c013c014c015c016 + c017c018c019c01ac01bc01cc01dc01ec01fff756ea1fa90adefe3 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "list") + val := obj.(List) + assert.Must(len(val) == 32) + for i := 0; i < 32; i++ { + assert.Must(string(val[i]) == strconv.Itoa(i)) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli sadd set1 ${i} +done +for ((i=0;i<32;i++)); do + ./redis-cli sadd set2 ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadSetAndSetIntset(t *testing.T) { + s := ` + 524544495330303036fe0002047365743220c016c00dc01bc012c01ac004c014 + c002c017c01dc01cc013c019c01ec008c006c000c001c007c00fc009c01fc00e + c003c00ac015c010c00bc018c011c00cc0050b04736574312802000000100000 + 0000000100020003000400050006000700080009000a000b000c000d000e000f + 00ff3a0a9697324d19c3 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "set1") + val1 := obj1.(Set) + set1 := make(map[string]bool) + for _, mem := range val1 { + set1[string(mem)] = true + } + assert.Must(len(set1) == 16) + assert.Must(len(set1) == len(val1)) + for i := 0; i < 16; i++ { + _, ok := set1[strconv.Itoa(i)] + assert.Must(ok) + } + + _, obj2 := getobj(t, entries, "set2") + val2 := obj2.(Set) + set2 := make(map[string]bool) + for _, mem := range val2 { + set2[string(mem)] = true + } + assert.Must(len(set2) == 32) + assert.Must(len(set2) == len(val2)) + for i := 0; i < 32; i++ { + _, ok := set2[strconv.Itoa(i)] + assert.Must(ok) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli hset hash1 ${i} +done +for ((i=-16;i<16;i++)); do + ./redis-cli hset hash2 ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadHashAndHashZiplist(t *testing.T) { + s := ` + 524544495330303036fe000405686173683220c00dc00dc0fcc0fcc0ffc0ffc0 + 04c004c002c002c0fbc0fbc0f0c0f0c0f9c0f9c008c008c0fac0fac006c006c0 + 00c000c001c001c0fec0fec007c007c0f6c0f6c00fc00fc009c009c0f7c0f7c0 + fdc0fdc0f1c0f1c0f2c0f2c0f3c0f3c00ec00ec003c003c00ac00ac00bc00bc0 + f8c0f8c00cc00cc0f5c0f5c0f4c0f4c005c0050d056861736831405151000000 + 4d000000200000f102f102f202f202f302f302f402f402f502f502f602f602f7 + 02f702f802f802f902f902fa02fa02fb02fb02fc02fc02fd02fd02fe0d03fe0d + 03fe0e03fe0e03fe0f03fe0fffffa423d3036c15e534 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "hash1") + val1 := obj1.(Hash) + hash1 := make(map[string]string) + for _, ent := range val1 { + hash1[string(ent.Field)] = string(ent.Value) + } + assert.Must(len(hash1) == 16) + assert.Must(len(hash1) == len(val1)) + for i := 0; i < 16; i++ { + s := strconv.Itoa(i) + assert.Must(hash1[s] == s) + } + + _, obj2 := getobj(t, entries, "hash2") + val2 := obj2.(Hash) + hash2 := make(map[string]string) + for _, ent := range val2 { + hash2[string(ent.Field)] = string(ent.Value) + } + assert.Must(len(hash2) == 32) + assert.Must(len(hash2) == len(val2)) + for i := -16; i < 16; i++ { + s := strconv.Itoa(i) + assert.Must(hash2[s] == s) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli zadd zset1 ${i} ${i} +done +for ((i=0;i<32;i++)); do + ./redis-cli zadd zset2 -${i} ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadZSetAndZSetZiplist(t *testing.T) { + s := ` + 524544495330303036fe0003057a7365743220c016032d3232c00d032d3133c0 + 1b032d3237c012032d3138c01a032d3236c004022d34c014032d3230c002022d + 32c017032d3233c01d032d3239c01c032d3238c013032d3139c019032d3235c0 + 1e032d3330c008022d38c006022d36c000022d30c001022d31c007022d37c009 + 022d39c00f032d3135c01f032d3331c00e032d3134c003022d33c00a032d3130 + c015032d3231c010032d3136c00b032d3131c018032d3234c011032d3137c00c + 032d3132c005022d350c057a736574314051510000004d000000200000f102f1 + 02f202f202f302f302f402f402f502f502f602f602f702f702f802f802f902f9 + 02fa02fa02fb02fb02fc02fc02fd02fd02fe0d03fe0d03fe0e03fe0e03fe0f03 + fe0fffff2addedbf4f5a8f93 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "zset1") + val1 := obj1.(ZSet) + zset1 := make(map[string]float64) + for _, ent := range val1 { + zset1[string(ent.Member)] = ent.Score + } + assert.Must(len(zset1) == 16) + assert.Must(len(zset1) == len(val1)) + for i := 0; i < 16; i++ { + s := strconv.Itoa(i) + score, ok := zset1[s] + assert.Must(ok) + assert.Must(math.Abs(score-float64(i)) < 1e-10) + } + + _, obj2 := getobj(t, entries, "zset2") + val2 := obj2.(ZSet) + zset2 := make(map[string]float64) + for _, ent := range val2 { + zset2[string(ent.Member)] = ent.Score + } + assert.Must(len(zset2) == 32) + assert.Must(len(zset2) == len(val2)) + for i := 0; i < 32; i++ { + s := strconv.Itoa(i) + score, ok := zset2[s] + assert.Must(ok) + assert.Must(math.Abs(score+float64(i)) < 1e-10) + } +} diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go new file mode 100644 index 0000000..43cff17 --- /dev/null +++ b/src/pkg/rdb/reader.go @@ -0,0 +1,521 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + // "runtime/debug" + "strconv" + + "pkg/libs/errors" + // "libs/log" +) + +var FromVersion int64 = 8 +var ToVersion int64 = 6 + +const ( + RdbTypeString = 0 + RdbTypeList = 1 + RdbTypeSet = 2 + RdbTypeZSet = 3 + RdbTypeHash = 4 + RdbTypeZSet2 = 5 + + RdbTypeHashZipmap = 9 + RdbTypeListZiplist = 10 + RdbTypeSetIntset = 11 + RdbTypeZSetZiplist = 12 + RdbTypeHashZiplist = 13 + RdbTypeQuicklist = 14 + + rdbFlagOnlyValue = 0xf9 + rdbFlagAUX = 0xfa + rdbFlagResizeDB = 0xfb + rdbFlagExpiryMS = 0xfc + rdbFlagExpiry = 0xfd + rdbFlagSelectDB = 0xfe + rdbFlagEOF = 0xff +) + +const ( + rdb6bitLen = 0 + rdb14bitLen = 1 + rdb32bitLen = 2 + rdbEncVal = 3 + + rdbEncInt8 = 0 + rdbEncInt16 = 1 + rdbEncInt32 = 2 + rdbEncLZF = 3 + + rdbZiplist6bitlenString = 0 + rdbZiplist14bitlenString = 1 + rdbZiplist32bitlenString = 2 + + rdbZiplistInt16 = 0xc0 + rdbZiplistInt32 = 0xd0 + rdbZiplistInt64 = 0xe0 + rdbZiplistInt24 = 0xf0 + rdbZiplistInt8 = 0xfe + rdbZiplistInt4 = 15 +) + +type rdbReader struct { + raw io.Reader + buf [8]byte + nread int64 + remainMember uint32 + lastReadCount uint32 + totMemberCount uint32 +} + +func NewRdbReader(r io.Reader) *rdbReader { + return &rdbReader{raw: r, remainMember: 0, lastReadCount: 0} +} + +func (r *rdbReader) Read(p []byte) (int, error) { + n, err := r.raw.Read(p) + r.nread += int64(n) + return n, errors.Trace(err) +} + +func (r *rdbReader) offset() int64 { + return r.nread +} + +func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { + var b bytes.Buffer + r = NewRdbReader(io.TeeReader(r, &b)) + lr := l.rdbReader + switch t { + default: + return nil, errors.Errorf("unknown object-type %02x", t) + case rdbFlagAUX: + fallthrough + case rdbFlagResizeDB: + fallthrough + case RdbTypeHashZipmap: + fallthrough + case RdbTypeListZiplist: + fallthrough + case RdbTypeSetIntset: + fallthrough + case RdbTypeZSetZiplist: + fallthrough + case RdbTypeHashZiplist: + fallthrough + case RdbTypeString: + lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0 + _, err := r.ReadString() + if err != nil { + return nil, err + } + case RdbTypeList, RdbTypeSet, RdbTypeQuicklist: + lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0 + if n, err := r.ReadLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.ReadString(); err != nil { + return nil, err + } + } + } + case RdbTypeZSet, RdbTypeZSet2: + lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0 + if n, err := r.ReadLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.ReadString(); err != nil { + return nil, err + } + if t == RdbTypeZSet2 { + if _, err := r.ReadDouble(); err != nil { + return nil, err + } + } else { + if _, err := r.ReadFloat(); err != nil { + return nil, err + } + } + } + } + case RdbTypeHash: + var n uint32 + if lr.remainMember != 0 { + n = lr.remainMember + } else { + rlen, err := r.ReadLength() + if err != nil { + return nil, err + } else { + n = rlen + lr.totMemberCount = rlen + } + } + lr.lastReadCount = 0 + for i := 0; i < int(n); i++ { + // read twice for hash field and value + if _, err := r.ReadString(); err != nil { + return nil, err + } + if _, err := r.ReadString(); err != nil { + return nil, err + } + lr.lastReadCount++ + if b.Len() > 16*1024*1024 && i != int(n-1) { + lr.remainMember = n - uint32(i) - 1 + // log.Infof("r %p", lr) + // log.Info("r: ", lr, " set remainMember:", lr.remainMember) + // debug.FreeOSMemory() + break + } + } + if lr.lastReadCount == n { + lr.remainMember = 0 + } + } + return b.Bytes(), nil +} + +func (r *rdbReader) ReadString() ([]byte, error) { + length, encoded, err := r.readEncodedLength() + if err != nil { + return nil, err + } + if !encoded { + return r.ReadBytes(int(length)) + } + switch t := uint8(length); t { + default: + return nil, errors.Errorf("invalid encoded-string %02x", t) + case rdbEncInt8: + i, err := r.readInt8() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt16: + i, err := r.readInt16() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt32: + i, err := r.readInt32() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncLZF: + var inlen, outlen uint32 + if inlen, err = r.ReadLength(); err != nil { + return nil, err + } + if outlen, err = r.ReadLength(); err != nil { + return nil, err + } + if in, err := r.ReadBytes(int(inlen)); err != nil { + return nil, err + } else { + return lzfDecompress(in, int(outlen)) + } + } +} + +func (r *rdbReader) readEncodedLength() (length uint32, encoded bool, err error) { + u, err := r.readUint8() + if err != nil { + return + } + length = uint32(u & 0x3f) + switch u >> 6 { + case rdb6bitLen: + case rdb14bitLen: + u, err = r.readUint8() + length = (length << 8) + uint32(u) + case rdbEncVal: + encoded = true + default: + length, err = r.readUint32BigEndian() + } + return +} + +func (r *rdbReader) ReadLength() (uint32, error) { + length, encoded, err := r.readEncodedLength() + if err == nil && encoded { + err = errors.Errorf("encoded-length") + } + return length, err +} + +func (r *rdbReader) ReadDouble() (float64, error) { + var buf = make([]byte, 8) + err := r.readFull(buf) + if err != nil { + return 0, err + } + bits := binary.LittleEndian.Uint64(buf) + return float64(math.Float64frombits(bits)), nil +} + +func (r *rdbReader) ReadFloat() (float64, error) { + u, err := r.readUint8() + if err != nil { + return 0, err + } + switch u { + case 253: + return math.NaN(), nil + case 254: + return math.Inf(0), nil + case 255: + return math.Inf(-1), nil + default: + if b, err := r.ReadBytes(int(u)); err != nil { + return 0, err + } else { + v, err := strconv.ParseFloat(string(b), 64) + return v, errors.Trace(err) + } + } +} + +func (r *rdbReader) ReadByte() (byte, error) { + b := r.buf[:1] + _, err := io.ReadFull(r, b) + return b[0], errors.Trace(err) +} + +func (r *rdbReader) readFull(p []byte) error { + _, err := io.ReadFull(r, p) + return errors.Trace(err) +} + +func (r *rdbReader) ReadBytes(n int) ([]byte, error) { + p := make([]byte, n) + return p, r.readFull(p) +} + +func (r *rdbReader) readUint8() (uint8, error) { + b, err := r.ReadByte() + return uint8(b), err +} + +func (r *rdbReader) readUint16() (uint16, error) { + b := r.buf[:2] + err := r.readFull(b) + return binary.LittleEndian.Uint16(b), err +} + +func (r *rdbReader) readUint32() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.LittleEndian.Uint32(b), err +} + +func (r *rdbReader) readUint64() (uint64, error) { + b := r.buf[:8] + err := r.readFull(b) + return binary.LittleEndian.Uint64(b), err +} + +func (r *rdbReader) readUint32BigEndian() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.BigEndian.Uint32(b), err +} + +func (r *rdbReader) readInt8() (int8, error) { + u, err := r.readUint8() + return int8(u), err +} + +func (r *rdbReader) readInt16() (int16, error) { + u, err := r.readUint16() + return int16(u), err +} + +func (r *rdbReader) readInt32() (int32, error) { + u, err := r.readUint32() + return int32(u), err +} + +func (r *rdbReader) readInt64() (int64, error) { + u, err := r.readUint64() + return int64(u), err +} + +func (r *rdbReader) readInt32BigEndian() (int32, error) { + u, err := r.readUint32BigEndian() + return int32(u), err +} + +func lzfDecompress(in []byte, outlen int) (out []byte, err error) { + defer func() { + if x := recover(); x != nil { + err = errors.Errorf("decompress exception: %v", x) + } + }() + out = make([]byte, outlen) + i, o := 0, 0 + for i < len(in) { + ctrl := int(in[i]) + i++ + if ctrl < 32 { + for x := 0; x <= ctrl; x++ { + out[o] = in[i] + i++ + o++ + } + } else { + length := ctrl >> 5 + if length == 7 { + length = length + int(in[i]) + i++ + } + ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 + i++ + for x := 0; x <= length+1; x++ { + out[o] = out[ref] + ref++ + o++ + } + } + } + if o != outlen { + return nil, errors.Errorf("decompress length is %d != expected %d", o, outlen) + } + return out, nil +} + +func (r *rdbReader) ReadZiplistLength(buf *sliceBuffer) (int64, error) { + buf.Seek(8, 0) // skip the zlbytes and zltail + lenBytes, err := buf.Slice(2) + if err != nil { + return 0, err + } + return int64(binary.LittleEndian.Uint16(lenBytes)), nil +} + +func (r *rdbReader) ReadZiplistEntry(buf *sliceBuffer) ([]byte, error) { + prevLen, err := buf.ReadByte() + if err != nil { + return nil, err + } + if prevLen == 254 { + buf.Seek(4, 1) // skip the 4-byte prevlen + } + + header, err := buf.ReadByte() + if err != nil { + return nil, err + } + switch { + case header>>6 == rdbZiplist6bitlenString: + return buf.Slice(int(header & 0x3f)) + case header>>6 == rdbZiplist14bitlenString: + b, err := buf.ReadByte() + if err != nil { + return nil, err + } + return buf.Slice((int(header&0x3f) << 8) | int(b)) + case header>>6 == rdbZiplist32bitlenString: + lenBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return buf.Slice(int(binary.BigEndian.Uint32(lenBytes))) + case header == rdbZiplistInt16: + intBytes, err := buf.Slice(2) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10)), nil + case header == rdbZiplistInt32: + intBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10)), nil + case header == rdbZiplistInt64: + intBytes, err := buf.Slice(8) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(intBytes)), 10)), nil + case header == rdbZiplistInt24: + intBytes := make([]byte, 4) + _, err := buf.Read(intBytes[1:]) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))>>8), 10)), nil + case header == rdbZiplistInt8: + b, err := buf.ReadByte() + return []byte(strconv.FormatInt(int64(int8(b)), 10)), err + case header>>4 == rdbZiplistInt4: + return []byte(strconv.FormatInt(int64(header&0x0f)-1, 10)), nil + } + + return nil, fmt.Errorf("rdb: unknown ziplist header byte: %d", header) +} + +func (r *rdbReader) ReadZipmapItem(buf *sliceBuffer, readFree bool) ([]byte, error) { + length, free, err := readZipmapItemLength(buf, readFree) + if err != nil { + return nil, err + } + if length == -1 { + return nil, nil + } + value, err := buf.Slice(length) + if err != nil { + return nil, err + } + _, err = buf.Seek(int64(free), 1) + return value, err +} + +func readZipmapItemLength(buf *sliceBuffer, readFree bool) (int, int, error) { + b, err := buf.ReadByte() + if err != nil { + return 0, 0, err + } + switch b { + case 253: + s, err := buf.Slice(5) + if err != nil { + return 0, 0, err + } + return int(binary.BigEndian.Uint32(s)), int(s[4]), nil + case 254: + return 0, 0, fmt.Errorf("rdb: invalid zipmap item length") + case 255: + return -1, 0, nil + } + var free byte + if readFree { + free, err = buf.ReadByte() + } + return int(b), int(free), err +} + +func (r *rdbReader) CountZipmapItems(buf *sliceBuffer) (int, error) { + n := 0 + for { + strLen, free, err := readZipmapItemLength(buf, n%2 != 0) + if err != nil { + return 0, err + } + if strLen == -1 { + break + } + _, err = buf.Seek(int64(strLen)+int64(free), 1) + if err != nil { + return 0, err + } + n++ + } + _, err := buf.Seek(0, 0) + return n, err +} diff --git a/src/pkg/rdb/slice_buffer.go b/src/pkg/rdb/slice_buffer.go new file mode 100644 index 0000000..28cc45d --- /dev/null +++ b/src/pkg/rdb/slice_buffer.go @@ -0,0 +1,67 @@ +package rdb + +import ( + "errors" + "io" +) + +type sliceBuffer struct { + s []byte + i int +} + +func NewSliceBuffer(s []byte) *sliceBuffer { + return &sliceBuffer{s, 0} +} + +func (s *sliceBuffer) Slice(n int) ([]byte, error) { + if s.i+n > len(s.s) { + return nil, io.EOF + } + b := s.s[s.i : s.i+n] + s.i += n + return b, nil +} + +func (s *sliceBuffer) ReadByte() (byte, error) { + if s.i >= len(s.s) { + return 0, io.EOF + } + b := s.s[s.i] + s.i++ + return b, nil +} + +func (s *sliceBuffer) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if s.i >= len(s.s) { + return 0, io.EOF + } + n := copy(b, s.s[s.i:]) + s.i += n + return n, nil +} + +func (s *sliceBuffer) Seek(offset int64, whence int) (int64, error) { + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(s.i) + offset + case 2: + abs = int64(len(s.s)) + offset + default: + return 0, errors.New("invalid whence") + } + if abs < 0 { + return 0, errors.New("negative position") + } + if abs >= 1<<31 { + return 0, errors.New("position out of range") + } + s.i = int(abs) + return abs, nil +} diff --git a/src/pkg/redis/decoder.go b/src/pkg/redis/decoder.go new file mode 100644 index 0000000..40b82b8 --- /dev/null +++ b/src/pkg/redis/decoder.go @@ -0,0 +1,192 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "bufio" + "bytes" + "io" + "strconv" + + "pkg/libs/errors" + "pkg/libs/log" +) + +var ( + ErrBadRespCRLFEnd = errors.New("bad resp CRLF end") + ErrBadRespBytesLen = errors.New("bad resp bytes len") + ErrBadRespArrayLen = errors.New("bad resp array len") +) + +type Decoder struct { + r *bufio.Reader +} + +func NewDecoder(r *bufio.Reader) *Decoder { + return &Decoder{r: r} +} + +func Decode(r *bufio.Reader) (Resp, error) { + d := &Decoder{r} + return d.decodeResp(0) +} + +func MustDecodeOpt(d *Decoder) Resp { + resp, err := d.decodeResp(0) + if err != nil { + log.PanicError(err, "decode redis resp failed") + } + return resp +} + +func MustDecode(r *bufio.Reader) Resp { + resp, err := Decode(r) + if err != nil { + log.PanicError(err, "decode redis resp failed") + } + return resp +} + +func DecodeFromBytes(p []byte) (Resp, error) { + r := bufio.NewReader(bytes.NewReader(p)) + return Decode(r) +} + +func MustDecodeFromBytes(p []byte) Resp { + resp, err := DecodeFromBytes(p) + if err != nil { + log.PanicError(err, "decode redis resp from bytes failed") + } + return resp +} + +func (d *Decoder) decodeResp(depth int) (Resp, error) { + t, err := d.decodeType() + if err != nil { + return nil, err + } + switch t { + case typeString: + resp := &String{} + resp.Value, err = d.decodeText() + return resp, err + case typeError: + resp := &Error{} + resp.Value, err = d.decodeText() + return resp, err + case typeInt: + resp := &Int{} + resp.Value, err = d.decodeInt() + return resp, err + case typeBulkBytes: + resp := &BulkBytes{} + resp.Value, err = d.decodeBulkBytes() + return resp, err + case typeArray: + resp := &Array{} + resp.Value, err = d.decodeArray(depth) + return resp, err + default: + if depth != 0 { + return nil, errors.Errorf("bad resp type %s", t) + } + if err = d.r.UnreadByte(); err != nil { + return nil, errors.Trace(err) + } + return d.decodeSingleLineBulkBytesArray() + } +} + +func (d *Decoder) decodeType() (respType, error) { + if b, err := d.r.ReadByte(); err != nil { + return 0, errors.Trace(err) + } else { + return respType(b), nil + } +} + +func (d *Decoder) decodeText() ([]byte, error) { + b, err := d.r.ReadBytes('\n') + if err != nil { + return make([]byte, 0, 0), errors.Trace(err) + } + if n := len(b) - 2; n < 0 || b[n] != '\r' { + return make([]byte, 0, 0), errors.Trace(ErrBadRespCRLFEnd) + } else { + //return string(b[:n]), nil + return b[:n], nil + } +} + +func (d *Decoder) decodeInt() (int64, error) { + b, err := d.decodeText() + if err != nil { + return 0, err + } + if n, err := strconv.ParseInt(string(b), 10, 64); err != nil { + return 0, errors.Trace(err) + } else { + return n, nil + } +} + +func (d *Decoder) decodeBulkBytes() ([]byte, error) { + n, err := d.decodeInt() + if err != nil { + return nil, err + } + if n < -1 { + return nil, errors.Trace(ErrBadRespBytesLen) + } else if n == -1 { + return nil, nil + } + b := make([]byte, n+2) + if _, err := io.ReadFull(d.r, b); err != nil { + return nil, errors.Trace(err) + } + if b[n] != '\r' || b[n+1] != '\n' { + return nil, errors.Trace(ErrBadRespCRLFEnd) + } + return b[:n], nil +} + +func (d *Decoder) decodeArray(depth int) ([]Resp, error) { + n, err := d.decodeInt() + if err != nil { + return nil, err + } + if n < -1 { + return nil, errors.Trace(ErrBadRespArrayLen) + } else if n == -1 { + return nil, nil + } + a := make([]Resp, n) + for i := 0; i < len(a); i++ { + if a[i], err = d.decodeResp(depth + 1); err != nil { + return nil, err + } + } + return a, nil +} + +func (d *Decoder) decodeSingleLineBulkBytesArray() (Resp, error) { + b, err := d.r.ReadBytes('\n') + if err != nil { + return nil, errors.Trace(err) + } + if n := len(b) - 2; n < 0 || b[n] != '\r' { + return nil, errors.Trace(ErrBadRespCRLFEnd) + } else { + resp := &Array{} + for l, r := 0, 0; r <= n; r++ { + if r == n || b[r] == ' ' { + if l < r { + resp.Value = append(resp.Value, &BulkBytes{b[l:r]}) + } + l = r + 1 + } + } + return resp, nil + } +} diff --git a/src/pkg/redis/decoder_test.go b/src/pkg/redis/decoder_test.go new file mode 100644 index 0000000..914e14a --- /dev/null +++ b/src/pkg/redis/decoder_test.go @@ -0,0 +1,112 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "bytes" + "testing" + + "pkg/libs/assert" +) + +func TestDecodeInvalidRequests(t *testing.T) { + test := []string{ + "*hello\r\n", + "*-100\r\n", + "*3\r\nhi", + "*3\r\nhi\r\n", + "*4\r\n$1", + "*4\r\n$1\r", + "*4\r\n$1\n", + "*2\r\n$3\r\nget\r\n$what?\r\nx\r\n", + "*4\r\n$3\r\nget\r\n$1\r\nx\r\n", + "*2\r\n$3\r\nget\r\n$1\r\nx", + "*2\r\n$3\r\nget\r\n$1\r\nx\r", + "*2\r\n$3\r\nget\r\n$100\r\nx\r\n", + "$6\r\nfoobar\r", + "$0\rn\r\n", + "$-1\n", + "*0", + "*2n$3\r\nfoo\r\n$3\r\nbar\r\n", + "*-\r\n", + "+OK\n", + "-Error message\r", + } + for _, s := range test { + _, err := DecodeFromBytes([]byte(s)) + assert.Must(err != nil) + } +} + +func TestDecodeSimpleRequest1(t *testing.T) { + resp, err := DecodeFromBytes([]byte("\r\n")) + assert.MustNoError(err) + x, ok := resp.(*Array) + assert.Must(ok) + assert.Must(len(x.Value) == 0) +} + +func TestDecodeSimpleRequest2(t *testing.T) { + test := []string{ + "hello world\r\n", + "hello world \r\n", + " hello world \r\n", + " hello world\r\n", + " hello world \r\n", + } + for _, s := range test { + resp, err := DecodeFromBytes([]byte(s)) + assert.MustNoError(err) + x, ok := resp.(*Array) + assert.Must(ok) + assert.Must(len(x.Value) == 2) + s1, ok := x.Value[0].(*BulkBytes) + assert.Must(ok && bytes.Equal(s1.Value, []byte("hello"))) + s2, ok := x.Value[1].(*BulkBytes) + assert.Must(ok && bytes.Equal(s2.Value, []byte("world"))) + } +} + +func TestDecodeSimpleRequest3(t *testing.T) { + test := []string{"\r", "\n", " \n"} + for _, s := range test { + _, err := DecodeFromBytes([]byte(s)) + assert.Must(err != nil) + } +} + +func TestDecodeBulkBytes(t *testing.T) { + test := "*2\r\n$4\r\nLLEN\r\n$6\r\nmylist\r\n" + resp, err := DecodeFromBytes([]byte(test)) + assert.MustNoError(err) + x, ok := resp.(*Array) + assert.Must(ok) + assert.Must(len(x.Value) == 2) + s1, ok := x.Value[0].(*BulkBytes) + assert.Must(ok) + assert.Must(bytes.Equal(s1.Value, []byte("LLEN"))) + s2, ok := x.Value[1].(*BulkBytes) + assert.Must(ok) + assert.Must(bytes.Equal(s2.Value, []byte("mylist"))) +} + +func TestDecoder(t *testing.T) { + test := []string{ + "$6\r\nfoobar\r\n", + "$0\r\n\r\n", + "$-1\r\n", + "*0\r\n", + "*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n", + "*3\r\n:1\r\n:2\r\n:3\r\n", + "*-1\r\n", + "+OK\r\n", + "-Error message\r\n", + "*2\r\n$1\r\n0\r\n*0\r\n", + "*3\r\n$4\r\nEVAL\r\n$31\r\nreturn {1,2,{3,'Hello World!'}}\r\n$1\r\n0\r\n", + } + for _, s := range test { + _, err := DecodeFromBytes([]byte(s)) + assert.MustNoError(err) + } +} diff --git a/src/pkg/redis/encoder.go b/src/pkg/redis/encoder.go new file mode 100644 index 0000000..5d470a4 --- /dev/null +++ b/src/pkg/redis/encoder.go @@ -0,0 +1,167 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "bufio" + "bytes" + "reflect" + "strconv" + + "pkg/libs/errors" + "pkg/libs/log" +) + +type encoder struct { + w *bufio.Writer +} + +var ( + imap []string +) + +func init() { + imap = make([]string, 1024*512+1024) + for i := 0; i < len(imap); i++ { + imap[i] = strconv.Itoa(i - 1024) + } +} + +func itos(i int64) string { + if n := i + 1024; n >= 0 && n < int64(len(imap)) { + return imap[n] + } else { + return strconv.FormatInt(i, 10) + } +} + +func Encode(w *bufio.Writer, r Resp, flush bool) error { + e := &encoder{w} + if err := e.encodeResp(r); err != nil { + return err + } + if !flush { + return nil + } + return errors.Trace(w.Flush()) +} + +func MustEncode(w *bufio.Writer, r Resp) { + if err := Encode(w, r, true); err != nil { + log.PanicError(err, "encode redis resp failed") + } +} + +func EncodeToBytes(r Resp) ([]byte, error) { + var b bytes.Buffer + err := Encode(bufio.NewWriter(&b), r, true) + return b.Bytes(), err +} + +func EncodeToString(r Resp) (string, error) { + var b bytes.Buffer + err := Encode(bufio.NewWriter(&b), r, true) + return b.String(), err +} + +func MustEncodeToBytes(r Resp) []byte { + b, err := EncodeToBytes(r) + if err != nil { + log.PanicError(err, "encode redis resp to bytes failed") + } + return b +} + +func (e *encoder) encodeResp(r Resp) error { + switch x := r.(type) { + default: + return errors.Errorf("bad resp type <%s>", reflect.TypeOf(r)) + case *String: + if err := e.encodeType(typeString); err != nil { + return err + } + return e.encodeText(x.Value) + case *Error: + if err := e.encodeType(typeError); err != nil { + return err + } + return e.encodeText(x.Value) + case *Int: + if err := e.encodeType(typeInt); err != nil { + return err + } + return e.encodeInt(x.Value) + case *BulkBytes: + if err := e.encodeType(typeBulkBytes); err != nil { + return err + } + return e.encodeBulkBytes(x.Value) + case *Array: + if err := e.encodeType(typeArray); err != nil { + return err + } + return e.encodeArray(x.Value) + } +} + +func (e *encoder) encodeType(t respType) error { + return errors.Trace(e.w.WriteByte(byte(t))) +} + +func (e *encoder) encodeString(s string) error { + if _, err := e.w.WriteString(s); err != nil { + return errors.Trace(err) + } + if _, err := e.w.WriteString("\r\n"); err != nil { + return errors.Trace(err) + } + return nil +} + +func (e *encoder) encodeText(s []byte) error { + if _, err := e.w.Write(s); err != nil { + return errors.Trace(err) + } + if _, err := e.w.WriteString("\r\n"); err != nil { + return errors.Trace(err) + } + return nil +} + +func (e *encoder) encodeInt(v int64) error { + return e.encodeString(itos(v)) +} + +func (e *encoder) encodeBulkBytes(b []byte) error { + if b == nil { + return e.encodeInt(-1) + } else { + if err := e.encodeInt(int64(len(b))); err != nil { + return err + } + if _, err := e.w.Write(b); err != nil { + return errors.Trace(err) + } + if _, err := e.w.WriteString("\r\n"); err != nil { + return errors.Trace(err) + } + return nil + } +} + +func (e *encoder) encodeArray(a []Resp) error { + if a == nil { + return e.encodeInt(-1) + } else { + if err := e.encodeInt(int64(len(a))); err != nil { + return err + } + for i := 0; i < len(a); i++ { + if err := e.encodeResp(a[i]); err != nil { + return err + } + } + return nil + } +} diff --git a/src/pkg/redis/encoder_test.go b/src/pkg/redis/encoder_test.go new file mode 100644 index 0000000..ae06e49 --- /dev/null +++ b/src/pkg/redis/encoder_test.go @@ -0,0 +1,68 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "bytes" + "strconv" + "testing" + + "pkg/libs/assert" +) + +func TestItos(t *testing.T) { + for i := 0; i < len(imap)*2; i++ { + n, p := -i, i + assert.Must(strconv.Itoa(n) == itos(int64(n))) + assert.Must(strconv.Itoa(p) == itos(int64(p))) + } +} + +func TestEncodeString(t *testing.T) { + resp := &String{"OK"} + testEncodeAndCheck(t, resp, []byte("+OK\r\n")) +} + +func TestEncodeError(t *testing.T) { + resp := &Error{"Error"} + testEncodeAndCheck(t, resp, []byte("-Error\r\n")) +} + +func TestEncodeInt(t *testing.T) { + resp := &Int{} + for _, v := range []int{-1, 0, 1024 * 1024} { + resp.Value = int64(v) + testEncodeAndCheck(t, resp, []byte(":"+strconv.FormatInt(int64(v), 10)+"\r\n")) + } +} + +func TestEncodeBulkBytes(t *testing.T) { + resp := &BulkBytes{} + resp.Value = nil + testEncodeAndCheck(t, resp, []byte("$-1\r\n")) + resp.Value = []byte{} + testEncodeAndCheck(t, resp, []byte("$0\r\n\r\n")) + resp.Value = []byte("helloworld!!") + testEncodeAndCheck(t, resp, []byte("$12\r\nhelloworld!!\r\n")) +} + +func TestEncodeArray(t *testing.T) { + resp := &Array{} + resp.Value = nil + testEncodeAndCheck(t, resp, []byte("*-1\r\n")) + resp.Value = []Resp{} + testEncodeAndCheck(t, resp, []byte("*0\r\n")) + resp.Append(&Int{0}) + testEncodeAndCheck(t, resp, []byte("*1\r\n:0\r\n")) + resp.Append(&BulkBytes{nil}) + testEncodeAndCheck(t, resp, []byte("*2\r\n:0\r\n$-1\r\n")) + resp.Append(&BulkBytes{[]byte("test")}) + testEncodeAndCheck(t, resp, []byte("*3\r\n:0\r\n$-1\r\n$4\r\ntest\r\n")) +} + +func testEncodeAndCheck(t *testing.T, resp Resp, expect []byte) { + b, err := EncodeToBytes(resp) + assert.MustNoError(err) + assert.Must(bytes.Equal(b, expect)) +} diff --git a/src/pkg/redis/handler.go b/src/pkg/redis/handler.go new file mode 100644 index 0000000..43f521d --- /dev/null +++ b/src/pkg/redis/handler.go @@ -0,0 +1,117 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "reflect" + "strings" + + "pkg/libs/errors" + "pkg/libs/log" +) + +type HandlerFunc func(arg0 interface{}, args ...[]byte) (Resp, error) + +type HandlerTable map[string]HandlerFunc + +func NewHandlerTable(o interface{}) (map[string]HandlerFunc, error) { + if o == nil { + return nil, errors.Errorf("handler is nil") + } + t := make(map[string]HandlerFunc) + r := reflect.TypeOf(o) + for i := 0; i < r.NumMethod(); i++ { + m := r.Method(i) + if m.Name[0] < 'A' || m.Name[0] > 'Z' { + continue + } + n := strings.ToLower(m.Name) + if h, err := createHandlerFunc(o, &m.Func); err != nil { + return nil, err + } else if _, exists := t[n]; exists { + return nil, errors.Errorf("func.name = '%s' has already exists", m.Name) + } else { + t[n] = h + } + } + return t, nil +} + +func MustHandlerTable(o interface{}) map[string]HandlerFunc { + t, err := NewHandlerTable(o) + if err != nil { + log.PanicError(err, "create redis handler map failed") + } + return t +} + +func createHandlerFunc(o interface{}, f *reflect.Value) (HandlerFunc, error) { + t := f.Type() + arg0Type := reflect.TypeOf((*interface{})(nil)).Elem() + argsType := reflect.TypeOf([][]byte{}) + if t.NumIn() != 3 || t.In(1) != arg0Type || t.In(2) != argsType { + return nil, errors.Errorf("register with invalid func type = '%s'", t) + } + ret0Type := reflect.TypeOf((*Resp)(nil)).Elem() + ret1Type := reflect.TypeOf((*error)(nil)).Elem() + if t.NumOut() != 2 || t.Out(0) != ret0Type || t.Out(1) != ret1Type { + return nil, errors.Errorf("register with invalid func type = '%s'", t) + } + return func(arg0 interface{}, args ...[]byte) (Resp, error) { + var arg0Value reflect.Value + if arg0 == nil { + arg0Value = reflect.ValueOf((*interface{})(nil)) + } else { + arg0Value = reflect.ValueOf(arg0) + } + var input, output []reflect.Value + input = []reflect.Value{reflect.ValueOf(o), arg0Value, reflect.ValueOf(args)} + if t.IsVariadic() { + output = f.CallSlice(input) + } else { + output = f.Call(input) + } + var ret0 Resp + var ret1 error + if i := output[0].Interface(); i != nil { + ret0 = i.(Resp) + } + if i := output[1].Interface(); i != nil { + ret1 = i.(error) + } + return ret0, ret1 + }, nil +} + +func ParseArgs(resp Resp) (cmd string, args [][]byte, err error) { + a, err := AsArray(resp, nil) + if err != nil { + return "", nil, err + } else if len(a) == 0 { + return "", nil, errors.Errorf("empty array") + } + bs := make([][]byte, len(a)) + for i := 0; i < len(a); i++ { + b, err := AsBulkBytes(a[i], nil) + if err != nil { + return "", nil, err + } else { + bs[i] = b + } + } + cmd = strings.ToLower(string(bs[0])) + if cmd == "" { + return "", nil, errors.Errorf("empty command") + } + return cmd, bs[1:], nil +} + +func ChangeArgsToResp(cmd []byte, args [][]byte) (resp Resp) { + array := make([]Resp, len(args)+1) + array[0] = &BulkBytes{cmd} + for i := 0; i < len(args); i++ { + array[i+1] = &BulkBytes{args[i]} + } + return &Array{array} +} diff --git a/src/pkg/redis/resp.go b/src/pkg/redis/resp.go new file mode 100644 index 0000000..ef87375 --- /dev/null +++ b/src/pkg/redis/resp.go @@ -0,0 +1,185 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "fmt" + "reflect" + + "pkg/libs/errors" +) + +type respType byte + +const ( + typeString respType = '+' + typeError respType = '-' + typeInt respType = ':' + typeBulkBytes respType = '$' + typeArray respType = '*' +) + +func (t respType) String() string { + switch t { + case typeString: + return "" + case typeError: + return "" + case typeInt: + return "" + case typeBulkBytes: + return "" + case typeArray: + return "" + default: + if c := uint8(t); c > 0x20 && c < 0x7F { + return fmt.Sprintf("", c) + } else { + return fmt.Sprintf("", c) + } + } +} + +type Resp interface { +} + +type String struct { + //Value string + Value []byte +} + +//func NewString(s string) *String { +// return &String{s} +//} + +type Error struct { + //Value string + Value []byte +} + +//func NewError(err error) *Error { +// return &Error{err.Error()} +//} + +type Int struct { + Value int64 +} + +func NewInt(n int64) *Int { + return &Int{n} +} + +type BulkBytes struct { + Value []byte +} + +func NewBulkBytes(p []byte) *BulkBytes { + return &BulkBytes{p} +} + +type Array struct { + Value []Resp +} + +func NewArray() *Array { + return &Array{} +} + +func (r *Array) Append(a Resp) { + r.Value = append(r.Value, a) +} + +//func (r *Array) AppendString(s string) { +// r.Append(NewString(s)) +//} + +func (r *Array) AppendBulkBytes(b []byte) { + r.Append(NewBulkBytes(b)) +} + +func (r *Array) AppendInt(n int64) { + r.Append(NewInt(n)) +} + +//func (r *Array) AppendError(err error) { +// r.Append(NewError(err)) +//} + +func AsString(r Resp, err error) ([]byte, error) { + if err != nil { + return make([]byte, 0, 0), err + } + x, ok := r.(*String) + if ok && x != nil { + return x.Value, nil + } else { + return make([]byte, 0, 0), errors.Errorf("expect String, but got <%s>", reflect.TypeOf(r)) + } +} + +func AsError(r Resp, err error) ([]byte, error) { + if err != nil { + return make([]byte, 0, 0), err + } + x, ok := r.(*Error) + if ok && x != nil { + return x.Value, nil + } else { + return make([]byte, 0, 0), errors.Errorf("expect Error, but got <%s>", reflect.TypeOf(r)) + } +} + +func AsBulkBytes(r Resp, err error) ([]byte, error) { + if err != nil { + return nil, err + } + x, ok := r.(*BulkBytes) + if ok && x != nil { + return x.Value, nil + } else { + return nil, errors.Errorf("expect BulkBytes, but got <%s>", reflect.TypeOf(r)) + } +} + +func AsInt(r Resp, err error) (int64, error) { + if err != nil { + return 0, err + } + x, ok := r.(*Int) + if ok && x != nil { + return x.Value, nil + } else { + return 0, errors.Errorf("expect Int, but got <%s>", reflect.TypeOf(r)) + } +} + +func AsArray(r Resp, err error) ([]Resp, error) { + if err != nil { + return nil, err + } + x, ok := r.(*Array) + if ok && x != nil { + return x.Value, nil + } else { + return nil, errors.Errorf("expect Array, but got <%s>", reflect.TypeOf(r)) + } +} + +func NewCommand(cmd string, args ...interface{}) Resp { + r := NewArray() + r.AppendBulkBytes([]byte(cmd)) + for i := 0; i < len(args); i++ { + switch x := args[i].(type) { + case nil: + r.AppendBulkBytes(nil) + case string: + r.AppendBulkBytes([]byte(x)) + case []byte: + r.AppendBulkBytes(x) + default: + r.AppendBulkBytes([]byte(fmt.Sprint(x))) + } + } + return r +} diff --git a/src/pkg/redis/server.go b/src/pkg/redis/server.go new file mode 100644 index 0000000..6bb0fc9 --- /dev/null +++ b/src/pkg/redis/server.go @@ -0,0 +1,39 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import "pkg/libs/errors" + +type Server struct { + t HandlerTable +} + +func NewServer(o interface{}) (*Server, error) { + t, err := NewHandlerTable(o) + if err != nil { + return nil, err + } + return &Server{t}, nil +} + +func NewServerWithTable(t HandlerTable) (*Server, error) { + if t == nil { + return nil, errors.Errorf("handler table is nil") + } + return &Server{t}, nil +} + +func MustServer(o interface{}) *Server { + return &Server{MustHandlerTable(o)} +} + +func (s *Server) Dispatch(arg0 interface{}, resp Resp) (Resp, error) { + if cmd, args, err := ParseArgs(resp); err != nil { + return nil, err + } else if f := s.t[cmd]; f == nil { + return nil, errors.Errorf("unknown command '%s'", cmd) + } else { + return f(arg0, args...) + } +} diff --git a/src/pkg/redis/server_test.go b/src/pkg/redis/server_test.go new file mode 100644 index 0000000..7a91d1e --- /dev/null +++ b/src/pkg/redis/server_test.go @@ -0,0 +1,68 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package redis + +import ( + "bufio" + "bytes" + "testing" + + "pkg/libs/assert" +) + +type testHandler struct { + c map[string]int +} + +func (h *testHandler) count(args ...[]byte) (Resp, error) { + for _, arg := range args { + h.c[string(arg)]++ + } + return nil, nil +} + +func (h *testHandler) Get(arg0 interface{}, args ...[]byte) (Resp, error) { + return h.count(args...) +} + +func (h *testHandler) Set(arg0 interface{}, args [][]byte) (Resp, error) { + return h.count(args...) +} + +func testmapcount(t *testing.T, m1, m2 map[string]int) { + assert.Must(len(m1) == len(m2)) + for k, _ := range m1 { + assert.Must(m1[k] == m2[k]) + } +} + +func TestHandlerFunc(t *testing.T) { + h := &testHandler{make(map[string]int)} + s, err := NewServer(h) + assert.MustNoError(err) + key1, key2, key3, key4 := "key1", "key2", "key3", "key4" + s.t["get"](nil) + testmapcount(t, h.c, map[string]int{}) + s.t["get"](nil, []byte(key1), []byte(key2)) + testmapcount(t, h.c, map[string]int{key1: 1, key2: 1}) + s.t["get"](nil, [][]byte{[]byte(key1), []byte(key3)}...) + testmapcount(t, h.c, map[string]int{key1: 2, key2: 1, key3: 1}) + s.t["set"](nil) + testmapcount(t, h.c, map[string]int{key1: 2, key2: 1, key3: 1}) + s.t["set"](nil, []byte(key1), []byte(key4)) + testmapcount(t, h.c, map[string]int{key1: 3, key2: 1, key3: 1, key4: 1}) + s.t["set"](nil, [][]byte{[]byte(key1), []byte(key2), []byte(key3)}...) + testmapcount(t, h.c, map[string]int{key1: 4, key2: 2, key3: 2, key4: 1}) +} + +func TestServerServe(t *testing.T) { + h := &testHandler{make(map[string]int)} + s, err := NewServer(h) + assert.MustNoError(err) + resp, err := Decode(bufio.NewReader(bytes.NewReader([]byte("*2\r\n$3\r\nset\r\n$3\r\nfoo\r\n")))) + assert.MustNoError(err) + _, err = s.Dispatch(nil, resp) + assert.MustNoError(err) + testmapcount(t, h.c, map[string]int{"foo": 1}) +} diff --git a/src/redis-shake/base/runner.go b/src/redis-shake/base/runner.go new file mode 100644 index 0000000..df6173e --- /dev/null +++ b/src/redis-shake/base/runner.go @@ -0,0 +1,14 @@ +package base + +var( + Status = "null" + AcceptDB = func(db uint32) bool { + return db >= 0 && db < 1024 + } +) + +type Runner interface{ + Main() + + GetDetailedInfo() []interface{} +} \ No newline at end of file diff --git a/src/redis-shake/command/redis-command.go b/src/redis-shake/command/redis-command.go new file mode 100644 index 0000000..2913836 --- /dev/null +++ b/src/redis-shake/command/redis-command.go @@ -0,0 +1,128 @@ +// redis command struct. +package command + +import ( + "strings" +) + +type getkeys_proc func(args []string) []int +type redisCommand struct { + getkey_proc getkeys_proc + firstkey, lastkey, keystep int +} + +var RedisCommands = map[string]redisCommand{ + "set": {nil, 1, 1, 1}, + "setnx": {nil, 1, 1, 1}, + "setex": {nil, 1, 1, 1}, + "psetex": {nil, 1, 1, 1}, + "append": {nil, 1, 1, 1}, + "del": {nil, 1, -1, 1}, + "unlink": {nil, 1, -1, 1}, + "setbit": {nil, 1, 1, 1}, + "bitfield": {nil, 1, 1, 1}, + "setrange": {nil, 1, 1, 1}, + "incr": {nil, 1, 1, 1}, + "decr": {nil, 1, 1, 1}, + "rpush": {nil, 1, 1, 1}, + "lpush": {nil, 1, 1, 1}, + "rpushx": {nil, 1, 1, 1}, + "lpushx": {nil, 1, 1, 1}, + "linsert": {nil, 1, 1, 1}, + "rpop": {nil, 1, 1, 1}, + "lpop": {nil, 1, 1, 1}, + "brpop": {nil, 1, -2, 1}, + "brpoplpush": {nil, 1, 2, 1}, + "blpop": {nil, 1, -2, 1}, + "lset": {nil, 1, 1, 1}, + "ltrim": {nil, 1, 1, 1}, + "lrem": {nil, 1, 1, 1}, + "rpoplpush": {nil, 1, 2, 1}, + "sadd": {nil, 1, 1, 1}, + "srem": {nil, 1, 1, 1}, + "smove": {nil, 1, 2, 1}, + "spop": {nil, 1, 1, 1}, + "sinterstore": {nil, 1, -1, 1}, + "sunionstore": {nil, 1, -1, 1}, + "sdiffstore": {nil, 1, -1, 1}, + "zadd": {nil, 1, 1, 1}, + "zincrby": {nil, 1, 1, 1}, + "zrem": {nil, 1, 1, 1}, + "zremrangebyscore": {nil, 1, 1, 1}, + "zremrangebyrank": {nil, 1, 1, 1}, + "zremrangebylex": {nil, 1, 1, 1}, + //"zunionstore", {zunionInterGetKeys, 0, 0, 0}, + //"zinterstore", {zunionInterGetKeys, 0, 0, 0}, + "hset": {nil, 1, 1, 1}, + "hsetnx": {nil, 1, 1, 1}, + "hmset": {nil, 1, 1, 1}, + "hincrby": {nil, 1, 1, 1}, + "hincrbyfloat": {nil, 1, 1, 1}, + "hdel": {nil, 1, 1, 1}, + "incrby": {nil, 1, 1, 1}, + "decrby": {nil, 1, 1, 1}, + "incrbyfloat": {nil, 1, 1, 1}, + "getset": {nil, 1, 1, 1}, + "mset": {nil, 1, -1, 2}, + "msetnx": {nil, 1, -1, 2}, + "move": {nil, 1, 1, 1}, + "rename": {nil, 1, 2, 1}, + "renamenx": {nil, 1, 2, 1}, + "expire": {nil, 1, 1, 1}, + "expireat": {nil, 1, 1, 1}, + "pexpire": {nil, 1, 1, 1}, + "pexpireat": {nil, 1, 1, 1}, + //"sort", {sortGetKeys, 1, 1, 1}, + "persist": {nil, 1, 1, 1}, + "restore": {nil, 1, 1, 1}, + "restore-asking": {nil, 1, 1, 1}, + //"eval", {evalGetKeys, 0, 0, 0}, + //"evalsha", {evalGetKeys, 0, 0, 0}, + "bitop": {nil, 2, -1, 1}, + "geoadd": {nil, 1, 1, 1}, + //"georadius", {georadiusGetKeys, 1, 1, 1}, + //"georadiusbymember", {georadiusGetKeys, 1, 1, 1}, + "pfadd": {nil, 1, 1, 1}, + "pfmerge": {nil, 1, -1, 1}, +} + +func GetMatchKeys(redis_cmd redisCommand, args [][]byte, filterkeys []string) (new_args [][]byte, ret bool) { + lastkey := redis_cmd.lastkey - 1 + keystep := redis_cmd.keystep + + if lastkey < 0 { + lastkey = lastkey + len(args) + } + + array := make([]int, len(args)) + number := 0 + for firstkey := redis_cmd.firstkey - 1; firstkey <= lastkey; firstkey += keystep { + key := string(args[firstkey]) + for i := 0; i < len(filterkeys); i++ { + if strings.HasPrefix(key, filterkeys[i]) { + array[number] = firstkey + number++ + break + } + } + } + + ret = false + new_args = make([][]byte, number*redis_cmd.keystep+len(args)-lastkey-redis_cmd.keystep) + if number > 0 { + ret = true + for i := 0; i < number; i++ { + for j := 0; j < redis_cmd.keystep; j++ { + new_args[i*redis_cmd.keystep+j] = args[array[i]+j] + } + } + } + + // add alis paramters + j := 0 + for i := lastkey + redis_cmd.keystep; i < len(args); i++ { + new_args[number*redis_cmd.keystep+j] = args[i] + j = j + 1 + } + return +} diff --git a/src/redis-shake/command/redis-command_test.go b/src/redis-shake/command/redis-command_test.go new file mode 100644 index 0000000..adff5ba --- /dev/null +++ b/src/redis-shake/command/redis-command_test.go @@ -0,0 +1,125 @@ +package command + +import ( + "testing" +) + +func Test_Get_Match_Keys_Mset_Cmd(t *testing.T) { + mset_cmd := redisCommands["mset"] + /*filterkey: x + *cmd: mset kk 1 + */ + args := make([][]byte, 2) + args[0] = []byte("kk") + args[1] = []byte("1") + filterkey := make([]string, 1) + filterkey[0] = "x" + new_args, ret := GetMatchKeys(mset_cmd, args, filterkey) + + if len(new_args) != 0 || ret != false { + t.Error("mset test fail") + } + + /*filterkey: k + *cmd: mset kk 1 + */ + args = make([][]byte, 2) + args[0] = []byte("kk") + args[1] = []byte("1") + filterkey = make([]string, 1) + filterkey[0] = "k" + new_args, ret = GetMatchKeys(mset_cmd, args, filterkey) + + if len(new_args) != 2 || ret != true { + t.Error("mset test fail") + } + + /*filterkey: k + *cmd: mset kk 1 gg ll zz nn k ll + */ + args = make([][]byte, 8) + args[0] = []byte("kk") + args[1] = []byte("1") + args[2] = []byte("gg") + args[3] = []byte("ll") + args[4] = []byte("zz") + args[5] = []byte("nn") + args[6] = []byte("k") + args[7] = []byte("ll") + filterkey = make([]string, 1) + filterkey[0] = "k" + new_args, ret = GetMatchKeys(mset_cmd, args, filterkey) + + if len(new_args) != 4 || ret != true || + string(new_args[0]) != "kk" || string(new_args[1]) != "1" || + string(new_args[2]) != "k" || string(new_args[3]) != "ll" { + t.Error("mset test fail") + } +} + +func Test_Get_Match_Keys_SetXX_Cmd(t *testing.T) { + set_cmd := redisCommands["set"] + /*filterkey: x + *cmd: set kk 1 + */ + args := make([][]byte, 2) + args[0] = []byte("kk") + args[1] = []byte("1") + filterkey := make([]string, 1) + filterkey[0] = "x" + new_args, ret := GetMatchKeys(set_cmd, args, filterkey) + + if ret != false { + t.Error("set test fail", ret, len(new_args)) + } + + /*filterkey: k + *cmd: set kk 1 + */ + args = make([][]byte, 2) + args[0] = []byte("kk") + args[1] = []byte("1") + filterkey = make([]string, 1) + filterkey[0] = "k" + new_args, ret = GetMatchKeys(set_cmd, args, filterkey) + + if len(new_args) != 2 || ret != true { + t.Error("set test fail") + } + + /*filterkey: k + *cmd: setex kk 3000 lll + */ + set_cmd = redisCommands["setex"] + args = make([][]byte, 3) + args[0] = []byte("kk") + args[1] = []byte("3000") + args[2] = []byte("lll") + filterkey = make([]string, 1) + filterkey[0] = "k" + new_args, ret = GetMatchKeys(set_cmd, args, filterkey) + + if len(new_args) != 3 || ret != true || + string(new_args[0]) != "kk" || string(new_args[1]) != "3000" || + string(new_args[2]) != "lll" { + t.Error("setex test fail") + } + + /*filterkey: k + *cmd: setrange kk 3000 lll + */ + set_cmd = redisCommands["setrange"] + args = make([][]byte, 3) + args[0] = []byte("kk") + args[1] = []byte("3000") + args[2] = []byte("lll") + filterkey = make([]string, 1) + filterkey[0] = "k" + new_args, ret = GetMatchKeys(set_cmd, args, filterkey) + + if len(new_args) != 3 || ret != true || + string(new_args[0]) != "kk" || string(new_args[1]) != "3000" || + string(new_args[2]) != "lll" { + t.Error("setrange test fail") + } +} diff --git a/src/redis-shake/common/common.go b/src/redis-shake/common/common.go new file mode 100644 index 0000000..7fed11f --- /dev/null +++ b/src/redis-shake/common/common.go @@ -0,0 +1,18 @@ +package utils + +import( + logRotate "gopkg.in/natefinch/lumberjack.v2" + "pkg/libs/bytesize" +) + +const( + // GolangSecurityTime = "2006-01-02T15:04:05Z" + GolangSecurityTime = "2006-01-02 15:04:05" + ReaderBufferSize = bytesize.MB * 32 + WriterBufferSize = bytesize.MB * 8 +) + +var( + Version = "$" + LogRotater *logRotate.Logger +) \ No newline at end of file diff --git a/src/redis-shake/common/crc16.go b/src/redis-shake/common/crc16.go new file mode 100644 index 0000000..5f4f9c4 --- /dev/null +++ b/src/redis-shake/common/crc16.go @@ -0,0 +1,88 @@ +package utils + +/* + * Copyright 2001-2010 Georges Menie (www.menie.org) + * Copyright 2010-2012 Salvatore Sanfilippo (adapted to Redis coding style) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California, Berkeley nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* CRC16 implementation according to CCITT standards. + * + * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the + * following parameters: + * + * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" + * Width : 16 bit + * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) + * Initialization : 0000 + * Reflect Input byte : False + * Reflect Output CRC : False + * Xor constant to output CRC : 0000 + * Output for "123456789" : 31C3 + */ + +var crc16tab = [256]uint16{ + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, + 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, + 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, + 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, + 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, + 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, + 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, + 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, + 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, + 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, + 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, + 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, + 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, + 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, + 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, + 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, + 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, + 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, + 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, + 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, + 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, + 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, + 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, + 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, + 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, + 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, + 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, +} + +func crc16(buf string) uint16 { + var crc uint16 + for _, n := range buf { + crc = (crc << uint16(8)) ^ crc16tab[((crc>>uint16(8))^uint16(n))&0x00FF] + } + return crc +} diff --git a/src/redis-shake/common/http.go b/src/redis-shake/common/http.go new file mode 100644 index 0000000..1abd65f --- /dev/null +++ b/src/redis-shake/common/http.go @@ -0,0 +1,13 @@ +package utils + +import ( + "github.com/gugemichael/nimo4go" +) + +var ( + HttpApi *nimo.HttpRestProvider +) + +func InitHttpApi(port int) { + HttpApi = nimo.NewHttpRestProvdier(port) +} \ No newline at end of file diff --git a/src/redis-shake/common/mix.go b/src/redis-shake/common/mix.go new file mode 100644 index 0000000..3945d14 --- /dev/null +++ b/src/redis-shake/common/mix.go @@ -0,0 +1,72 @@ +package utils + +import ( + "os" + "path/filepath" + + "pkg/libs/log" + + "github.com/nightlyone/lockfile" +) + +func WritePid(id string) (err error) { + var lock lockfile.Lockfile + lock, err = lockfile.New(id) + if err != nil { + return err + } + if err = lock.TryLock(); err != nil { + return err + } + + return nil +} + +func WritePidById(id string) error { + dir, _ := os.Getwd() + pidfile := filepath.Join(dir, id) + ".pid" + if err := WritePid(pidfile); err != nil { + return err + } + return nil +} + +func Welcome() { + welcome := + `______________________________ +\ \ _ ______ | + \ \ / \___-=O'/|O'/__| + \ redis-shake, here we go !! \_______\ / | / ) + / / '/-==__ _/__|/__=-| -GM + / / * \ | | +/ / (o) +------------------------------ +` + + log.Warn("\n", welcome) +} + +func Goodbye() { + goodbye := ` + ##### | ##### +Oh we finish ? # _ _ #|# _ _ # + # | # + | ############ + # # + | # # + # # + | | # # | | + | | # # | + | | | # .-. # | + #( O )# | | | + | ################. .############### | + ## _ _|____| ### |_ __| _ ## + # | | # + # | | | | | | | | # + ###################################### + # # + ##### +` + + log.Warn(goodbye) +} diff --git a/src/redis-shake/common/slot.go b/src/redis-shake/common/slot.go new file mode 100644 index 0000000..62a6a82 --- /dev/null +++ b/src/redis-shake/common/slot.go @@ -0,0 +1,19 @@ +package utils + +func KeyToSlot(key string) uint16 { + hashtag := "" + for i, s := range key { + if s == '{' { + for k := i; k < len(key); k++ { + if key[k] == '}' { + hashtag = key[i+1 : k] + break + } + } + } + } + if len(hashtag) > 0 { + return crc16(hashtag) & 0x3fff + } + return crc16(key) & 0x3fff +} diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go new file mode 100644 index 0000000..98d5bd4 --- /dev/null +++ b/src/redis-shake/common/utils.go @@ -0,0 +1,864 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package utils + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" + "time" + + "pkg/libs/atomic2" + "pkg/libs/errors" + "pkg/libs/log" + "pkg/libs/stats" + "pkg/rdb" + "pkg/redis" + redigo "github.com/garyburd/redigo/redis" + "redis-shake/configure" +) + +func OpenRedisConn(target, auth_type, passwd string) redigo.Conn { + return redigo.NewConn(OpenNetConn(target, auth_type, passwd), 0, 0) +} + +func OpenRedisConnWithTimeout(target, auth_type, passwd string, readTimeout, writeTimeout time.Duration) redigo.Conn { + return redigo.NewConn(OpenNetConn(target, auth_type, passwd), readTimeout, writeTimeout) +} + +func OpenNetConn(target, auth_type, passwd string) net.Conn { + c, err := net.Dial("tcp", target) + if err != nil { + log.PanicErrorf(err, "cannot connect to '%s'", target) + } + + AuthPassword(c, auth_type, passwd) + return c +} + +func OpenNetConnSoft(target, auth_type, passwd string) net.Conn { + c, err := net.Dial("tcp", target) + if err != nil { + return nil + } + AuthPassword(c, auth_type, passwd) + return c +} + +func OpenReadFile(name string) (*os.File, int64) { + f, err := os.Open(name) + if err != nil { + log.PanicErrorf(err, "cannot open file-reader '%s'", name) + } + s, err := f.Stat() + if err != nil { + log.PanicErrorf(err, "cannot stat file-reader '%s'", name) + } + return f, s.Size() +} + +func OpenWriteFile(name string) *os.File { + f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + if err != nil { + log.PanicErrorf(err, "cannot open file-writer '%s'", name) + } + return f +} + +func OpenReadWriteFile(name string) *os.File { + f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + log.PanicErrorf(err, "cannot open file-readwriter '%s'", name) + } + return f +} + +func SendPSyncListeningPort(c net.Conn, port int) { + + _, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand("replconf", "listening-port", port))) + if err != nil { + log.PanicError(errors.Trace(err), "write replconf listening-port failed") + } + var b = make([]byte, 5) + if _, err := io.ReadFull(c, b); err != nil { + log.PanicError(errors.Trace(err), "read auth response failed") + } + if strings.ToUpper(string(b)) != "+OK\r\n" { + log.Panic("repl listening-port failed: ", string(b)) + } +} + +func AuthPassword(c net.Conn, auth_type, passwd string) { + if passwd == "" { + return + } + + _, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand(auth_type, passwd))) + if err != nil { + log.PanicError(errors.Trace(err), "write auth command failed") + } + var b = make([]byte, 5) + if _, err := io.ReadFull(c, b); err != nil { + log.PanicError(errors.Trace(err), "read auth response failed") + } + if strings.ToUpper(string(b)) != "+OK\r\n" { + log.Panicf("auth failed[%s]", string(b)) + } +} + +func OpenSyncConn(target string, auth_type, passwd string) (net.Conn, <-chan int64) { + c := OpenNetConn(target, auth_type, passwd) + if _, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand("sync"))); err != nil { + log.PanicError(errors.Trace(err), "write sync command failed") + } + return c, waitRdbDump(c) +} + +func waitRdbDump(r io.Reader) <-chan int64 { + size := make(chan int64) + go func() { + var rsp string + for { + b := []byte{0} + if _, err := r.Read(b); err != nil { + log.PanicErrorf(err, "read sync response = '%s'", rsp) + } + if len(rsp) == 0 && b[0] == '\n' { + size <- 0 + continue + } + rsp += string(b) + if strings.HasSuffix(rsp, "\r\n") { + break + } + } + if rsp[0] != '$' { + log.Panicf("invalid sync response, rsp = '%s'", rsp) + } + n, err := strconv.Atoi(rsp[1 : len(rsp)-2]) + if err != nil || n <= 0 { + log.PanicErrorf(err, "invalid sync response = '%s', n = %d", rsp, n) + } + size <- int64(n) + }() + return size +} + +func SendPSyncFullsync(br *bufio.Reader, bw *bufio.Writer) (string, int64, <-chan int64) { + cmd := redis.NewCommand("psync", "?", -1) + if err := redis.Encode(bw, cmd, true); err != nil { + log.PanicError(err, "write psync command failed, fullsync") + } + r, err := redis.Decode(br) + if err != nil { + log.PanicError(err, "invalid psync response, fullsync") + } + if e, ok := r.(*redis.Error); ok { + log.Panicf("invalid psync response, fullsync, %s", e.Value) + } + x, err := redis.AsString(r, nil) + if err != nil { + log.PanicError(err, "invalid psync response, fullsync") + } + xx := strings.Split(string(x), " ") + if len(xx) < 3 || strings.ToLower(xx[0]) != "fullresync" { + log.Panicf("invalid psync response = '%s', should be fullsync", x) + } + v, err := strconv.ParseInt(xx[2], 10, 64) + if err != nil { + log.PanicError(err, "parse psync offset failed") + } + + // log.PurePrintf("%s\n", NewLogItem("FullSyncStart", "INFO", LogDetail{})) + log.Infof("Event:FullSyncStart\tId:%s\t", conf.Options.Id) + runid, offset := xx[1], v + return runid, offset, waitRdbDump(br) +} + +func SendPSyncContinue(br *bufio.Reader, bw *bufio.Writer, runid string, offset int64) { + cmd := redis.NewCommand("psync", runid, offset+1) + if err := redis.Encode(bw, cmd, true); err != nil { + log.PanicError(err, "write psync command failed, continue") + } + r, err := redis.Decode(br) + if err != nil { + log.PanicError(err, "invalid psync response, continue") + } + if e, ok := r.(*redis.Error); ok { + log.Panicf("invalid psync response, continue, %s", e.Value) + } + x, err := redis.AsString(r, nil) + if err != nil { + log.PanicError(err, "invalid psync response, continue") + } + xx := strings.Split(string(x), " ") + if len(xx) != 1 || strings.ToLower(xx[0]) != "continue" { + if strings.ToLower(xx[0]) == "fullresync" { + + // log.PurePrintf("%s\n", NewLogItem("ExpectContinueButFullSync", "ERROR", NewErrorLogDetail(string(x), ""))) + log.Errorf("Event:ExpectContinueButFullSync\tId:%s\tReply:%s", conf.Options.Id, x) + } + log.Panicf("invalid psync response = '%s', should be continue", x) + } + + // log.PurePrintf("%s\n", NewLogItem("IncSyncStart", "INFO", LogDetail{})) + log.Infof("Event:IncSyncStart\tId:%s\t", conf.Options.Id) +} + +func SendPSyncAck(bw *bufio.Writer, offset int64) error { + cmd := redis.NewCommand("replconf", "ack", offset) + return redis.Encode(bw, cmd, true) +} + +// TODO +func paclusterSlotsHB(c redigo.Conn) { + _, err := redigo.Values(c.Do("pacluster", "slotshb")) + if err != nil { + log.PanicError(err, "pacluster slotshb error") + } +} + +func SelectDB(c redigo.Conn, db uint32) { + s, err := redigo.String(c.Do("select", db)) + if err != nil { + log.PanicError(err, "select command error") + } + if s != "OK" { + log.Panicf("select command response = '%s', should be 'OK'", s) + } +} + +func lpush(c redigo.Conn, key []byte, field []byte) { + _, err := redigo.Int64(c.Do("lpush", string(key), string(field))) + if err != nil { + log.PanicError(err, "lpush command error") + } +} + +func rpush(c redigo.Conn, key []byte, field []byte) { + _, err := redigo.Int64(c.Do("rpush", string(key), string(field))) + if err != nil { + log.PanicError(err, "rpush command error") + } +} + +func Float64ToByte(float float64) string { + return strconv.FormatFloat(float, 'f', -1, 64) +} + +func zadd(c redigo.Conn, key []byte, score []byte, member []byte) { + _, err := redigo.Int64(c.Do("zadd", string(key), string(score), string(member))) + if err != nil { + log.PanicError(err, "zadd command error key ", string(key)) + } +} + +func sadd(c redigo.Conn, key []byte, member []byte) { + _, err := redigo.Int64(c.Do("sadd", key, member)) + if err != nil { + log.PanicError(err, "sadd command error key:", string(key)) + } +} + +func hset(c redigo.Conn, key []byte, field []byte, value []byte) { + _, err := redigo.Int64(c.Do("hset", string(key), string(field), string(value))) + if err != nil { + log.PanicError(err, "hset command error key ", string(key)) + } +} + +func set(c redigo.Conn, key []byte, value []byte) { + s, err := redigo.String(c.Do("set", string(key), string(value))) + if err != nil { + log.PanicError(err, "set command error") + } + if s != "OK" { + log.Panicf("set command response = '%s', should be 'OK'", s) + } +} + +func flushAndCheckReply(c redigo.Conn, count int) { + c.Flush() + for j := 0; j < count; j++ { + _, err := c.Receive() + if err != nil { + log.PanicError(err, "flush command to redis failed") + } + } +} + +func restoreQuicklistEntry(c redigo.Conn, e *rdb.BinEntry) { + + r := rdb.NewRdbReader(bytes.NewReader(e.Value)) + _, err := r.ReadByte() + if err != nil { + log.PanicError(err, "read rdb ") + } + //log.Info("restore quicklist key: ", string(e.Key), ", type: ", t) + + count := 0 + if n, err := r.ReadLength(); err != nil { + log.PanicError(err, "read rdb ") + } else { + //log.Info("quicklist item size: ", int(n)) + for i := 0; i < int(n); i++ { + ziplist, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(ziplist) + if zln, err := r.ReadZiplistLength(buf); err != nil { + log.PanicError(err, "read rdb") + } else { + //log.Info("ziplist one of quicklist, size: ", int(zln)) + for i := int64(0); i < zln; i++ { + entry, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + c.Send("RPUSH", e.Key, entry) + if count == 100 { + flushAndCheckReply(c, count) + count = 0 + } + } + flushAndCheckReply(c, count) + count = 0 + } + } + } +} + +func restoreBigRdbEntry(c redigo.Conn, e *rdb.BinEntry) { + //read type + r := rdb.NewRdbReader(bytes.NewReader(e.Value)) + t, err := r.ReadByte() + if err != nil { + log.PanicError(err, "read rdb ") + } + log.Info("restore big key ", string(e.Key), " Value Length ", len(e.Value), " type ", t) + count := 0 + switch t { + case rdb.RdbTypeHashZiplist: + //ziplist + ziplist, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(ziplist) + length, err := r.ReadZiplistLength(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + length /= 2 + log.Info("restore big hash key ", string(e.Key), " field count ", length) + for i := int64(0); i < length; i++ { + field, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + value, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + c.Send("HSET", e.Key, field, value) + if (count == 100) || (i == (length - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //hset(c, e.Key, field, value) + } + case rdb.RdbTypeZSetZiplist: + ziplist, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(ziplist) + cardinality, err := r.ReadZiplistLength(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + cardinality /= 2 + log.Info("restore big zset key ", string(e.Key), " field count ", cardinality) + for i := int64(0); i < cardinality; i++ { + member, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + scoreBytes, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + _, err = strconv.ParseFloat(string(scoreBytes), 64) + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + c.Send("ZADD", e.Key, scoreBytes, member) + if (count == 100) || (i == (cardinality - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //zadd(c, e.Key, scoreBytes, member) + } + case rdb.RdbTypeSetIntset: + intset, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(intset) + intSizeBytes, err := buf.Slice(4) + if err != nil { + log.PanicError(err, "read rdb ") + } + intSize := binary.LittleEndian.Uint32(intSizeBytes) + + if intSize != 2 && intSize != 4 && intSize != 8 { + log.PanicError(err, "rdb: unknown intset encoding ") + } + + lenBytes, err := buf.Slice(4) + if err != nil { + log.PanicError(err, "read rdb ") + } + cardinality := binary.LittleEndian.Uint32(lenBytes) + + log.Info("restore big set key ", string(e.Key), " field count ", cardinality) + for i := uint32(0); i < cardinality; i++ { + intBytes, err := buf.Slice(int(intSize)) + if err != nil { + log.PanicError(err, "read rdb ") + } + var intString string + switch intSize { + case 2: + intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) + case 4: + intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) + case 8: + intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) + } + count++ + c.Send("SADD", e.Key, []byte(intString)) + if (count == 100) || (i == (cardinality - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //sadd(c, e.Key, []byte(intString)) + } + case rdb.RdbTypeListZiplist: + ziplist, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(ziplist) + length, err := r.ReadZiplistLength(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + log.Info("restore big list key ", string(e.Key), " field count ", length) + for i := int64(0); i < length; i++ { + entry, err := r.ReadZiplistEntry(buf) + if err != nil { + log.PanicError(err, "read rdb ") + } + //rpush(c, e.Key, entry) + count++ + c.Send("RPUSH", e.Key, entry) + if (count == 100) || (i == (length - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + } + case rdb.RdbTypeHashZipmap: + var length int + zipmap, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + buf := rdb.NewSliceBuffer(zipmap) + lenByte, err := buf.ReadByte() + if err != nil { + log.PanicError(err, "read rdb ") + } + if lenByte >= 254 { // we need to count the items manually + length, err = r.CountZipmapItems(buf) + length /= 2 + if err != nil { + log.PanicError(err, "read rdb ") + } + } else { + length = int(lenByte) + } + log.Info("restore big hash key ", string(e.Key), " field count ", length) + for i := 0; i < length; i++ { + field, err := r.ReadZipmapItem(buf, false) + if err != nil { + log.PanicError(err, "read rdb ") + } + value, err := r.ReadZipmapItem(buf, true) + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + c.Send("HSET", e.Key, field, value) + if (count == 100) || (i == (int(length) - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //hset(c, e.Key, field, value) + } + case rdb.RdbTypeString: + value, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + set(c, e.Key, value) + case rdb.RdbTypeList: + if n, err := r.ReadLength(); err != nil { + log.PanicError(err, "read rdb ") + } else { + log.Info("restore big list key ", string(e.Key), " field count ", int(n)) + for i := 0; i < int(n); i++ { + field, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + //rpush(c, e.Key, field) + count++ + c.Send("RPUSH", e.Key, field) + if (count == 100) || (i == (int(n) - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + } + } + case rdb.RdbTypeSet: + if n, err := r.ReadLength(); err != nil { + log.PanicError(err, "read rdb ") + } else { + log.Info("restore big set key ", string(e.Key), " field count ", int(n)) + for i := 0; i < int(n); i++ { + member, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + c.Send("SADD", e.Key, member) + if (count == 100) || (i == (int(n) - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //sadd(c, e.Key, member) + } + } + case rdb.RdbTypeZSet, rdb.RdbTypeZSet2: + if n, err := r.ReadLength(); err != nil { + log.PanicError(err, "read rdb ") + } else { + log.Info("restore big zset key ", string(e.Key), " field count ", int(n)) + for i := 0; i < int(n); i++ { + member, err := r.ReadString() + if err != nil { + log.PanicError(err, "read rdb ") + } + var score float64 + if t == rdb.RdbTypeZSet2 { + score, err = r.ReadDouble() + } else { + score, err = r.ReadFloat() + } + if err != nil { + log.PanicError(err, "read rdb ") + } + count++ + log.Info("restore big zset key ", string(e.Key), " score ", (Float64ToByte(score)), " member ", string(member)) + c.Send("ZADD", e.Key, Float64ToByte(score), member) + if (count == 100) || (i == (int(n) - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + //zadd(c, e.Key, Float64ToByte(score), member) + } + } + case rdb.RdbTypeHash: + var n uint32 + if e.NeedReadLen == 1 { + rlen, err := r.ReadLength() + if err != nil { + log.PanicError(err, "read rdb ") + } + if e.RealMemberCount != 0 { + n = e.RealMemberCount + } else { + n = rlen + } + } else { + n = e.RealMemberCount + } + log.Info("restore big hash key ", string(e.Key), " field count ", int(n)) + for i := 0; i < int(n); i++ { + field, err := r.ReadString() + if err != nil { + log.Info("idx: ", i, " n: ", n) + log.PanicError(err, "read rdb ") + } + value, err := r.ReadString() + if err != nil { + log.Info("idx: ", i, " n: ", n) + log.PanicError(err, "read rdb ") + } + //hset(c, e.Key, field, value) + count++ + c.Send("HSET", e.Key, field, value) + if (count == 100) || (i == (int(n) - 1)) { + flushAndCheckReply(c, count) + count = 0 + } + } + log.Info("complete restore big hash key: ", string(e.Key), " field:", n) + default: + log.PanicError(fmt.Errorf("cann't deal rdb type:%d", t), "restore big key fail") + } +} + +func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { + var ttlms uint64 + if conf.Options.ReplaceHashTag { + e.Key = bytes.Replace(e.Key, []byte("{"), []byte(""), 1) + e.Key = bytes.Replace(e.Key, []byte("}"), []byte(""), 1) + } + if e.ExpireAt != 0 { + now := uint64(time.Now().Add(conf.Options.ShiftTime).UnixNano()) + now /= uint64(time.Millisecond) + if now >= e.ExpireAt { + ttlms = 1 + } else { + ttlms = e.ExpireAt - now + } + } + if e.Type == rdb.RdbTypeQuicklist { + exist, err := redigo.Bool(c.Do("exists", e.Key)) + if err != nil { + log.Panicf(err.Error()) + } + if exist { + if conf.Options.Rewrite { + if !conf.Options.Metric { + log.Infof("warning, rewrite key: %v", string(e.Key)) + } + _, err := redigo.Int64(c.Do("del", e.Key)) + if err != nil { + log.Panicf("del ", string(e.Key), err) + } + } else { + log.Panicf("target key name is busy: %s", string(e.Key)) + } + } + restoreQuicklistEntry(c, e) + if e.ExpireAt != 0 { + r, err := redigo.Int64(c.Do("expire", e.Key, ttlms)) + if err != nil && r != 1 { + log.Panicf("expire ", string(e.Key), err) + } + } + return + } + + if uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0 { + //use command + if conf.Options.Rewrite && e.NeedReadLen == 1 { + if !conf.Options.Metric { + log.Infof("warning, rewrite big key:", string(e.Key)) + } + _, err := redigo.Int64(c.Do("del", e.Key)) + if err != nil { + log.Panicf("del ", string(e.Key), err) + } + } + restoreBigRdbEntry(c, e) + if e.ExpireAt != 0 { + r, err := redigo.Int64(c.Do("expire", e.Key, ttlms)) + if err != nil && r != 1 { + log.Panicf("expire ", string(e.Key), err) + } + } + return + } + s, err := redigo.String(c.Do("restore", e.Key, ttlms, e.Value)) + if err != nil { + /*The reply value of busykey in 2.8 kernel is "target key name is busy", + but in 4.0 kernel is "BUSYKEY Target key name already exists"*/ + if strings.Contains(err.Error(), "Target key name is busy") || strings.Contains(err.Error(), "BUSYKEY Target key name already exists") { + if conf.Options.Rewrite { + if !conf.Options.Metric { + log.Infof("warning, rewrite key: %v", string(e.Key)) + } + var s2 string + var rerr error + if conf.Options.TargetReplace { + s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value, "replace")) + } else { + _, _ = redigo.String(c.Do("del", e.Key)) + s2, rerr = redigo.String(c.Do("restore", e.Key, ttlms, e.Value)) + } + if rerr != nil { + log.Info(s2, rerr, "key ", string(e.Key)) + } + } else { + log.Panicf("target key name is busy:", string(e.Key)) + } + } else { + log.PanicError(err, "restore command error key:", string(e.Key), "err:", err.Error()) + } + } else { + if s != "OK" { + log.Panicf("restore command response = '%s', should be 'OK'", s) + } + } +} + +func Iocopy(r io.Reader, w io.Writer, p []byte, max int) int { + if max <= 0 || len(p) == 0 { + log.Panicf("invalid max = %d, len(p) = %d", max, len(p)) + } + if len(p) > max { + p = p[:max] + } + if n, err := r.Read(p); err != nil { + log.PanicError(err, "read error") + } else { + p = p[:n] + } + if _, err := w.Write(p); err != nil { + log.PanicError(err, "write error") + } + return len(p) +} + +func FlushWriter(w *bufio.Writer) { + if err := w.Flush(); err != nil { + log.PanicError(err, "flush error") + } +} + +func NewRDBLoader(reader *bufio.Reader, rbytes *atomic2.Int64, size int) chan *rdb.BinEntry { + pipe := make(chan *rdb.BinEntry, size) + go func() { + defer close(pipe) + l := rdb.NewLoader(stats.NewCountReader(reader, rbytes)) + if err := l.Header(); err != nil { + log.PanicError(err, "parse rdb header error") + } + for { + if entry, err := l.NextBinEntry(); err != nil { + log.PanicError(err, "parse rdb entry error, if the err is :EOF, please check that if the src db log has client outout buffer oom, if so set output buffer larger.") + } else { + if entry != nil { + pipe <- entry + } else { + if rdb.FromVersion > 2 { + if err := l.Footer(); err != nil { + log.PanicError(err, "parse rdb checksum error") + } + } + return + } + } + } + }() + return pipe +} + +func ParseRedisInfo(content []byte) map[string]string { + result := make(map[string]string, 10) + lines := bytes.Split(content, []byte("\r\n")) + for i := 0; i < len(lines); i++ { + items := bytes.SplitN(lines[i], []byte(":"), 2) + if len(items) != 2 { + continue + } + result[string(items[0])] = string(items[1]) + } + return result +} + +func GetRedisVersion(target, authType, auth string) (string, error) { + c := OpenRedisConn(target, authType, auth) + infoStr, err := redigo.Bytes(c.Do("info", "server")) + if err != nil { + return "", err + } + infoKV := ParseRedisInfo(infoStr) + if value, ok := infoKV["redis_version"]; ok { + return value, nil + } else { + return "", fmt.Errorf("MissingRedisVersionInInfo") + } +} + +func CheckHandleNetError(err error) bool { + if err == io.EOF { + return true + } else if _, ok := err.(net.Error); ok { + return true + } + return false +} + +func GetFakeSlaveOffset(c redigo.Conn) (string, error) { + infoStr, err := redigo.Bytes(c.Do("info", "Replication")) + if err != nil { + return "", err + } + + kv := ParseRedisInfo(infoStr) + + for k, v := range kv { + if strings.Contains(k, "slave") && strings.Contains(v, fmt.Sprintf("port=%d", conf.Options.HttpProfile)) { + list := strings.Split(v, ",") + for _, item := range list { + if strings.HasPrefix(item, "offset=") { + return strings.Split(item, "=")[1], nil + } + } + } + } + return "", fmt.Errorf("OffsetNotFoundInInfo") +} + +func GetLocalIp(preferdInterfaces []string) (ip string, interfaceName string, err error) { + var addr net.Addr + ip = "" + for _, name := range preferdInterfaces { + i, err := net.InterfaceByName(name) + if err != nil { + continue + } + addrs, err := i.Addrs() + if err != nil || len(addrs) == 0 { + continue + } + addr = addrs[0] + + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP.String() + case *net.IPAddr: + ip = v.IP.String() + } + if len(ip) != 0 { + return ip, name, nil + } + } + return ip, "", fmt.Errorf("fetch local ip failed, interfaces: %s", strings.Join(preferdInterfaces, ",")) +} diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go new file mode 100644 index 0000000..ec16e8d --- /dev/null +++ b/src/redis-shake/configure/configure.go @@ -0,0 +1,56 @@ +package conf + +import "time" + +type Configuration struct { + // config file variables + Id string `config:"id"` + LogFile string `config:"log_file"` + SystemProfile int `config:"system_profile"` + HttpProfile int `config:"http_profile"` + NCpu int `config:"ncpu"` + Parallel int `config:"parallel"` + InputRdb string `config:"input_rdb"` + OutputRdb string `config:"output_rdb"` + SourceAddress string `config:"source.address"` + SourcePasswordRaw string `config:"source.password_raw"` + SourcePasswordEncoding string `config:"source.password_encoding"` + SourceVersion uint `config:"source.version"` + SourceAuthType string `config:"source.auth_type"` + TargetAddress string `config:"target.address"` + TargetPasswordRaw string `config:"target.password_raw"` + TargetPasswordEncoding string `config:"target.password_encoding"` + TargetVersion uint `config:"target.version"` + TargetDB int `config:"target.db"` + TargetAuthType string `config:"target.auth_type"` + FakeTime string `config:"fake_time"` + Rewrite bool `config:"rewrite"` + FilterDB string `config:"filter.db"` + FilterKey []string `config:"filter.key"` + FilterSlot []string `config:"filter.slot"` + BigKeyThreshold uint64 `config:"big_key_threshold"` + Psync bool `config:"psync"` + Metric bool `config:"metric"` + MetricPrintLog bool `config:"metric.print_log"` + HeartbeatUrl string `config:"heartbeat.url"` + HeartbeatInterval uint `config:"heartbeat.interval"` + HeartbeatExternal string `config:"heartbeat.external"` + HeartbeatNetworkInterface string `config:"heartbeat.network_interface"` + SenderSize uint64 `config:"sender.size"` + SenderCount uint `config:"sender.count"` + SenderDelayChannelSize uint `config:"sender.delay_channel_size"` + + // inner variables + ReplaceHashTag bool `config:"replace_hash_tag"` + ExtraInfo bool `config:"extra"` + SockFileName string `config:"sock.file_name"` + SockFileSize uint `config:"sock.file_size"` + + // generated variables + HeartbeatIp string + ShiftTime time.Duration // shift + TargetRedisVersion string // to_redis_version + TargetReplace bool // to_replace +} + +var Options Configuration diff --git a/src/redis-shake/decode.go b/src/redis-shake/decode.go new file mode 100644 index 0000000..a452caa --- /dev/null +++ b/src/redis-shake/decode.go @@ -0,0 +1,241 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package run + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "time" + + "pkg/libs/atomic2" + "pkg/libs/log" + "pkg/rdb" + "redis-shake/common" + "redis-shake/configure" +) + +type CmdDecode struct { + rbytes, wbytes, nentry atomic2.Int64 +} + +type cmdDecodeStat struct { + rbytes, wbytes, nentry int64 +} + +func (cmd *CmdDecode) Stat() *cmdDecodeStat { + return &cmdDecodeStat{ + rbytes: cmd.rbytes.Get(), + wbytes: cmd.wbytes.Get(), + nentry: cmd.nentry.Get(), + } +} + +func (cmd *CmdDecode) GetDetailedInfo() []interface{} { + return nil +} + +func (cmd *CmdDecode) Main() { + input, output := conf.Options.InputRdb, conf.Options.OutputRdb + if len(input) == 0 { + input = "/dev/stdin" + } + if len(output) == 0 { + output = "/dev/stdout" + } + + log.Infof("decode from '%s' to '%s'\n", input, output) + + var readin io.ReadCloser + var nsize int64 + if input != "/dev/stdin" { + readin, nsize = utils.OpenReadFile(input) + defer readin.Close() + } else { + readin, nsize = os.Stdin, 0 + } + + var saveto io.WriteCloser + if output != "/dev/stdout" { + saveto = utils.OpenWriteFile(output) + defer saveto.Close() + } else { + saveto = os.Stdout + } + + reader := bufio.NewReaderSize(readin, utils.ReaderBufferSize) + writer := bufio.NewWriterSize(saveto, utils.WriterBufferSize) + + ipipe := utils.NewRDBLoader(reader, &cmd.rbytes, int(conf.Options.Parallel) * 32) + opipe := make(chan string, cap(ipipe)) + + go func() { + defer close(opipe) + group := make(chan int, conf.Options.Parallel) + for i := 0; i < cap(group); i++ { + go func() { + defer func() { + group <- 0 + }() + cmd.decoderMain(ipipe, opipe) + }() + } + for i := 0; i < cap(group); i++ { + <-group + } + }() + + wait := make(chan struct{}) + go func() { + defer close(wait) + for s := range opipe { + cmd.wbytes.Add(int64(len(s))) + if _, err := writer.WriteString(s); err != nil { + log.PanicError(err, "write string failed") + } + utils.FlushWriter(writer) + } + }() + + for done := false; !done; { + select { + case <-wait: + done = true + case <-time.After(time.Second): + } + stat := cmd.Stat() + var b bytes.Buffer + fmt.Fprintf(&b, "decode: ") + if nsize != 0 { + fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize) + } else { + fmt.Fprintf(&b, "total = %12d", stat.rbytes) + } + fmt.Fprintf(&b, " write=%-12d", stat.wbytes) + fmt.Fprintf(&b, " entry=%-12d", stat.nentry) + log.Info(b.String()) + } + log.Info("decode: done") +} + +func (cmd *CmdDecode) decoderMain(ipipe <-chan *rdb.BinEntry, opipe chan<- string) { + toText := func(p []byte) string { + var b bytes.Buffer + for _, c := range p { + switch { + case c >= '#' && c <= '~': + b.WriteByte(c) + default: + b.WriteByte('.') + } + } + return b.String() + } + toBase64 := func(p []byte) string { + return base64.StdEncoding.EncodeToString(p) + } + toJson := func(o interface{}) string { + b, err := json.Marshal(o) + if err != nil { + log.PanicError(err, "encode to json failed") + } + return string(b) + } + for e := range ipipe { + o, err := rdb.DecodeDump(e.Value) + if err != nil { + log.PanicError(err, "decode failed") + } + var b bytes.Buffer + switch obj := o.(type) { + default: + log.Panicf("unknown object %v", o) + case rdb.String: + o := &struct { + DB uint32 `json:"db"` + Type string `json:"type"` + ExpireAt uint64 `json:"expireat"` + Key string `json:"key"` + Key64 string `json:"key64"` + Value64 string `json:"value64"` + }{ + e.DB, "string", e.ExpireAt, toText(e.Key), toBase64(e.Key), + toBase64(obj), + } + fmt.Fprintf(&b, "%s\n", toJson(o)) + case rdb.List: + for i, ele := range obj { + o := &struct { + DB uint32 `json:"db"` + Type string `json:"type"` + ExpireAt uint64 `json:"expireat"` + Key string `json:"key"` + Key64 string `json:"key64"` + Index int `json:"index"` + Value64 string `json:"value64"` + }{ + e.DB, "list", e.ExpireAt, toText(e.Key), toBase64(e.Key), + i, toBase64(ele), + } + fmt.Fprintf(&b, "%s\n", toJson(o)) + } + case rdb.Hash: + for _, ele := range obj { + o := &struct { + DB uint32 `json:"db"` + Type string `json:"type"` + ExpireAt uint64 `json:"expireat"` + Key string `json:"key"` + Key64 string `json:"key64"` + Field string `json:"field"` + Field64 string `json:"field64"` + Value64 string `json:"value64"` + }{ + e.DB, "hash", e.ExpireAt, toText(e.Key), toBase64(e.Key), + toText(ele.Field), toBase64(ele.Field), toBase64(ele.Value), + } + fmt.Fprintf(&b, "%s\n", toJson(o)) + } + case rdb.Set: + for _, mem := range obj { + o := &struct { + DB uint32 `json:"db"` + Type string `json:"type"` + ExpireAt uint64 `json:"expireat"` + Key string `json:"key"` + Key64 string `json:"key64"` + Member string `json:"member"` + Member64 string `json:"member64"` + }{ + e.DB, "set", e.ExpireAt, toText(e.Key), toBase64(e.Key), + toText(mem), toBase64(mem), + } + fmt.Fprintf(&b, "%s\n", toJson(o)) + } + case rdb.ZSet: + for _, ele := range obj { + o := &struct { + DB uint32 `json:"db"` + Type string `json:"type"` + ExpireAt uint64 `json:"expireat"` + Key string `json:"key"` + Key64 string `json:"key64"` + Member string `json:"member"` + Member64 string `json:"member64"` + Score float64 `json:"score"` + }{ + e.DB, "zset", e.ExpireAt, toText(e.Key), toBase64(e.Key), + toText(ele.Member), toBase64(ele.Member), ele.Score, + } + fmt.Fprintf(&b, "%s\n", toJson(o)) + } + } + cmd.nentry.Incr() + opipe <- b.String() + } +} diff --git a/src/redis-shake/dump.go b/src/redis-shake/dump.go new file mode 100644 index 0000000..f73b450 --- /dev/null +++ b/src/redis-shake/dump.go @@ -0,0 +1,120 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package run + +import ( + "bufio" + "io" + "net" + "os" + "time" + + "pkg/libs/atomic2" + "pkg/libs/log" + "redis-shake/configure" + "redis-shake/common" +) + +type CmdDump struct { +} + +func (cmd *CmdDump) GetDetailedInfo() []interface{} { + return nil +} + +func (cmd *CmdDump) Main() { + from, output := conf.Options.SourceAddress, conf.Options.OutputRdb + if len(from) == 0 { + log.Panic("invalid argument: from") + } + if len(output) == 0 { + output = "/dev/stdout" + } + + log.Infof("dump from '%s' to '%s'\n", from, output) + + var dumpto io.WriteCloser + if output != "/dev/stdout" { + dumpto = utils.OpenWriteFile(output) + defer dumpto.Close() + } else { + dumpto = os.Stdout + } + + master, nsize := cmd.SendCmd(from, conf.Options.SourceAuthType, conf.Options.SourcePasswordRaw) + defer master.Close() + + log.Infof("rdb file = %d\n", nsize) + + reader := bufio.NewReaderSize(master, utils.ReaderBufferSize) + writer := bufio.NewWriterSize(dumpto, utils.WriterBufferSize) + + cmd.DumpRDBFile(reader, writer, nsize) + + if !conf.Options.ExtraInfo { + return + } + + cmd.DumpCommand(reader, writer, nsize) +} + +func (cmd *CmdDump) SendCmd(master, auth_type, passwd string) (net.Conn, int64) { + c, wait := utils.OpenSyncConn(master, auth_type, passwd) + var nsize int64 + for nsize == 0 { + select { + case nsize = <-wait: + if nsize == 0 { + log.Info("+") + } + case <-time.After(time.Second): + log.Info("-") + } + } + return c, nsize +} + +func (cmd *CmdDump) DumpRDBFile(reader *bufio.Reader, writer *bufio.Writer, nsize int64) { + var nread atomic2.Int64 + wait := make(chan struct{}) + go func() { + defer close(wait) + p := make([]byte, utils.WriterBufferSize) + for nsize != nread.Get() { + nstep := int(nsize - nread.Get()) + ncopy := int64(utils.Iocopy(reader, writer, p, nstep)) + nread.Add(ncopy) + utils.FlushWriter(writer) + } + }() + + for done := false; !done; { + select { + case <-wait: + done = true + case <-time.After(time.Second): + } + n := nread.Get() + p := 100 * n / nsize + log.Infof("total = %d - %12d [%3d%%]\n", nsize, n, p) + } + log.Info("dump: rdb done") +} + +func (cmd *CmdDump) DumpCommand(reader *bufio.Reader, writer *bufio.Writer, nsize int64) { + var nread atomic2.Int64 + go func() { + p := make([]byte, utils.ReaderBufferSize) + for { + ncopy := int64(utils.Iocopy(reader, writer, p, len(p))) + nread.Add(ncopy) + utils.FlushWriter(writer) + } + }() + + for { + time.Sleep(time.Second) + log.Infof("dump: total = %d\n", nsize+nread.Get()) + } +} diff --git a/src/redis-shake/heartbeat/heartbeat.go b/src/redis-shake/heartbeat/heartbeat.go new file mode 100644 index 0000000..ae01a3d --- /dev/null +++ b/src/redis-shake/heartbeat/heartbeat.go @@ -0,0 +1,85 @@ +package heartbeat + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "time" + + "pkg/libs/log" + "redis-shake/configure" + "redis-shake/common" +) + +type HeartbeatController struct { + ServerUrl string + Interval int32 +} + +type HeartbeatData struct { + Id string `json:"id"` + Ip string `json:"ip"` + Port int32 `json:"port"` + Ts int64 `json:"ts"` + Version string `json:"version"` + External string `json:"external"` +} + +type HeartbeatResponse struct { + Error int32 `json:"error"` + Message string `json:"msg"` + Data string `json:"data"` +} + +func (c *HeartbeatController) Start() { + data := &HeartbeatData{ + Id: conf.Options.Id, + Ip: conf.Options.HeartbeatIp, + Port: int32(conf.Options.HttpProfile), + Version: utils.Version, + External: conf.Options.HeartbeatExternal, + } + + ticker := time.NewTicker(time.Second * time.Duration(conf.Options.HeartbeatInterval)) + defer ticker.Stop() + + for range ticker.C { + c.run(data) + } +} + +func (c *HeartbeatController) run(data *HeartbeatData) { + data.Ts = time.Now().UnixNano() / int64(time.Millisecond) + dataStr, _ := json.MarshalIndent(data, "", " ") + + client := http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Post(conf.Options.HeartbeatUrl, "application/json", bytes.NewBuffer(dataStr)) + if err != nil { + // log.PurePrintf("%s\n", NewLogItem("SendHearbeatFail", "WARN", NewErrorLogDetail(conf.Options.HeartbeatUrl, err.Error()))) + log.Warnf("Event:SendHearbeatFail\tId:%s\tURL:%s\tError:%s", conf.Options.Id, conf.Options.HeartbeatUrl, err.Error()) + return + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + // log.PurePrintf("%s\n", NewLogItem("SendHearbeatFail", "WARN", NewErrorLogDetail(conf.Options.HeartbeatUrl, "ReadBodyFail"))) + log.Warnf("Event:SendHearbeatFail\tId:%s\tURL:%s\tReason:ReadBodyFail\tError:%s\t", conf.Options.Id, conf.Options.HeartbeatUrl, err.Error()) + return + } + + hbResp := HeartbeatResponse{} + if err := json.Unmarshal(body, &hbResp); err != nil { + log.Warnf("Event:SendHearbeatFail\tId:%s\tURL:%s\tReason:InvalidResponseBody\tResponse:%s\tError:%s\t", conf.Options.Id, conf.Options.HeartbeatUrl, body, err.Error()) + return + } + + if hbResp.Error != 0 { + log.Warnf("Event:SendHearbeatFail\tId:%s\tURL:%s\tReason:ErrorResponse\tResponse:%s\t", conf.Options.Id, conf.Options.HeartbeatUrl, body) + return + } + log.Infof("Event: SendHearbeatDone\tId:%s\t", conf.Options.Id) +} \ No newline at end of file diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go new file mode 100644 index 0000000..6833f24 --- /dev/null +++ b/src/redis-shake/main/main.go @@ -0,0 +1,317 @@ +// Copyright 2019 Aliyun Cloud. +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package main + +import ( + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "runtime/debug" + "time" + "runtime" + "math" + _ "net/http/pprof" + "strings" + "strconv" + "encoding/json" + "reflect" + + "redis-shake/common" + "redis-shake/configure" + "redis-shake/metric" + "redis-shake" + "redis-shake/base" + "redis-shake/restful" + "pkg/libs/log" + + "github.com/gugemichael/nimo4go" + logRotate "gopkg.in/natefinch/lumberjack.v2" +) + +type Exit struct {Code int} + +const( + TypeDecode = "decode" + TypeRestore = "restore" + TypeDump = "dump" + TypeSync = "sync" +) + +func main() { + var err error + defer handleExit() + defer utils.Goodbye() + + // argument options + configuration := flag.String("conf", "", "configure file absolute path") + tp := flag.String("type", "", "run type: decode, restore, dump, sync") + version := flag.Bool("version", false, "show version") + flag.Parse() + + if *configuration == "" || *version { + fmt.Println(utils.Version) + flag.PrintDefaults() + return + } + + var file *os.File + if file, err = os.Open(*configuration); err != nil { + crash(fmt.Sprintf("Configure file open failed. %v", err), -1) + } + + configure := nimo.NewConfigLoader(file) + configure.SetDateFormat(utils.GolangSecurityTime) + if err := configure.Load(&conf.Options); err != nil { + crash(fmt.Sprintf("Configure file %s parse failed. %v", *configuration, err), -2) + } + + // verify parameters + if err = sanitizeOptions(*tp); err != nil { + crash(fmt.Sprintf("Conf.Options check failed: %s", err.Error()), -4) + } + + initSignal() + initFreeOS() + nimo.Profiling(int(conf.Options.SystemProfile)) + utils.Welcome() + + if err = utils.WritePidById(conf.Options.Id); err != nil { + crash(fmt.Sprintf("write pid failed. %v", err), -5) + } + + // create runner + var runner base.Runner + switch *tp { + case TypeDecode: + runner = new(run.CmdDecode) + case TypeRestore: + runner = new(run.CmdRestore) + case TypeDump: + runner = new(run.CmdDump) + case TypeSync: + runner = new(run.CmdSync) + } + + // create metric + metric.CreateMetric(runner) + go startHttpServer() + + // print configuration + if opts, err := json.Marshal(conf.Options); err != nil { + crash(fmt.Sprintf("marshal configuration failed[%v]", err), -6) + } else { + log.Infof("redis-shake configuration: %s", string(opts)) + } + + // run + runner.Main() + + log.Infof("execute runner[%v] finished!", reflect.TypeOf(runner)) +} + +func initSignal() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigs + log.Info("receive signal: ", sig) + + if utils.LogRotater != nil { + utils.LogRotater.Rotate() + } + + os.Exit(0) + }() +} + +func initFreeOS() { + go func() { + for { + debug.FreeOSMemory() + time.Sleep(5 * time.Second) + } + }() +} + +func startHttpServer() { + utils.InitHttpApi(conf.Options.HttpProfile) + utils.HttpApi.RegisterAPI("/conf", nimo.HttpGet, func([]byte) interface{} { + return &conf.Options + }) + restful.RestAPI() + + if err := utils.HttpApi.Listen(); err != nil { + crash(fmt.Sprintf("start http listen error[%v]", err), -4) + } +} + +// sanitize options +func sanitizeOptions(tp string) error { + var err error + if tp != TypeDecode && tp != TypeRestore && tp != TypeDump && tp != TypeSync { + return fmt.Errorf("unknown type[%v]", tp) + } + + if conf.Options.Id == "" { + return fmt.Errorf("id shoudn't be empty") + } + + if conf.Options.NCpu < 0 || conf.Options.NCpu > 1024 { + return fmt.Errorf("invalid ncpu[%v]", conf.Options.NCpu) + } else if conf.Options.NCpu == 0 { + runtime.GOMAXPROCS(runtime.NumCPU()) + } else { + runtime.GOMAXPROCS(conf.Options.NCpu) + } + + if conf.Options.Parallel == 0 || conf.Options.Parallel > 1024 { + return fmt.Errorf("parallel[%v] should in (0, 1024]", conf.Options.Parallel) + } else { + conf.Options.Parallel = int(math.Max(float64(conf.Options.Parallel), float64(conf.Options.NCpu))) + } + + if conf.Options.BigKeyThreshold > 524288000 { + return fmt.Errorf("BigKeyThreshold[%v] should <= 524288000", conf.Options.BigKeyThreshold) + } + + if (tp == TypeRestore || tp == TypeSync) && conf.Options.TargetAddress == "" { + return fmt.Errorf("target address shouldn't be empty when type in {restore, sync}") + } + if (tp == TypeDump || tp == TypeSync) && conf.Options.SourceAddress == "" { + return fmt.Errorf("source address shouldn't be empty when type in {dump, sync}") + } + + if conf.Options.SourcePasswordRaw != "" && conf.Options.SourcePasswordEncoding != "" { + return fmt.Errorf("only one of source password_raw or password_encoding should be given") + } else if conf.Options.SourcePasswordEncoding != "" { + sourcePassword := "" // todo, inner version + conf.Options.SourcePasswordRaw = string(sourcePassword) + } + + if conf.Options.TargetPasswordRaw != "" && conf.Options.TargetPasswordEncoding != "" { + return fmt.Errorf("only one of target password_raw or password_encoding should be given") + } else if conf.Options.TargetPasswordEncoding != "" { + targetPassword := "" // todo, inner version + conf.Options.TargetPasswordRaw = string(targetPassword) + } + + if conf.Options.LogFile != "" { + conf.Options.LogFile = fmt.Sprintf("%s.log", conf.Options.Id) + + utils.LogRotater = &logRotate.Logger{ + Filename: conf.Options.LogFile, + MaxSize: 100, //MB + MaxBackups: 10, + MaxAge: 0, + } + log.StdLog = log.New(utils.LogRotater, "") + } + + // heartbeat + if conf.Options.HeartbeatInterval <= 0 || conf.Options.HeartbeatInterval > 86400 { + return fmt.Errorf("HeartbeatInterval[%v] should in (0, 86400]", conf.Options.HeartbeatInterval) + } + if conf.Options.HeartbeatNetworkInterface == "" { + conf.Options.HeartbeatIp = "127.0.0.1" + } else { + conf.Options.HeartbeatIp, _, err = utils.GetLocalIp([]string{conf.Options.HeartbeatNetworkInterface}) + if err != nil { + return fmt.Errorf("get ip failed[%v]", err) + } + } + + if conf.Options.FakeTime != "" { + switch conf.Options.FakeTime[0] { + case '-', '+': + if d, err := time.ParseDuration(strings.ToLower(conf.Options.FakeTime)); err != nil { + return fmt.Errorf("parse fake_time failed[%v]", err) + } else { + conf.Options.ShiftTime = d + } + case '@': + if n, err := strconv.ParseInt(conf.Options.FakeTime[1:], 10, 64); err != nil { + return fmt.Errorf("parse fake_time failed[%v]", err) + } else { + conf.Options.ShiftTime = time.Duration(n * int64(time.Millisecond) - time.Now().UnixNano()) + } + default: + if t, err := time.Parse("2006-01-02 15:04:05", conf.Options.FakeTime); err != nil { + return fmt.Errorf("parse fake_time failed[%v]", err) + } else { + conf.Options.ShiftTime = time.Duration(t.UnixNano() - time.Now().UnixNano()) + } + } + } + + if conf.Options.FilterDB != "" { + if n, err := strconv.ParseInt(conf.Options.FilterDB, 10, 32); err != nil { + return fmt.Errorf("parse FilterDB failed[%v]", err) + } else { + base.AcceptDB = func(db uint32) bool { + return db == uint32(n) + } + } + } + + if len(conf.Options.FilterSlot) > 0 { + for i, val := range conf.Options.FilterSlot { + if _, err := strconv.Atoi(val); err != nil { + return fmt.Errorf("parse FilterSlot with index[%v] failed[%v]", i, err) + } + } + } + + if conf.Options.TargetDB >= 0 { + // pass, >= 0 means enable + } + + if conf.Options.HttpProfile <= 0 || conf.Options.HttpProfile > 65535 { + return fmt.Errorf("HttpProfile[%v] should in (0, 65535]", conf.Options.HttpProfile) + } + if conf.Options.SystemProfile <= 0 || conf.Options.SystemProfile > 65535 { + return fmt.Errorf("SystemProfile[%v] should in (0, 65535]", conf.Options.SystemProfile) + } + + if conf.Options.SenderSize <= 0 || conf.Options.SenderSize >= 1073741824 { + return fmt.Errorf("SenderSize[%v] should in (0, 1073741824]", conf.Options.SenderSize) + } + + if conf.Options.SenderCount <= 0 || conf.Options.SenderCount >= 100000 { + return fmt.Errorf("SenderCount[%v] should in (0, 100000]", conf.Options.SenderCount) + } + + if tp == TypeRestore || tp == TypeSync { + // get target redis version and set TargetReplace. + if conf.Options.TargetRedisVersion, err = utils.GetRedisVersion(conf.Options.TargetAddress, + conf.Options.TargetAuthType, conf.Options.TargetPasswordRaw); err != nil { + return fmt.Errorf("get target redis version failed[%v]", err) + } else { + if strings.HasPrefix(conf.Options.TargetRedisVersion, "4.") || + strings.HasPrefix(conf.Options.TargetRedisVersion, "3.") { + conf.Options.TargetReplace = true + } else { + conf.Options.TargetReplace = false + } + } + } + + return nil +} + +func crash(msg string, errCode int) { + fmt.Println(msg) + panic(Exit{errCode}) +} + +func handleExit() { + if e := recover(); e != nil { + if exit, ok := e.(Exit); ok == true { + os.Exit(exit.Code) + } + panic(e) + } +} \ No newline at end of file diff --git a/src/redis-shake/metric/metric.go b/src/redis-shake/metric/metric.go new file mode 100644 index 0000000..5b43f80 --- /dev/null +++ b/src/redis-shake/metric/metric.go @@ -0,0 +1,227 @@ +package metric + +import ( + "time" + "sync/atomic" + "fmt" + "encoding/json" + + "redis-shake/base" + "redis-shake/configure" + "pkg/libs/log" +) + +const ( + updateInterval = 10 // seconds +) + +var ( + MetricVar *Metric + runner base.Runner +) + +type Op interface { + Update() +} + +type Percent struct { + Dividend uint64 + Divisor uint64 +} + +func (p *Percent) Set(dividend, divisor uint64) { + atomic.AddUint64(&p.Dividend, dividend) + atomic.AddUint64(&p.Divisor, divisor) +} + +// input: return string? +func (p *Percent) Get(returnString bool) interface{} { + if divisor := atomic.LoadUint64(&p.Divisor); divisor == 0 { + if returnString { + return "null" + } else { + return int64(^uint64(0) >> 1) // int64_max + } + } else { + dividend := atomic.LoadUint64(&p.Dividend) + if returnString { + return fmt.Sprintf("%.02f", float64(dividend)/float64(divisor)) + } else { + return dividend / divisor + } + } +} + +func (p *Percent) Update() { + p.Dividend = 0 + p.Divisor = 0 +} + +type Delta struct { + Value uint64 // current value +} + +func (d *Delta) Update() { + d.Value = 0 +} + +type Combine struct { + Total uint64 // total number + Delta // delta +} + +func (c *Combine) Set(val uint64) { + atomic.AddUint64(&c.Delta.Value, val) + atomic.AddUint64(&(c.Total), val) +} + +// main struct +type Metric struct { + PullCmdCount Combine + BypassCmdCount Combine + PushCmdCount Combine + SuccessCmdCount Combine + FailCmdCount Combine + + Delay Percent // ms + AvgDelay Percent // ms + NetworkFlow Combine // +speed + + FullSyncProgress uint64 +} + +func CreateMetric(r base.Runner) { + runner = r + MetricVar = &Metric{} + + go MetricVar.run() +} + +func (m *Metric) resetEverySecond(items []Op) { + for _, item := range items { + item.Update() + } +} + +func (m *Metric) run() { + resetItems := []Op{ + &m.PullCmdCount.Delta, + &m.BypassCmdCount.Delta, + &m.PushCmdCount.Delta, + &m.SuccessCmdCount.Delta, + &m.FailCmdCount.Delta, + &m.Delay, + &m.NetworkFlow.Delta, + } + + go func() { + tick := 0 + for range time.NewTicker(1 * time.Second).C { + tick++ + if tick % updateInterval == 0 && conf.Options.MetricPrintLog { + stat := NewMetricRest() + if opts, err := json.Marshal(stat); err != nil { + log.Infof("marshal metric stat error[%v]", err) + } else { + log.Info(string(opts)) + } + } + + m.resetEverySecond(resetItems) + } + }() +} + +func (m *Metric) AddPullCmdCount(val uint64) { + m.PullCmdCount.Set(val) +} + +func (m *Metric) GetPullCmdCount() interface{} { + return atomic.LoadUint64(&m.PullCmdCount.Value) +} + +func (m *Metric) GetPullCmdCountTotal() interface{} { + return atomic.LoadUint64(&m.PullCmdCount.Total) +} + +func (m *Metric) AddBypassCmdCount(val uint64) { + m.BypassCmdCount.Set(val) +} + +func (m *Metric) GetBypassCmdCount() interface{} { + return atomic.LoadUint64(&m.BypassCmdCount.Value) +} + +func (m *Metric) GetBypassCmdCountTotal() interface{} { + return atomic.LoadUint64(&m.BypassCmdCount.Total) +} + +func (m *Metric) AddPushCmdCount(val uint64) { + m.PushCmdCount.Set(val) +} + +func (m *Metric) GetPushCmdCount() interface{} { + return atomic.LoadUint64(&m.PushCmdCount.Value) +} + +func (m *Metric) GetPushCmdCountTotal() interface{} { + return atomic.LoadUint64(&m.PushCmdCount.Total) +} + +func (m *Metric) AddSuccessCmdCount(val uint64) { + m.SuccessCmdCount.Set(val) +} + +func (m *Metric) GetSuccessCmdCount() interface{} { + return atomic.LoadUint64(&m.SuccessCmdCount.Value) +} + +func (m *Metric) GetSuccessCmdCountTotal() interface{} { + return atomic.LoadUint64(&m.SuccessCmdCount.Total) +} + +func (m *Metric) AddFailCmdCount(val uint64) { + m.FailCmdCount.Set(val) +} + +func (m *Metric) GetFailCmdCount() interface{} { + return atomic.LoadUint64(&m.FailCmdCount.Value) +} + +func (m *Metric) GetFailCmdCountTotal() interface{} { + return atomic.LoadUint64(&m.FailCmdCount.Total) +} + +func (m *Metric) AddDelay(val uint64) { + m.Delay.Set(val, 1) + m.AvgDelay.Set(val, 1) +} + +func (m *Metric) GetDelay() interface{} { + return m.Delay.Get(true) +} + +func (m *Metric) GetAvgDelay() interface{} { + return m.AvgDelay.Get(true) +} + +func (m *Metric) AddNetworkFlow(val uint64) { + // atomic.AddUint64(&m.NetworkFlow.Value, val) + m.NetworkFlow.Set(val) +} + +func (m *Metric) GetNetworkFlow() interface{} { + return atomic.LoadUint64(&m.NetworkFlow.Value) +} + +func (m *Metric) GetNetworkFlowTotal() interface{} { + return atomic.LoadUint64(&m.NetworkFlow.Total) +} + +func (m *Metric) SetFullSyncProgress(val uint64) { + m.FullSyncProgress = val +} + +func (m *Metric) GetFullSyncProgress() interface{} { + return m.FullSyncProgress +} diff --git a/src/redis-shake/metric/variables.go b/src/redis-shake/metric/variables.go new file mode 100644 index 0000000..e9969b4 --- /dev/null +++ b/src/redis-shake/metric/variables.go @@ -0,0 +1,63 @@ +package metric + +import( + "fmt" + "redis-shake/base" +) + +type MetricRest struct { + PullCmdCount interface{} + PullCmdCountTotal interface{} + BypassCmdCount interface{} + BypassCmdCountTotal interface{} + PushCmdCount interface{} + PushCmdCountTotal interface{} + SuccessCmdCount interface{} + SuccessCmdCountTotal interface{} + FailCmdCount interface{} + FailCmdCountTotal interface{} + Delay interface{} + AvgDelay interface{} + NetworkSpeed interface{} // network speed + NetworkFlowTotal interface{} // total network speed + FullSyncProgress interface{} + Status interface{} + SenderBufCount interface{} // length of sender buffer + ProcessingCmdCount interface{} // length of delay channel + TargetDBOffset interface{} // target redis offset + SourceDBOffset interface{} // source redis offset +} + +func NewMetricRest() *MetricRest { + detailedInfo := runner.GetDetailedInfo() + if len(detailedInfo) < 4 { + return &MetricRest{} + } + senderBufCount := detailedInfo[0] + processingCmdCount := detailedInfo[1] + targetDbOffset := detailedInfo[2] + sourceDbOffset := detailedInfo[3] + + return &MetricRest{ + PullCmdCount: MetricVar.GetPullCmdCount(), + PullCmdCountTotal: MetricVar.GetPullCmdCountTotal(), + BypassCmdCount: MetricVar.GetBypassCmdCount(), + BypassCmdCountTotal: MetricVar.GetBypassCmdCountTotal(), + PushCmdCount: MetricVar.GetPushCmdCount(), + PushCmdCountTotal: MetricVar.GetPushCmdCountTotal(), + SuccessCmdCount: MetricVar.GetSuccessCmdCount(), + SuccessCmdCountTotal: MetricVar.GetSuccessCmdCountTotal(), + FailCmdCount: MetricVar.GetFailCmdCount(), + FailCmdCountTotal: MetricVar.GetFailCmdCountTotal(), + Delay: fmt.Sprintf("%s ms", MetricVar.GetDelay()), + AvgDelay: fmt.Sprintf("%s ms", MetricVar.GetAvgDelay()), + NetworkSpeed: MetricVar.GetNetworkFlow(), + NetworkFlowTotal: MetricVar.GetNetworkFlowTotal(), + FullSyncProgress: MetricVar.GetFullSyncProgress(), + Status: base.Status, + SenderBufCount: senderBufCount, + ProcessingCmdCount: processingCmdCount, + TargetDBOffset: targetDbOffset, + SourceDBOffset: sourceDbOffset, + } +} \ No newline at end of file diff --git a/src/redis-shake/reader/reader.go b/src/redis-shake/reader/reader.go new file mode 100644 index 0000000..76028f8 --- /dev/null +++ b/src/redis-shake/reader/reader.go @@ -0,0 +1,61 @@ +package reader + +import ( + "bufio" + "fmt" +) + +type ErrorReply string + +var ( + IgnoreReply string = "ignore" +) + +type ReplyReader struct { + reader *bufio.Reader +} + +func NewReplyReader(r *bufio.Reader) *ReplyReader { + return &ReplyReader{r} +} + +func (r *ReplyReader) readLine() ([]byte, error) { + p, err := r.reader.ReadSlice('\n') + + if err == bufio.ErrBufferFull { + return nil, fmt.Errorf("ReaderBufferFull") + } + + if err != nil { + return nil, fmt.Errorf("ReadSliceFail:%s", err.Error()) + } + + pos := len(p) - 2 + if pos < 0 || p[pos] != '\r' { + return nil, fmt.Errorf("BadRedisReply") + } + + return p[:pos], nil +} + +func (r *ReplyReader) ReadNextReply() (interface{}, error) { + for { + line, err := r.readLine() + if err != nil { + return nil, err + } + + if len(line) == 0 { + return nil, fmt.Errorf("EmptyReply") + } + + switch line[0] { + case '-': + return ErrorReply(line[1:]), nil + case '+', ':', '$', '*': + return IgnoreReply, nil + } + continue + } + return "", fmt.Errorf("BUG") +} diff --git a/src/redis-shake/restful/restful.go b/src/redis-shake/restful/restful.go new file mode 100644 index 0000000..06eb06f --- /dev/null +++ b/src/redis-shake/restful/restful.go @@ -0,0 +1,19 @@ +package restful + +import ( + "github.com/gugemichael/nimo4go" + "redis-shake/metric" + "redis-shake/common" +) + +// register all rest api +func RestAPI() { + registerMetric() // register metric + // add below if has more +} + +func registerMetric() { + utils.HttpApi.RegisterAPI("/metric", nimo.HttpGet, func([]byte) interface{} { + return metric.NewMetricRest() + }) +} diff --git a/src/redis-shake/restore.go b/src/redis-shake/restore.go new file mode 100644 index 0000000..3734a40 --- /dev/null +++ b/src/redis-shake/restore.go @@ -0,0 +1,206 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package run + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "time" + + "pkg/libs/atomic2" + "pkg/libs/log" + "pkg/redis" + "redis-shake/configure" + "redis-shake/common" + "strconv" + "redis-shake/base" +) + +type CmdRestore struct { + rbytes, ebytes, nentry, ignore atomic2.Int64 + + forward, nbypass atomic2.Int64 +} + +type cmdRestoreStat struct { + rbytes, ebytes, nentry, ignore int64 + + forward, nbypass int64 +} + +func (cmd *CmdRestore) Stat() *cmdRestoreStat { + return &cmdRestoreStat{ + rbytes: cmd.rbytes.Get(), + ebytes: cmd.ebytes.Get(), + nentry: cmd.nentry.Get(), + ignore: cmd.ignore.Get(), + + forward: cmd.forward.Get(), + nbypass: cmd.nbypass.Get(), + } +} + +func (cmd *CmdRestore) GetDetailedInfo() []interface{} { + return nil +} + +func (cmd *CmdRestore) Main() { + input, target := conf.Options.InputRdb, conf.Options.TargetAddress + if len(target) == 0 { + log.Panic("invalid argument: target") + } + if len(input) == 0 { + input = "/dev/stdin" + } + + log.Infof("restore from '%s' to '%s'\n", input, target) + + base.Status = "waitRestore" + var readin io.ReadCloser + var nsize int64 + if input != "/dev/stdin" { + readin, nsize = utils.OpenReadFile(input) + defer readin.Close() + } else { + readin, nsize = os.Stdin, 0 + } + + base.Status = "restore" + reader := bufio.NewReaderSize(readin, utils.ReaderBufferSize) + + cmd.RestoreRDBFile(reader, target, conf.Options.TargetAuthType, conf.Options.TargetPasswordRaw, nsize) + + base.Status = "extra" + if conf.Options.ExtraInfo && (nsize == 0 || nsize != cmd.rbytes.Get()) { + cmd.RestoreCommand(reader, target, conf.Options.TargetAuthType, conf.Options.TargetPasswordRaw) + } + + if conf.Options.HttpProfile > 0 { + //fake status if set http_port. and wait forever + base.Status = "incr" + log.Infof("Enabled http stats, set status (incr), and wait forever.") + select{} + } +} + +func (cmd *CmdRestore) RestoreRDBFile(reader *bufio.Reader, target, auth_type, passwd string, nsize int64) { + pipe := utils.NewRDBLoader(reader, &cmd.rbytes, conf.Options.Parallel * 32) + wait := make(chan struct{}) + go func() { + defer close(wait) + group := make(chan int, conf.Options.Parallel) + for i := 0; i < cap(group); i++ { + go func() { + defer func() { + group <- 0 + }() + c := utils.OpenRedisConn(target, auth_type, passwd) + defer c.Close() + var lastdb uint32 = 0 + for e := range pipe { + if !base.AcceptDB(e.DB) { + cmd.ignore.Incr() + } else { + cmd.nentry.Incr() + if conf.Options.TargetDB != -1 { + if conf.Options.TargetDB != int(lastdb) { + lastdb = uint32(conf.Options.TargetDB) + utils.SelectDB(c, lastdb) + } + } else { + if e.DB != lastdb { + lastdb = e.DB + utils.SelectDB(c, lastdb) + } + } + utils.RestoreRdbEntry(c, e) + } + } + }() + } + for i := 0; i < cap(group); i++ { + <-group + } + }() + + for done := false; !done; { + select { + case <-wait: + done = true + case <-time.After(time.Second): + } + stat := cmd.Stat() + var b bytes.Buffer + if nsize != 0 { + fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize) + } else { + fmt.Fprintf(&b, "total = %12d", stat.rbytes) + } + fmt.Fprintf(&b, " entry=%-12d", stat.nentry) + if stat.ignore != 0 { + fmt.Fprintf(&b, " ignore=%-12d", stat.ignore) + } + log.Info(b.String()) + } + log.Info("restore: rdb done") +} + +func (cmd *CmdRestore) RestoreCommand(reader *bufio.Reader, target, auth_type, passwd string) { + c := utils.OpenNetConn(target, auth_type, passwd) + defer c.Close() + + writer := bufio.NewWriterSize(c, utils.WriterBufferSize) + defer utils.FlushWriter(writer) + + go func() { + p := make([]byte, utils.ReaderBufferSize) + for { + utils.Iocopy(c, ioutil.Discard, p, len(p)) + } + }() + + go func() { + var bypass bool = false + for { + resp := redis.MustDecode(reader) + if scmd, args, err := redis.ParseArgs(resp); err != nil { + log.PanicError(err, "parse command arguments failed") + } else if scmd != "ping" { + if scmd == "select" { + if len(args) != 1 { + log.Panicf("select command len(args) = %d", len(args)) + } + s := string(args[0]) + n, err := strconv.Atoi(s) + if err != nil { + log.PanicErrorf(err, "parse db = %s failed", s) + } + bypass = !base.AcceptDB(uint32(n)) + } + if bypass { + cmd.nbypass.Incr() + continue + } + } + cmd.forward.Incr() + redis.MustEncode(writer, resp) + utils.FlushWriter(writer) + } + }() + + for lstat := cmd.Stat(); ; { + time.Sleep(time.Second) + nstat := cmd.Stat() + var b bytes.Buffer + fmt.Fprintf(&b, "restore: ") + fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward) + fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass) + log.Info(b.String()) + lstat = nstat + } +} diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go new file mode 100644 index 0000000..b164894 --- /dev/null +++ b/src/redis-shake/sync.go @@ -0,0 +1,590 @@ +// Copyright 2016 CodisLabs. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package run + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" + "time" + + "pkg/libs/atomic2" + "pkg/libs/io/pipe" + "pkg/libs/log" + "pkg/redis" + "redis-shake/common" + "redis-shake/configure" + "redis-shake/command" + "redis-shake/base" + "redis-shake/heartbeat" + "redis-shake/metric" +) + +type delayNode struct { + t time.Time // timestamp + id int64 // id +} + +type CmdSync struct { + rbytes, wbytes, nentry, ignore atomic2.Int64 + + forward, nbypass atomic2.Int64 + + targetOffset atomic2.Int64 + sourceOffset int64 + + /* + * this channel is used to calculate delay between redis-shake and target redis. + * Once oplog sent, the corresponding delayNode push back into this queue. Next time + * receive reply from target redis, the front node poped and then delay calculated. + */ + delayChannel chan *delayNode + + // sending queue + sendBuf chan cmdDetail + + wait_full chan struct{} + + status string +} + +type cmdSyncStat struct { + rbytes, wbytes, nentry, ignore int64 + + forward, nbypass int64 +} + +type cmdDetail struct { + Cmd string + Args [][]byte +} + +func (c *cmdDetail) String() string { + str := c.Cmd + for _, s := range c.Args { + str += " " + string(s) + } + return str +} + +func (cmd *CmdSync) Stat() *cmdSyncStat { + return &cmdSyncStat{ + rbytes: cmd.rbytes.Get(), + wbytes: cmd.wbytes.Get(), + nentry: cmd.nentry.Get(), + ignore: cmd.ignore.Get(), + + forward: cmd.forward.Get(), + nbypass: cmd.nbypass.Get(), + } +} + +// return send buffer length, delay channel length, target db offset +func (cmd *CmdSync) GetDetailedInfo() []interface{} { + + return []interface{}{len(cmd.sendBuf), len(cmd.delayChannel), cmd.targetOffset.Get(), cmd.sourceOffset} +} + +func (cmd *CmdSync) Main() { + from, target := conf.Options.SourceAddress, conf.Options.TargetAddress + if len(from) == 0 { + log.Panic("invalid argument: from") + } + if len(target) == 0 { + log.Panic("invalid argument: target") + } + + log.Infof("sync from '%s' to '%s' http '%d'\n", from, target, conf.Options.HttpProfile) + + cmd.wait_full = make(chan struct{}) + log.Infof("sync from '%s' to '%s'\n", from, target) + + var sockfile *os.File + if len(conf.Options.SockFileName) != 0 { + sockfile = utils.OpenReadWriteFile(conf.Options.SockFileName) + defer sockfile.Close() + } + + base.Status = "waitfull" + var input io.ReadCloser + var nsize int64 + if conf.Options.Psync { + input, nsize = cmd.SendPSyncCmd(from, conf.Options.SourceAuthType, conf.Options.SourcePasswordRaw) + } else { + input, nsize = cmd.SendSyncCmd(from, conf.Options.SourceAuthType, conf.Options.SourcePasswordRaw) + } + defer input.Close() + + log.Infof("rdb file = %d\n", nsize) + + if sockfile != nil { + r, w := pipe.NewFilePipe(int(conf.Options.SockFileSize), sockfile) + defer r.Close() + go func(r io.Reader) { + defer w.Close() + p := make([]byte, utils.ReaderBufferSize) + for { + utils.Iocopy(r, w, p, len(p)) + } + }(input) + input = r + } + + if len(conf.Options.HeartbeatUrl) > 0 { + heartbeatCtl := heartbeat.HeartbeatController{ + ServerUrl: conf.Options.HeartbeatUrl, + Interval: int32(conf.Options.HeartbeatInterval), + } + go heartbeatCtl.Start() + } + + reader := bufio.NewReaderSize(input, utils.ReaderBufferSize) + + base.Status = "full" + cmd.SyncRDBFile(reader, target, conf.Options.TargetAuthType, conf.Options.TargetPasswordRaw, nsize) + + base.Status = "incr" + close(cmd.wait_full) + cmd.SyncCommand(reader, target, conf.Options.TargetAuthType, conf.Options.TargetPasswordRaw) +} + +func (cmd *CmdSync) SendSyncCmd(master, auth_type, passwd string) (net.Conn, int64) { + c, wait := utils.OpenSyncConn(master, auth_type, passwd) + for { + select { + case nsize := <-wait: + if nsize == 0 { + log.Info("+") + } else { + return c, nsize + } + case <-time.After(time.Second): + log.Info("-") + } + } +} + +func (cmd *CmdSync) SendPSyncCmd(master, auth_type, passwd string) (pipe.Reader, int64) { + c := utils.OpenNetConn(master, auth_type, passwd) + utils.SendPSyncListeningPort(c, conf.Options.HttpProfile) + br := bufio.NewReaderSize(c, utils.ReaderBufferSize) + bw := bufio.NewWriterSize(c, utils.WriterBufferSize) + + runid, offset, wait := utils.SendPSyncFullsync(br, bw) + cmd.targetOffset.Set(offset) + log.Infof("psync runid = %s offset = %d, fullsync", runid, offset) + + var nsize int64 + for nsize == 0 { + select { + case nsize = <-wait: + if nsize == 0 { + log.Info("+") + } + case <-time.After(time.Second): + log.Info("-") + } + } + + piper, pipew := pipe.NewSize(utils.ReaderBufferSize) + + go func() { + defer pipew.Close() + p := make([]byte, 8192) + for rdbsize := int(nsize); rdbsize != 0; { + rdbsize -= utils.Iocopy(br, pipew, p, rdbsize) + } + for { + n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew) + if err != nil { + log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset) + } + offset += n + cmd.targetOffset.Set(offset) + for { + // cmd.SyncStat.SetStatus("reopen") + base.Status = "reopen" + time.Sleep(time.Second) + c = utils.OpenNetConnSoft(master, auth_type, passwd) + if c != nil { + // log.PurePrintf("%s\n", NewLogItem("SourceConnReopenSuccess", "INFO", LogDetail{Info: strconv.FormatInt(offset, 10)})) + log.Infof("Event:SourceConnReopenSuccess\tId: %s\toffset = %d", conf.Options.Id, offset) + // cmd.SyncStat.SetStatus("incr") + base.Status = "incr" + break + } else { + // log.PurePrintf("%s\n", NewLogItem("SourceConnReopenFail", "WARN", NewErrorLogDetail("", ""))) + log.Errorf("Event:SourceConnReopenFail\tId: %s", conf.Options.Id) + } + } + utils.AuthPassword(c, auth_type, passwd) + utils.SendPSyncListeningPort(c, conf.Options.HttpProfile) + br = bufio.NewReaderSize(c, utils.ReaderBufferSize) + bw = bufio.NewWriterSize(c, utils.WriterBufferSize) + utils.SendPSyncContinue(br, bw, runid, offset) + } + }() + return piper, nsize +} + +func (cmd *CmdSync) PSyncPipeCopy(c net.Conn, br *bufio.Reader, bw *bufio.Writer, offset int64, copyto io.Writer) (int64, error) { + defer c.Close() + var nread atomic2.Int64 + go func() { + defer c.Close() + for { + time.Sleep(time.Second * 1) + select { + case <-cmd.wait_full: + if err := utils.SendPSyncAck(bw, offset+nread.Get()); err != nil { + return + } + default: + if err := utils.SendPSyncAck(bw, 0); err != nil { + return + } + } + } + }() + + var p = make([]byte, 8192) + for { + n, err := br.Read(p) + if err != nil { + return nread.Get(), nil + } + if _, err := copyto.Write(p[:n]); err != nil { + return nread.Get(), err + } + nread.Add(int64(n)) + } +} + +func (cmd *CmdSync) SyncRDBFile(reader *bufio.Reader, target, auth_type, passwd string, nsize int64) { + pipe := utils.NewRDBLoader(reader, &cmd.rbytes, conf.Options.Parallel * 32) + wait := make(chan struct{}) + go func() { + defer close(wait) + group := make(chan int, conf.Options.Parallel) + for i := 0; i < cap(group); i++ { + go func() { + defer func() { + group <- 0 + }() + c := utils.OpenRedisConn(target, auth_type, passwd) + defer c.Close() + var lastdb uint32 = 0 + for e := range pipe { + if !base.AcceptDB(e.DB) { + cmd.ignore.Incr() + } else { + cmd.nentry.Incr() + if conf.Options.TargetDB != -1 { + if conf.Options.TargetDB != int(lastdb) { + lastdb = uint32(conf.Options.TargetDB) + utils.SelectDB(c, uint32(conf.Options.TargetDB)) + } + } else { + if e.DB != lastdb { + lastdb = e.DB + utils.SelectDB(c, lastdb) + } + } + + if len(conf.Options.FilterKey) != 0 { + for i := 0; i < len(conf.Options.FilterKey); i++ { + if strings.HasPrefix(string(e.Key), conf.Options.FilterKey[i]) { + utils.RestoreRdbEntry(c, e) + break + } + } + } else if len(conf.Options.FilterSlot) > 0 { + for _, slot := range conf.Options.FilterSlot { + slotInt, _ := strconv.Atoi(slot) + if int(utils.KeyToSlot(string(e.Key))) == slotInt { + utils.RestoreRdbEntry(c, e) + break + } + } + } else { + utils.RestoreRdbEntry(c, e) + } + } + } + }() + } + for i := 0; i < cap(group); i++ { + <-group + } + }() + + var stat *cmdSyncStat + + for done := false; !done; { + select { + case <-wait: + done = true + case <-time.After(time.Second): + } + stat = cmd.Stat() + var b bytes.Buffer + fmt.Fprintf(&b, "total=%d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize) + fmt.Fprintf(&b, " entry=%-12d", stat.nentry) + if stat.ignore != 0 { + fmt.Fprintf(&b, " ignore=%-12d", stat.ignore) + } + log.Info(b.String()) + metric.MetricVar.SetFullSyncProgress(uint64(100 * stat.rbytes / nsize)) + } + log.Info("sync rdb done") +} + +func (cmd *CmdSync) SyncCommand(reader *bufio.Reader, target, auth_type, passwd string) { + c := utils.OpenRedisConnWithTimeout(target, auth_type, passwd, time.Duration(10)*time.Minute, time.Duration(10)*time.Minute) + defer c.Close() + + cmd.sendBuf = make(chan cmdDetail, conf.Options.SenderCount) + cmd.delayChannel = make(chan *delayNode, conf.Options.SenderDelayChannelSize) + var sendId, recvId atomic2.Int64 + + go func() { + srcConn := utils.OpenRedisConnWithTimeout(conf.Options.SourceAddress, conf.Options.SourceAuthType, + conf.Options.SourcePasswordRaw, time.Duration(10)*time.Minute, time.Duration(10)*time.Minute) + ticker := time.NewTicker(10 * time.Second) + for range ticker.C { + offset, err := utils.GetFakeSlaveOffset(srcConn) + if err != nil { + // log.PurePrintf("%s\n", NewLogItem("GetFakeSlaveOffsetFail", "WARN", NewErrorLogDetail("", err.Error()))) + log.Warnf("Event:GetFakeSlaveOffsetFail\tId:%s\tError:%s", conf.Options.Id, err.Error()) + + // Reconnect while network error happen + if err == io.EOF { + srcConn = utils.OpenRedisConnWithTimeout(conf.Options.SourceAddress, conf.Options.SourceAuthType, + conf.Options.SourcePasswordRaw, time.Duration(10)*time.Minute, time.Duration(10)*time.Minute) + } else if _, ok := err.(net.Error); ok { + srcConn = utils.OpenRedisConnWithTimeout(conf.Options.SourceAddress, conf.Options.SourceAuthType, + conf.Options.SourcePasswordRaw, time.Duration(10)*time.Minute, time.Duration(10)*time.Minute) + } + } else { + // cmd.SyncStat.SetOffset(offset) + if cmd.sourceOffset, err = strconv.ParseInt(offset, 10, 64); err != nil { + log.Errorf("Event:GetFakeSlaveOffsetFail\tId:%s\tError:%s", conf.Options.Id, err.Error()) + } + } + // cmd.SyncStat.SendBufCount = int64(len(sendBuf)) + // cmd.SyncStat.ProcessingCmdCount = int64(len(cmd.delayChannel)) + //log.Infof("%s", cmd.SyncStat.Roll()) + // cmd.SyncStat.Roll() + // log.PurePrintf("%s\n", NewLogItem("Metric", "INFO", cmd.SyncStat.Snapshot())) + } + }() + + go func() { + var node *delayNode + for { + _, err := c.Receive() + if conf.Options.Metric == false { + continue + } + recvId.Incr() + + if err == nil { + // cmd.SyncStat.SuccessCmdCount.Incr() + metric.MetricVar.AddSuccessCmdCount(1) + } else { + // cmd.SyncStat.FailCmdCount.Incr() + metric.MetricVar.AddFailCmdCount(1) + if utils.CheckHandleNetError(err) { + // log.PurePrintf("%s\n", NewLogItem("NetErrorWhileReceive", "ERROR", NewErrorLogDetail("", err.Error()))) + log.Panicf("Event:NetErrorWhileReceive\tId:%s\tError:%s", conf.Options.Id, err.Error()) + } else { + // log.PurePrintf("%s\n", NewLogItem("ErrorReply", "ERROR", NewErrorLogDetail("", err.Error()))) + log.Panicf("Event:ErrorReply\tId:%s\tCommand: [unknown]\tError: %s", + conf.Options.Id, err.Error()) + } + } + + if node == nil { + // non-blocking read from delay channel + select { + case node = <-cmd.delayChannel: + default: + // it's ok, channel is empty + } + } + + if node != nil && node.id == recvId.Get() { + // cmd.SyncStat.Delay.Add(time.Now().Sub(node.t).Nanoseconds()) + metric.MetricVar.AddDelay(uint64(time.Now().Sub(node.t).Nanoseconds()) / 1000000) // ms + node = nil + } + } + }() + + go func() { + var lastdb int32 = 0 + var bypass bool = false + var isselect bool = false + + var scmd string + var argv, new_argv [][]byte + var err error + + decoder := redis.NewDecoder(reader) + + // log.PurePrintf("%s\n", NewLogItem("IncrSyncStart", "INFO", LogDetail{})) + log.Infof("Event:IncrSyncStart\tId:%s\t", conf.Options.Id) + + for { + ignorecmd := false + isselect = false + resp := redis.MustDecodeOpt(decoder) + + if scmd, argv, err = redis.ParseArgs(resp); err != nil { + log.PanicError(err, "parse command arguments failed") + } else { + // cmd.SyncStat.PullCmdCount.Incr() + metric.MetricVar.AddPullCmdCount(1) + if scmd != "ping" { + if strings.EqualFold(scmd, "select") { + if len(argv) != 1 { + log.Panicf("select command len(args) = %d", len(argv)) + } + s := string(argv[0]) + n, err := strconv.Atoi(s) + if err != nil { + log.PanicErrorf(err, "parse db = %s failed", s) + } + bypass = !base.AcceptDB(uint32(n)) + isselect = true + } else if strings.EqualFold(scmd, "opinfo") { + ignorecmd = true + } + if bypass || ignorecmd { + cmd.nbypass.Incr() + // cmd.SyncStat.BypassCmdCount.Incr() + metric.MetricVar.AddBypassCmdCount(1) + continue + } + } + + is_filter := false + if len(conf.Options.FilterKey) != 0 { + cmd, ok := command.RedisCommands[scmd] + if ok && len(argv) > 0 { + new_argv, is_filter = command.GetMatchKeys(cmd, argv, conf.Options.FilterKey) + } else { + is_filter = true + new_argv = argv + } + } else { + is_filter = true + new_argv = argv + } + if bypass || ignorecmd || !is_filter { + cmd.nbypass.Incr() + continue + } + } + + if isselect && conf.Options.TargetDB != -1 { + if conf.Options.TargetDB != int(lastdb) { + lastdb = int32(conf.Options.TargetDB) + //sendBuf <- cmdDetail{Cmd: scmd, Args: argv, Timestamp: time.Now()} + /* send select command. */ + cmd.sendBuf <- cmdDetail{Cmd: "SELECT", Args: [][]byte{[]byte(strconv.FormatInt(int64(lastdb), 10))}} + } else { + cmd.nbypass.Incr() + // cmd.SyncStat.BypassCmdCount.Incr() + metric.MetricVar.AddBypassCmdCount(1) + } + continue + } + cmd.sendBuf <- cmdDetail{Cmd: scmd, Args: new_argv} + } + }() + + go func() { + var noFlushCount uint + var cachedSize uint64 + + for item := range cmd.sendBuf { + length := len(item.Cmd) + data := make([]interface{}, len(item.Args)) + for i := range item.Args { + data[i] = item.Args[i] + length += len(item.Args[i]) + } + err := c.Send(item.Cmd, data...) + if err != nil { + // log.PurePrintf("%s\n", NewLogItem("SendToTargetFail", "ERROR", NewErrorLogDetail("", err.Error()))) + log.Panicf("Event:SendToTargetFail\tId:%s\tError:%s\t", conf.Options.Id, err.Error()) + } + noFlushCount += 1 + + cmd.forward.Incr() + // cmd.SyncStat.PushCmdCount.Incr() + metric.MetricVar.AddPushCmdCount(1) + // cmd.SyncStat.NetworkFlow.Add(int64(length)) // 发送流量 + metric.MetricVar.AddNetworkFlow(uint64(length)) + sendId.Incr() + + if conf.Options.Metric { + // delay channel + cmd.addDelayChan(sendId.Get()) + } + + if noFlushCount > conf.Options.SenderCount || cachedSize > conf.Options.SenderSize || + len(cmd.sendBuf) == 0 { // 5000 cmd in a batch + err := c.Flush() + noFlushCount = 0 + cachedSize = 0 + if utils.CheckHandleNetError(err) { + // log.PurePrintf("%s\n", NewLogItem("NetErrorWhileFlush", "ERROR", NewErrorLogDetail("", err.Error()))) + log.Panicf("Event:NetErrorWhileFlush\tId:%s\tError:%s\t", conf.Options.Id, err.Error()) + } + } + } + }() + + for lstat := cmd.Stat(); ; { + time.Sleep(time.Second) + nstat := cmd.Stat() + var b bytes.Buffer + fmt.Fprintf(&b, "sync: ") + fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward) + fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass) + fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes) + log.Info(b.String()) + lstat = nstat + } +} + +func (cmd *CmdSync) addDelayChan(id int64) { + // send + /* + * available >=4096: 1:1 sampling + * available >=1024: 1:10 sampling + * available >=128: 1:100 sampling + * else: 1:1000 sampling + */ + used := cap(cmd.delayChannel) - len(cmd.delayChannel) + if used >= 4096 || + used >= 1024 && id % 10 == 0 || + used >= 128 && id % 100 == 0 || + id % 1000 == 0 { + // non-blocking add + select { + case cmd.delayChannel <- &delayNode{t: time.Now(), id: id}: + default: + // do nothing but print when channel is full + log.Warn("delayChannel is full") + } + } +} diff --git a/wandoujia_license.txt b/wandoujia_license.txt new file mode 100644 index 0000000..23320dc --- /dev/null +++ b/wandoujia_license.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Wandoujia Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. From b9ec1d38e9faaae2dfabc92086cc1239000eb2dd Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 16:46:56 +0800 Subject: [PATCH 02/27] update .gitignore --- .gitignore | 36 +++++++++++------------ src/vendor/vendor.json | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 src/vendor/vendor.json diff --git a/.gitignore b/.gitignore index dbc5657..f1146c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,26 @@ -*.log -*.rdb -*.sw[ap] -*.out -*.aof -*.dump -*.log.* -.idea bin +pkg +.gopath +.idea +*.iml +logs +*.pprof +*.output +*.data +*.sw[ap] +*.yml +tags +result.db.* -.DS_Store +dump.data +runtime.trace -/Godeps/_workspace -/cmd/version.go -.htaccess +.DS_Store -tags +data -logs .cache/ diagnostic/ *.pid -src/vendor/ -!vendor.json - -*.tar.gz +src/vendor/* +!src/vendor/vendor.json diff --git a/src/vendor/vendor.json b/src/vendor/vendor.json new file mode 100644 index 0000000..3d15e8e --- /dev/null +++ b/src/vendor/vendor.json @@ -0,0 +1,66 @@ +{ + "comment": "", + "ignore": "test", + "package": [ + { + "checksumSHA1": "1/Kf0ihi6RtfNt0JjAtZLKvfqJY=", + "path": "github.com/cupcake/rdb", + "revision": "43ba34106c765f2111c0dc7b74cdf8ee437411e0", + "revisionTime": "2016-08-25T15:38:14Z" + }, + { + "checksumSHA1": "vAbNAfcRbUYe8mO/t/C2266gvxc=", + "path": "github.com/cupcake/rdb/crc64", + "revision": "43ba34106c765f2111c0dc7b74cdf8ee437411e0", + "revisionTime": "2016-08-25T15:38:14Z" + }, + { + "checksumSHA1": "VbYBp7hfr7vdfliXjcqwrxSsMVg=", + "path": "github.com/docopt/docopt-go", + "revision": "ee0de3bc6815ee19d4a46c7eb90f829db0e014b1", + "revisionTime": "2018-01-11T23:17:33Z" + }, + { + "checksumSHA1": "+T1INwDFb2OfA5AfJlREhKdRZgM=", + "path": "github.com/garyburd/redigo", + "revision": "569eae59ada904ea8db7a225c2b47165af08735a", + "revisionTime": "2018-04-04T16:07:26Z" + }, + { + "checksumSHA1": "2UmMbNHc8FBr98mJFN1k8ISOIHk=", + "path": "github.com/garyburd/redigo/internal", + "revision": "569eae59ada904ea8db7a225c2b47165af08735a", + "revisionTime": "2018-04-04T16:07:26Z" + }, + { + "checksumSHA1": "3kM5bMyfqvdCfgQdOb01CR1aA74=", + "path": "github.com/garyburd/redigo/redis", + "revision": "569eae59ada904ea8db7a225c2b47165af08735a", + "revisionTime": "2018-04-04T16:07:26Z" + }, + { + "checksumSHA1": "63olncsMU/K9MMKxt4zuKSyoNg0=", + "path": "github.com/gugemichael/nimo4go", + "revision": "cbcfac21339d78efaed9ed09dcba7296d9976118", + "revisionTime": "2018-09-04T03:08:18Z" + }, + { + "checksumSHA1": "W7TFJ3aRSUenZvYVnzrOZPoIOjs=", + "path": "github.com/nightlyone/lockfile", + "revision": "0ad87eef1443f64d3d8c50da647e2b1552851124", + "revisionTime": "2018-06-18T18:06:23Z" + }, + { + "checksumSHA1": "Tutue3nEgM/87jitUcYv6ODwyNE=", + "path": "github.com/satori/go.uuid", + "revision": "b2ce2384e17bbe0c6d34077efa39dbab3e09123b", + "revisionTime": "2018-10-28T12:50:25Z" + }, + { + "checksumSHA1": "TM3Neoy1xRAKyZYMGzKc41sDFW4=", + "path": "gopkg.in/natefinch/lumberjack.v2", + "revision": "a96e63847dc3c67d17befa69c303767e2f84e54f", + "revisionTime": "2017-05-31T16:03:50Z" + } + ] +} From 80789dc785fe041c0e08eb94cc23251bb7d5db93 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 16:54:00 +0800 Subject: [PATCH 03/27] polish README --- README.md | 1 + src/redis-shake/main/main.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c64e484..a170507 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ Redis-shake is mainly used to synchronize data from one redis databases to another.
+Thanks to the Douyu's WSD team for the support. 感谢斗鱼公司的web服务部提供的支持。
# Redis-Shake --- diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 6833f24..6f4e37b 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -47,7 +47,7 @@ func main() { defer utils.Goodbye() // argument options - configuration := flag.String("conf", "", "configure file absolute path") + configuration := flag.String("conf", "", "configuration path") tp := flag.String("type", "", "run type: decode, restore, dump, sync") version := flag.Bool("version", false, "show version") flag.Parse() @@ -314,4 +314,4 @@ func handleExit() { } panic(e) } -} \ No newline at end of file +} From 1c8d952d2cf601687de4a70ee98decccfa1eb9ee Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 17:25:19 +0800 Subject: [PATCH 04/27] polish README again --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a170507..0cd9915 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -Redis-shake is mainly used to synchronize data from one redis databases to another.
-Thanks to the Douyu's WSD team for the support. 感谢斗鱼公司的web服务部提供的支持。
+Redis-shake is mainly used to synchronize data from one redis database to another.
+Thanks to the Douyu's WSD team for the support.
# Redis-Shake --- From 78e2774c1acfc7ca78c583795eb52cc4a30298ce Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 17:34:23 +0800 Subject: [PATCH 05/27] add series in README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 0cd9915..9ed4e95 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,12 @@ Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0. * GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies * cd ../../ && ./build.sh * ./bin/collector -type=$(type_must_be_sync,_dump,_restore_or_decode) -conf=conf/redis-shake.conf #please note: user must modify collector.conf first to match needs. + +# Shake series tool +--- +We also provide some tools for synchronization in Shake series.
+ +* [mongo-shake](https://github.com/aliyun/mongo-shake): mongodb data synchronization tool. +* [redis-shake](https://github.com/aliyun/redis-shake): redis data synchronization tool. +* [redis-full-check](https://github.com/aliyun/redis-full-check): redis data synchronization verification tool. + From 739d3927f0ecaf816254723a349248e558a56a65 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 17:41:30 +0800 Subject: [PATCH 06/27] polish README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ed4e95..26715d3 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0. * cd redis-shake/src/vendor * GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies * cd ../../ && ./build.sh -* ./bin/collector -type=$(type_must_be_sync,_dump,_restore_or_decode) -conf=conf/redis-shake.conf #please note: user must modify collector.conf first to match needs. +* ./bin/collector -type=$(type_must_be_sync_dump_restore_or_decode) -conf=conf/redis-shake.conf #please note: user must modify collector.conf first to match needs. # Shake series tool --- From 1c2b04627bb9cb6ab5e65449891b959c51b1ff2d Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 17:48:08 +0800 Subject: [PATCH 07/27] add wechat group --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 26715d3..7661782 100644 --- a/README.md +++ b/README.md @@ -63,3 +63,4 @@ We also provide some tools for synchronization in Shake series.
* [redis-shake](https://github.com/aliyun/redis-shake): redis data synchronization tool. * [redis-full-check](https://github.com/aliyun/redis-full-check): redis data synchronization verification tool. +Plus, we have a WeChat group so that users can join and discuss, but the group user number is limited. So please add my WeChat number: `vinllen_xingge` first, and I will add you to this group.
From 46be70060bc8c486d5339e2b6a913272430a68d1 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 18:02:35 +0800 Subject: [PATCH 08/27] add chinese document --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7661782..441a12a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ Redis-shake is mainly used to synchronize data from one redis database to another.
Thanks to the Douyu's WSD team for the support.
+* [中文文档](https://yq.aliyun.com/articles/691794?spm=a2c4e.11155435.0.0.7c3f3312RCZgel) + # Redis-Shake --- Redis-shake is developed and maintained by NoSQL Team in Alibaba-Cloud Database department.
From d1d32cd4c9bfb76580928b1aa75da359f0267bc7 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 28 Feb 2019 18:20:52 +0800 Subject: [PATCH 09/27] polish --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 441a12a..e0bbd6a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Redis-shake is mainly used to synchronize data from one redis database to another.
Thanks to the Douyu's WSD team for the support.
-* [中文文档](https://yq.aliyun.com/articles/691794?spm=a2c4e.11155435.0.0.7c3f3312RCZgel) +* [中文文档](https://yq.aliyun.com/articles/691794?spm=a2c4e.11155435.0.0.7c3f3312RCZgel): please wait, coming soon. # Redis-Shake --- From a64b9586fb837ef43dfc3ff9a3bcba26adb0f43b Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 4 Mar 2019 11:41:44 +0800 Subject: [PATCH 10/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0bbd6a..be5e5ae 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Version rules: a.b.c. | branch name | rules | | - | :- | | master | master branch, do not allowed push code. store the latest stable version. develop branch will merge into this branch once new version created.| -| develop | develop branch. all the bellowing branches fork from this. | +| **develop**(main branch) | develop branch. all the bellowing branches fork from this. | | feature-\* | new feature branch. forked from develop branch and then merge back after finish developing, testing, and code review. | | bugfix-\* | bugfix branch. forked from develop branch and then merge back after finish developing, testing, and code review. | | improve-\* | improvement branch. forked from develop branch and then merge back after finish developing, testing, and code review. | From 383d3341b1764ae9eb50b8d268a88c32cc6518da Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 4 Mar 2019 21:21:53 +0800 Subject: [PATCH 11/27] polish --- src/redis-shake/sync.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index b164894..636b8ef 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -419,10 +419,15 @@ func (cmd *CmdSync) SyncCommand(reader *bufio.Reader, target, auth_type, passwd } } - if node != nil && node.id == recvId.Get() { - // cmd.SyncStat.Delay.Add(time.Now().Sub(node.t).Nanoseconds()) - metric.MetricVar.AddDelay(uint64(time.Now().Sub(node.t).Nanoseconds()) / 1000000) // ms - node = nil + if node != nil { + id := recvId.Get() // receive id + if node.id == id { + // cmd.SyncStat.Delay.Add(time.Now().Sub(node.t).Nanoseconds()) + metric.MetricVar.AddDelay(uint64(time.Now().Sub(node.t).Nanoseconds()) / 1000000) // ms + node = nil + } else if node.id > id { + log.Panicf("receive id invalid: node-id[%v] > receive-id[%v]", node.id, id) + } } } }() From 2e5233ee5373570091c760a7f1e63c72870f8e80 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Wed, 6 Mar 2019 15:06:19 +0800 Subject: [PATCH 12/27] bugfix: node.id < id --- src/redis-shake/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 636b8ef..5c86495 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -425,8 +425,8 @@ func (cmd *CmdSync) SyncCommand(reader *bufio.Reader, target, auth_type, passwd // cmd.SyncStat.Delay.Add(time.Now().Sub(node.t).Nanoseconds()) metric.MetricVar.AddDelay(uint64(time.Now().Sub(node.t).Nanoseconds()) / 1000000) // ms node = nil - } else if node.id > id { - log.Panicf("receive id invalid: node-id[%v] > receive-id[%v]", node.id, id) + } else if node.id < id { + log.Panicf("receive id invalid: node-id[%v] < receive-id[%v]", node.id, id) } } } From 709c1da12f2b165b7c2346872c3028a180e0c130 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 7 Mar 2019 21:55:30 +0800 Subject: [PATCH 13/27] polish code, conf, readme --- README.md | 2 +- conf/redis-shake.conf | 50 +++++++++++++++++++++++++----------- src/redis-shake/main/main.go | 5 +++- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index be5e5ae..670cd7d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Redis-shake is mainly used to synchronize data from one redis database to another.
Thanks to the Douyu's WSD team for the support.
-* [中文文档](https://yq.aliyun.com/articles/691794?spm=a2c4e.11155435.0.0.7c3f3312RCZgel): please wait, coming soon. +* [中文文档](https://yq.aliyun.com/articles/691794): please wait, coming soon. # Redis-Shake --- diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 92bca58..65f5b6f 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -2,12 +2,12 @@ # id id = redis-shake -# log file -log_file = +# log file，日志文件，不配置将打印到stdout +log_file = # pprof port system_profile = 9310 -# restful port +# restful port，查看metric端口 http_profile = 9320 # runtime.GOMAXPROCS, 0 means use cpu core number: runtime.NumCPU() @@ -18,17 +18,20 @@ parallel = 4 # input RDB file. read from stdin, default is stdin ('/dev/stdin'). # used in `decode` and `restore`. +# 如果是decode或者restore，这个参数表示读取的rdb文件 input_rdb = local_dump # output RDB file. default is stdout ('/dev/stdout'). # used in `decode` and `dump`. +# 如果是decode或者dump，这个参数表示输出的rdb output_rdb = local_dump # source redis configuration. # used in `dump` and `sync`. # ip:port -source.address = 10.101.72.137:20441 -# password. +# 源redis地址 +source.address = 127.0.0.1:20441 +# password. source.password_raw = kLNIl691OZctWST # auth type, don't modify it source.auth_type = auth @@ -38,8 +41,9 @@ source.version = 6 # target redis configuration. used in `restore` and `sync`. # used in `restore` and `sync`. # ip:port +# 目的redis地址 target.address = 10.101.72.137:20551 -# password. +# password. target.password_raw = kLNIl691OZctWST # auth type, don't modify it target.auth_type = auth @@ -50,63 +54,79 @@ target.version = 6 target.db = -1 # use for expire key, set the time gap when source and target timestamp are not the same. +# 用于处理过期的键值，当迁移两端不一致的时候，目的端需要加上这个值 fake_time = # force rewrite when destination restore has the key # used in `restore` and `sync`. +# 当源目的有重复key，是否进行覆写 rewrite = true # filter db or key or slot # choose these db, e.g., 5, only choose db5. defalut is all. # used in `restore` and `sync`. -filter.db = +# 支持过滤db，只让指定的db通过 +filter.db = # filter key with prefix string. multiple keys are separated by ';'. # e.g., a;b;c # default is all. # used in `restore` and `sync`. -filter.key = +# 支持过滤key，只让指定的key通过，分号分隔 +filter.key = # filter given slot, multiple slots are separated by ';'. # e.g., 1;2;3 # used in `sync`. -filter.slot = +# 指定过滤slot，只让指定的slot通过 +filter.slot = -# big key threshold, default is 500 * 1024 * 1024. The field of big key will be split in processing. +# big key threshold, the default is 500 * 1024 * 1024. The field of the big key will be split in processing. +# 我们对大key有特殊的处理，此处需要指定大key的阈值 big_key_threshold = 524288000 # use psync command. # used in `sync`. +# 默认使用sync命令，启用将会使用psync命令 psync = false # enable metric # used in `sync`. +# 是否启用metric metric = true # print in log +# 是否将metric打印到log中 metric.print_log = true -# heartbeat +# heartbeat # send heartbeat to this url # used in `sync`. +# 心跳的url地址，redis-shake将会发送到这个地址 heartbeat.url = http://127.0.0.1:8000 # interval by seconds +# 心跳保活周期 heartbeat.interval = 3 # external info which will be included in heartbeat data. +# 在心跳报文中添加额外的信息 heartbeat.external = test external # local network card to get ip address, e.g., "lo", "eth0", "en0" -heartbeat.network_interface = +# 获取ip的网卡 +heartbeat.network_interface = # sender information. # sender flush buffer size of byte. # used in `sync`. -sender.size = 104857600 +# 发送缓存的字节长度，超过这个阈值将会强行刷缓存发送 +sender.size = 104857600 # sender flush buffer size of oplog number. # used in `sync`. +# 发送缓存的报文个数，超过这个阈值将会强行刷缓存发送 sender.count = 5000 -# delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also stored in this delay queue. this timestamp will be used to calculate the time delay when receive ack from target redis. +# delay channel size. once one oplog is sent to target redis, the oplog id and timestamp will also stored in this delay queue. this timestamp will be used to calculate the time delay when receiving ack from target redis. # used in `sync`. +# 用于metric统计时延的队列 sender.delay_channel_size = 65535 # ----------------splitter---------------- -# belowing variables are useless for current opensource version so don't set. +# below variables are useless for current opensource version so don't set. # replace hash tag. # used in `sync`. diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 6f4e37b..6b2456b 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -52,7 +52,10 @@ func main() { version := flag.Bool("version", false, "show version") flag.Parse() - if *configuration == "" || *version { + if *configuration == "" || *tp == "" || *version { + if !*version { + fmt.Println("Please show me the '-conf' and '-type'") + } fmt.Println(utils.Version) flag.PrintDefaults() return From d83688c693083f68b6512eaab96902efdeeabb43 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Fri, 8 Mar 2019 10:29:32 +0800 Subject: [PATCH 14/27] polish README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 670cd7d..47ce440 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Redis-shake is mainly used to synchronize data from one redis database to another.
Thanks to the Douyu's WSD team for the support.
-* [中文文档](https://yq.aliyun.com/articles/691794): please wait, coming soon. +* [中文文档](https://yq.aliyun.com/articles/691794) # Redis-Shake --- From 5ae097aa5037690b005b611aa18e25eb1c9fd7b8 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Fri, 8 Mar 2019 13:46:09 +0800 Subject: [PATCH 15/27] polish --- README.md | 2 +- build.sh | 2 +- conf/redis-shake.conf | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 47ce440..8b0c473 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0. --- * git clone https://github.com/aliyun/redis-shake.git * cd redis-shake/src/vendor -* GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies +* GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies: `go get -u github.com/kardianos/govendor` * cd ../../ && ./build.sh * ./bin/collector -type=$(type_must_be_sync_dump_restore_or_decode) -conf=conf/redis-shake.conf #please note: user must modify collector.conf first to match needs. diff --git a/build.sh b/build.sh index f20416e..48e9af0 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -o errexit diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 65f5b6f..ed46eb3 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -32,7 +32,7 @@ output_rdb = local_dump # 源redis地址 source.address = 127.0.0.1:20441 # password. -source.password_raw = kLNIl691OZctWST +source.password_raw = 123456 # auth type, don't modify it source.auth_type = auth # version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) @@ -42,9 +42,9 @@ source.version = 6 # used in `restore` and `sync`. # ip:port # 目的redis地址 -target.address = 10.101.72.137:20551 +target.address = 127.0.0.1:20551 # password. -target.password_raw = kLNIl691OZctWST +target.password_raw = 123456 # auth type, don't modify it target.auth_type = auth # version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) @@ -100,7 +100,8 @@ metric.print_log = true # send heartbeat to this url # used in `sync`. # 心跳的url地址，redis-shake将会发送到这个地址 -heartbeat.url = http://127.0.0.1:8000 +#heartbeat.url = http://127.0.0.1:8000 +heartbeat.url = # interval by seconds # 心跳保活周期 heartbeat.interval = 3 From 34f1d2151548a41654b10be7e7fefcb9df4067ab Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 11 Mar 2019 17:46:08 +0800 Subject: [PATCH 16/27] support 5.0 --- src/pkg/rdb/reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index 43cff17..aad9f41 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -16,7 +16,7 @@ import ( // "libs/log" ) -var FromVersion int64 = 8 +var FromVersion int64 = 9 var ToVersion int64 = 6 const ( From 09e64bbf01666010535d66d7d97f0069a6bd4338 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 11 Mar 2019 17:54:02 +0800 Subject: [PATCH 17/27] support redis 5.0 --- ChangeLog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6400947..27036d2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2019-03-11 Alibaba Cloud. + * version: 1.2.0 + * redis-shake: support 5.0. 2019-02-21 Alibaba Cloud. * version: 1.0.0 - * mongo-shake: initial release. + * redis-shake: initial release. From 55cb059e473336e5dbe09e46fd4f4d95a41e9e32 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 14 Mar 2019 17:27:14 +0800 Subject: [PATCH 18/27] support 5.0 stream --- src/pkg/rdb/loader.go | 4 +- src/pkg/rdb/reader.go | 147 +++++++++++++++++++++++++++++--- src/redis-shake/common/utils.go | 14 ++- 3 files changed, 148 insertions(+), 17 deletions(-) diff --git a/src/pkg/rdb/loader.go b/src/pkg/rdb/loader.go index b174463..03997d9 100644 --- a/src/pkg/rdb/loader.go +++ b/src/pkg/rdb/loader.go @@ -154,12 +154,13 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { default: var key []byte if l.remainMember == 0 { + // first time visit this key. rkey, err := l.ReadString() if err != nil { return nil, err } key = rkey - entry.NeedReadLen = 1 + entry.NeedReadLen = 1 // read value length when it's the first time. } else { key = l.lastEntry.Key } @@ -178,6 +179,7 @@ func (l *Loader) NextBinEntry() (*BinEntry, error) { if l.lastReadCount == l.totMemberCount { entry.RealMemberCount = 0 } else { + // RealMemberCount > 0 means this is big entry which also is a split key. entry.RealMemberCount = l.lastReadCount } l.lastEntry = entry diff --git a/src/pkg/rdb/reader.go b/src/pkg/rdb/reader.go index aad9f41..f608b00 100644 --- a/src/pkg/rdb/reader.go +++ b/src/pkg/rdb/reader.go @@ -27,12 +27,13 @@ const ( RdbTypeHash = 4 RdbTypeZSet2 = 5 - RdbTypeHashZipmap = 9 - RdbTypeListZiplist = 10 - RdbTypeSetIntset = 11 - RdbTypeZSetZiplist = 12 - RdbTypeHashZiplist = 13 - RdbTypeQuicklist = 14 + RdbTypeHashZipmap = 9 + RdbTypeListZiplist = 10 + RdbTypeSetIntset = 11 + RdbTypeZSetZiplist = 12 + RdbTypeHashZiplist = 13 + RdbTypeQuicklist = 14 + RDBTypeStreamListPacks = 15 // stream rdbFlagOnlyValue = 0xf9 rdbFlagAUX = 0xfa @@ -46,7 +47,8 @@ const ( const ( rdb6bitLen = 0 rdb14bitLen = 1 - rdb32bitLen = 2 + rdb32bitLen = 0x80 + rdb64bitLen = 0x81 rdbEncVal = 3 rdbEncInt8 = 0 @@ -91,7 +93,7 @@ func (r *rdbReader) offset() int64 { func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { var b bytes.Buffer - r = NewRdbReader(io.TeeReader(r, &b)) + r = NewRdbReader(io.TeeReader(r, &b)) // the result will be written into b when calls r.Read() lr := l.rdbReader switch t { default: @@ -181,6 +183,112 @@ func (r *rdbReader) readObjectValue(t byte, l *Loader) ([]byte, error) { if lr.lastReadCount == n { lr.remainMember = 0 } + case RDBTypeStreamListPacks: + // TODO, need to judge big key + lr.lastReadCount, lr.remainMember, lr.totMemberCount = 0, 0, 0 + // list pack length + nListPacks, err := r.ReadLength() + if err != nil { + return nil, err + } + for i := 0; i < int(nListPacks); i++ { + // read twice + if _, err := r.ReadString(); err != nil { + return nil, err + } + if _, err := r.ReadString(); err != nil { + return nil, err + } + } + + // items + if _, err := r.ReadLength(); err != nil { + return nil, err + } + // last_entry_id timestamp second + if _, err := r.ReadLength(); err != nil { + return nil, err + } + // last_entry_id timestamp millisecond + if _, err := r.ReadLength(); err != nil { + return nil, err + } + + // cgroups length + nCgroups, err := r.ReadLength() + if err != nil { + return nil, err + } + for i := 0; i < int(nCgroups); i++ { + // cname + if _, err := r.ReadString(); err != nil { + return nil, err + } + + // last_cg_entry_id timestamp second + if _, err := r.ReadLength(); err != nil { + return nil, err + } + // last_cg_entry_id timestamp millisecond + if _, err := r.ReadLength(); err != nil { + return nil, err + } + + // pending number + nPending, err := r.ReadLength() + if err != nil { + return nil, err + } + for i := 0; i < int(nPending); i++ { + // eid, read 16 bytes + b := make([]byte, 16) + if err := r.readFull(b); err != nil { + return nil, err + } + + // seen_time + b = make([]byte, 8) + if err := r.readFull(b); err != nil { + return nil, err + } + + // delivery_count + if _, err := r.ReadLength(); err != nil { + return nil, err + } + } + + // consumers + nConsumers, err := r.ReadLength() + if err != nil { + return nil, err + } + for i := 0; i < int(nConsumers); i++ { + // cname + if _, err := r.ReadString(); err != nil { + return nil, err + } + + // seen_time + b := make([]byte, 8) + if err := r.readFull(b); err != nil { + return nil, err + } + + // pending + nPending2, err := r.ReadLength() + if err != nil { + return nil, err + } + for i := 0; i < int(nPending2); i++ { + // eid, read 16 bytes + b := make([]byte, 16) + if err := r.readFull(b); err != nil { + return nil, err + } + } + } + } } return b.Bytes(), nil } @@ -226,16 +334,25 @@ func (r *rdbReader) readEncodedLength() (length uint32, encoded bool, err error) if err != nil { return } - length = uint32(u & 0x3f) switch u >> 6 { case rdb6bitLen: + length = uint32(u & 0x3f) case rdb14bitLen: - u, err = r.readUint8() - length = (length << 8) + uint32(u) + var u2 uint8 + u2, err = r.readUint8() + length = (uint32(u & 0x3f) << 8) + uint32(u2) case rdbEncVal: encoded = true + length = uint32(u & 0x3f) default: - length, err = r.readUint32BigEndian() + switch u { + case rdb32bitLen: + length, err = r.readUint32BigEndian() + case rdb64bitLen: + length, err = r.readUint64BigEndian() + default: + length, err = 0, fmt.Errorf("unknown encoding length[%v]", u) + } } return } @@ -325,6 +442,12 @@ func (r *rdbReader) readUint32BigEndian() (uint32, error) { return binary.BigEndian.Uint32(b), err } +func (r *rdbReader) readUint64BigEndian() (uint32, error) { + b := r.buf[:8] + err := r.readFull(b) + return binary.BigEndian.Uint32(b), err +} + func (r *rdbReader) readInt8() (int8, error) { u, err := r.readUint8() return int8(u), err diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index 98d5bd4..3be3bc9 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -301,15 +301,16 @@ func restoreQuicklistEntry(c redigo.Conn, e *rdb.BinEntry) { if err != nil { log.PanicError(err, "read rdb ") } - //log.Info("restore quicklist key: ", string(e.Key), ", type: ", t) + // log.Info("restore quicklist key: ", string(e.Key), ", type: ", e.Type) count := 0 if n, err := r.ReadLength(); err != nil { log.PanicError(err, "read rdb ") } else { - //log.Info("quicklist item size: ", int(n)) + // log.Info("quicklist item size: ", int(n)) for i := 0; i < int(n); i++ { ziplist, err := r.ReadString() + // log.Info("zipList: ", ziplist) if err != nil { log.PanicError(err, "read rdb ") } @@ -317,12 +318,13 @@ func restoreQuicklistEntry(c redigo.Conn, e *rdb.BinEntry) { if zln, err := r.ReadZiplistLength(buf); err != nil { log.PanicError(err, "read rdb") } else { - //log.Info("ziplist one of quicklist, size: ", int(zln)) + // log.Info("ziplist one of quicklist, size: ", int(zln)) for i := int64(0); i < zln; i++ { entry, err := r.ReadZiplistEntry(buf) if err != nil { log.PanicError(err, "read rdb ") } + // log.Info("rpush key: ", e.Key, " value: ", entry) count++ c.Send("RPUSH", e.Key, entry) if count == 100 { @@ -674,7 +676,9 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { return } - if uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0 { + // TODO, need to judge big key + if e.Type != rdb.RDBTypeStreamListPacks && + (uint64(len(e.Value)) > conf.Options.BigKeyThreshold || e.RealMemberCount != 0) { //use command if conf.Options.Rewrite && e.NeedReadLen == 1 { if !conf.Options.Metric { @@ -694,6 +698,8 @@ func RestoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { } return } + + // fmt.Printf("kkkey: %v, value: %v\n", string(e.Key), e.Value) s, err := redigo.String(c.Do("restore", e.Key, ttlms, e.Value)) if err != nil { /*The reply value of busykey in 2.8 kernel is "target key name is busy", From 92ae3c48298dca8a55fab5e425571f98e7c6a82f Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 14 Mar 2019 17:28:42 +0800 Subject: [PATCH 19/27] modify conf --- conf/redis-shake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index ed46eb3..1d5e3c8 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -94,7 +94,7 @@ psync = false metric = true # print in log # 是否将metric打印到log中 -metric.print_log = true +metric.print_log = false # heartbeat # send heartbeat to this url From c42be8380add4d5e85a6cb6d2eb8ad12ea9313dc Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Thu, 14 Mar 2019 17:41:22 +0800 Subject: [PATCH 20/27] modify conf2 --- conf/redis-shake.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 1d5e3c8..7a0eb2b 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -19,7 +19,7 @@ parallel = 4 # input RDB file. read from stdin, default is stdin ('/dev/stdin'). # used in `decode` and `restore`. # 如果是decode或者restore，这个参数表示读取的rdb文件 -input_rdb = local_dump +input_rdb = local # output RDB file. default is stdout ('/dev/stdout'). # used in `decode` and `dump`. @@ -35,8 +35,8 @@ source.address = 127.0.0.1:20441 source.password_raw = 123456 # auth type, don't modify it source.auth_type = auth -# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) -source.version = 6 +# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0, 9 for >= 5.0) +source.version = 9 # target redis configuration. used in `restore` and `sync`. # used in `restore` and `sync`. @@ -47,7 +47,7 @@ target.address = 127.0.0.1:20551 target.password_raw = 123456 # auth type, don't modify it target.auth_type = auth -# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0) +# version number, default is 6 (6 for Redis Version <= 3.0.7, 7 for >=3.2.0, 9 for >= 5.0) target.version = 6 # all the data will come into this db. < 0 means disable. # used in `restore` and `sync`. From 2e614e081ccb89478ef94f6735aa3782a0ca6986 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Fri, 15 Mar 2019 18:04:31 +0800 Subject: [PATCH 21/27] support tcp keepalive --- conf/redis-shake.conf | 6 ++++++ src/redis-shake/common/utils.go | 5 ++++- src/redis-shake/configure/configure.go | 1 + src/redis-shake/sync.go | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/conf/redis-shake.conf b/conf/redis-shake.conf index 7a0eb2b..fff9078 100644 --- a/conf/redis-shake.conf +++ b/conf/redis-shake.conf @@ -126,6 +126,12 @@ sender.count = 5000 # 用于metric统计时延的队列 sender.delay_channel_size = 65535 +# enable keep_alive option in TCP when connecting redis. +# the unit is second. +# 0 means disable. +# TCP keep-alive保活参数，单位秒，0表示不启用。 +keep_alive = 0 + # ----------------splitter---------------- # below variables are useless for current opensource version so don't set. diff --git a/src/redis-shake/common/utils.go b/src/redis-shake/common/utils.go index 3be3bc9..483a6ef 100644 --- a/src/redis-shake/common/utils.go +++ b/src/redis-shake/common/utils.go @@ -34,7 +34,10 @@ func OpenRedisConnWithTimeout(target, auth_type, passwd string, readTimeout, wri } func OpenNetConn(target, auth_type, passwd string) net.Conn { - c, err := net.Dial("tcp", target) + d := net.Dialer{ + KeepAlive: time.Duration(conf.Options.KeepAlive) * time.Second, + } + c, err := d.Dial("tcp", target) if err != nil { log.PanicErrorf(err, "cannot connect to '%s'", target) } diff --git a/src/redis-shake/configure/configure.go b/src/redis-shake/configure/configure.go index ec16e8d..44437ad 100644 --- a/src/redis-shake/configure/configure.go +++ b/src/redis-shake/configure/configure.go @@ -39,6 +39,7 @@ type Configuration struct { SenderSize uint64 `config:"sender.size"` SenderCount uint `config:"sender.count"` SenderDelayChannelSize uint `config:"sender.delay_channel_size"` + KeepAlive uint `config:"keep_alive"` // inner variables ReplaceHashTag bool `config:"replace_hash_tag"` diff --git a/src/redis-shake/sync.go b/src/redis-shake/sync.go index 5c86495..6521d90 100644 --- a/src/redis-shake/sync.go +++ b/src/redis-shake/sync.go @@ -361,7 +361,7 @@ func (cmd *CmdSync) SyncCommand(reader *bufio.Reader, target, auth_type, passwd offset, err := utils.GetFakeSlaveOffset(srcConn) if err != nil { // log.PurePrintf("%s\n", NewLogItem("GetFakeSlaveOffsetFail", "WARN", NewErrorLogDetail("", err.Error()))) - log.Warnf("Event:GetFakeSlaveOffsetFail\tId:%s\tError:%s", conf.Options.Id, err.Error()) + log.Warnf("Event:GetFakeSlaveOffsetFail\tId:%s\tWarn:%s", conf.Options.Id, err.Error()) // Reconnect while network error happen if err == io.EOF { From db16f2272eb34bb2041299c7c78e654d2bb1a6ed Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 18 Mar 2019 11:10:13 +0800 Subject: [PATCH 22/27] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8b0c473..2e9acdf 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,9 @@ Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0. --- We also provide some tools for synchronization in Shake series.
-* [mongo-shake](https://github.com/aliyun/mongo-shake): mongodb data synchronization tool. -* [redis-shake](https://github.com/aliyun/redis-shake): redis data synchronization tool. -* [redis-full-check](https://github.com/aliyun/redis-full-check): redis data synchronization verification tool. +* [MongoShake](https://github.com/aliyun/MongoShake): mongodb data synchronization tool. +* [RedisShake](https://github.com/aliyun/RedisShake): redis data synchronization tool. +* [RedisFullCheck](https://github.com/aliyun/RedisFullCheck): redis data synchronization verification tool. Plus, we have a WeChat group so that users can join and discuss, but the group user number is limited. So please add my WeChat number: `vinllen_xingge` first, and I will add you to this group.
+ From ae506fe7f6925f6838bcbf837dbfeb68d3b025e4 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Mon, 18 Mar 2019 19:18:29 +0800 Subject: [PATCH 23/27] add binary in the bin directory --- .gitignore | 3 ++- README.md | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f1146c2..16b099d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -bin pkg .gopath .idea @@ -9,6 +8,8 @@ logs *.data *.sw[ap] *.yml +*.pid +*.tar.gz tags result.db.* diff --git a/README.md b/README.md index 8b0c473..00e731c 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Add tag when releasing: "release-v{version}-{date}". for example: "release-v1.0. # Usage --- +Run `./bin/redis-shake.darwin64` or `redis-shake.linux64` which is built in OSX and Linux respectively.
+Or you can build redis-shake yourself according to the following steps: * git clone https://github.com/aliyun/redis-shake.git * cd redis-shake/src/vendor * GOPATH=\`pwd\`/../..; govendor sync #please note: must install govendor first and then pull all dependencies: `go get -u github.com/kardianos/govendor` From 72655adb4289f2dfffaea224736f8639fe03c922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=83=9B=E6=98=AD?= Date: Tue, 19 Mar 2019 16:34:59 +0800 Subject: [PATCH 24/27] move lib --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8cc268..9432895 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Please check out the `conf/redis-shake.conf` to see the detailed parameters desc # Verification --- -User can use [redis-full-check](https://github.com/aliyun/redis-full-check) to verify correctness.
+User can use [redis-full-check](https://github.com/alibaba/RedisFullCheck) to verify correctness.
# Metric --- From 03e065d8757074920bd3d4adde06df3e5949d70c Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Tue, 19 Mar 2019 17:27:51 +0800 Subject: [PATCH 25/27] test two branches --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9432895..98b1656 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Please check out the `conf/redis-shake.conf` to see the detailed parameters desc # Verification --- -User can use [redis-full-check](https://github.com/alibaba/RedisFullCheck) to verify correctness.
+User can use [RedisFullCheck](https://github.com/alibaba/RedisFullCheck) to verify correctness.
# Metric --- From a7a0b1357536308e76a562c338634b54df691237 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Tue, 19 Mar 2019 17:38:14 +0800 Subject: [PATCH 26/27] test two branches --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98b1656..abf789d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Redis-shake is mainly used to synchronize data from one redis database to another.
+RedisShake is mainly used to synchronize data from one redis database to another.
Thanks to the Douyu's WSD team for the support.
* [中文文档](https://yq.aliyun.com/articles/691794) From 46e6ccd3eb4de6c5cebed527d7097463c5a89810 Mon Sep 17 00:00:00 2001 From: "zhuzhao.cx" Date: Tue, 19 Mar 2019 21:27:45 +0800 Subject: [PATCH 27/27] add start time metric --- src/redis-shake/common/common.go | 5 +++-- src/redis-shake/main/main.go | 1 + src/redis-shake/metric/variables.go | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/redis-shake/common/common.go b/src/redis-shake/common/common.go index 7fed11f..6bfd69f 100644 --- a/src/redis-shake/common/common.go +++ b/src/redis-shake/common/common.go @@ -6,8 +6,8 @@ import( ) const( - // GolangSecurityTime = "2006-01-02T15:04:05Z" - GolangSecurityTime = "2006-01-02 15:04:05" + GolangSecurityTime = "2006-01-02T15:04:05Z" + // GolangSecurityTime = "2006-01-02 15:04:05" ReaderBufferSize = bytesize.MB * 32 WriterBufferSize = bytesize.MB * 8 ) @@ -15,4 +15,5 @@ const( var( Version = "$" LogRotater *logRotate.Logger + StartTime string ) \ No newline at end of file diff --git a/src/redis-shake/main/main.go b/src/redis-shake/main/main.go index 6b2456b..373584f 100644 --- a/src/redis-shake/main/main.go +++ b/src/redis-shake/main/main.go @@ -81,6 +81,7 @@ func main() { initFreeOS() nimo.Profiling(int(conf.Options.SystemProfile)) utils.Welcome() + utils.StartTime = fmt.Sprintf("%v", time.Now().Format(utils.GolangSecurityTime)) if err = utils.WritePidById(conf.Options.Id); err != nil { crash(fmt.Sprintf("write pid failed. %v", err), -5) diff --git a/src/redis-shake/metric/variables.go b/src/redis-shake/metric/variables.go index e9969b4..619e7d3 100644 --- a/src/redis-shake/metric/variables.go +++ b/src/redis-shake/metric/variables.go @@ -3,9 +3,11 @@ package metric import( "fmt" "redis-shake/base" + "redis-shake/common" ) type MetricRest struct { + StartTime interface{} PullCmdCount interface{} PullCmdCountTotal interface{} BypassCmdCount interface{} @@ -39,6 +41,7 @@ func NewMetricRest() *MetricRest { sourceDbOffset := detailedInfo[3] return &MetricRest{ + StartTime: utils.StartTime, PullCmdCount: MetricVar.GetPullCmdCount(), PullCmdCountTotal: MetricVar.GetPullCmdCountTotal(), BypassCmdCount: MetricVar.GetBypassCmdCount(),