suxb201
1 year ago
committed by
suxb201
parent
4262d728bd
commit
acd3e2eed0
19 changed files with 895 additions and 505 deletions
@ -1,33 +1,63 @@ |
||||
# Sample workflow for building and deploying a VitePress site to GitHub Pages |
||||
# |
||||
name: Pages |
||||
|
||||
on: |
||||
workflow_dispatch: { } |
||||
# Runs on pushes targeting the `main` branch. Change this to `master` if you're |
||||
# using the `master` branch as the default branch. |
||||
push: |
||||
branches: |
||||
- main |
||||
branches: [ v4 ] |
||||
|
||||
# Allows you to run this workflow manually from the Actions tab |
||||
workflow_dispatch: |
||||
|
||||
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages |
||||
permissions: |
||||
contents: read |
||||
pages: write |
||||
id-token: write |
||||
|
||||
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. |
||||
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. |
||||
concurrency: |
||||
group: pages |
||||
cancel-in-progress: false |
||||
|
||||
jobs: |
||||
deploy: |
||||
# Build job |
||||
build: |
||||
runs-on: ubuntu-latest |
||||
permissions: |
||||
pages: write |
||||
id-token: write |
||||
environment: |
||||
name: github-pages |
||||
url: ${{ steps.deployment.outputs.page_url }} |
||||
steps: |
||||
- uses: actions/checkout@v3 |
||||
with: |
||||
fetch-depth: 0 |
||||
- uses: actions/setup-node@v3 |
||||
- name: Checkout |
||||
uses: actions/checkout@v3 |
||||
- name: Setup Node |
||||
uses: actions/setup-node@v3 |
||||
with: |
||||
node-version: 16 |
||||
node-version: 18 |
||||
cache: npm |
||||
- run: npm ci |
||||
- name: Build |
||||
run: npm run docs:build |
||||
- uses: actions/configure-pages@v2 |
||||
- uses: actions/upload-pages-artifact@v1 |
||||
cache-dependency-path: docs/package-lock.json |
||||
- name: Setup Pages |
||||
uses: actions/configure-pages@v3 |
||||
- name: Install dependencies |
||||
run: npm ci # or pnpm install / yarn install |
||||
working-directory: docs |
||||
- name: Build with VitePress |
||||
run: npm run docs:build # or pnpm docs:build / yarn docs:build |
||||
working-directory: docs |
||||
- name: Upload artifact |
||||
uses: actions/upload-pages-artifact@v2 |
||||
with: |
||||
path: docs/.vitepress/dist |
||||
- name: Deploy |
||||
|
||||
# Deployment job |
||||
deploy: |
||||
environment: |
||||
name: github-pages |
||||
url: ${{ steps.deployment.outputs.page_url }} |
||||
needs: build |
||||
runs-on: ubuntu-latest |
||||
name: Deploy |
||||
steps: |
||||
- name: Deploy to GitHub Pages |
||||
id: deployment |
||||
uses: actions/deploy-pages@v1 |
||||
uses: actions/deploy-pages@v2 |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,25 @@ |
||||
--- |
||||
# https://vitepress.dev/reference/default-theme-home-page |
||||
layout: home |
||||
|
||||
hero: |
||||
name: "RedisShake" |
||||
# text: "用于 Redis-like 数据库的数据迁移与处理服务" |
||||
tagline: 用于 Redis-like 数据库的数据迁移与处理服务 |
||||
actions: |
||||
- theme: brand |
||||
text: 快速上手 |
||||
link: /zh/guide/getting-started |
||||
# - theme: alt |
||||
# text: 云原生内存数据库Tair |
||||
# link: https://www.aliyun.com/product/apsaradb/kvstore/tair |
||||
|
||||
features: |
||||
- title: 数据迁移 |
||||
details: 支持 sync、scan 和 restore 三种数据迁移模式 |
||||
- title: 数据处理 |
||||
details: 支持使用 lua 脚本对数据进行过滤与修改 |
||||
- title: 云数据库支持 |
||||
details: 兼容主流云厂商的多种架构：主从、集群等 |
||||
--- |
||||
|
||||
@ -0,0 +1,155 @@ |
||||
--- |
||||
outline: deep |
||||
--- |
||||
# 配置文件 |
||||
|
||||
RedisShake 使用 [TOML](https://toml.io/cn/) 语言书写，所有的配置参数在 all.toml 中均有说明。 |
||||
|
||||
配置文件的组成如下： |
||||
|
||||
```toml |
||||
transform = "..." |
||||
|
||||
[xxx_reader] |
||||
... |
||||
|
||||
[xxx_writer] |
||||
... |
||||
|
||||
[advanced] |
||||
... |
||||
``` |
||||
|
||||
一般用法下，只需要书写 `xxx_reader`、`xxx_writer` 两个部分即可，`transform` 和 `advanced` 部分为进阶用法，用户可以根据自己的需求进行配置。 |
||||
|
||||
## reader 配置 |
||||
|
||||
根据源端的类型，RedisShake 提供了不同的 Reader 配置，用来对接不同的源端。 |
||||
|
||||
* 对于支持 [Redis Sync/Psync 协议](https://redis.io/docs/management/replication/)的源端，推荐使用 `sync_xxx_reader` |
||||
* 对于不支持 [Redis Sync/Psync 协议](https://redis.io/docs/management/replication/)的源端，可以使用 `scan_xxx_reader` |
||||
* 对于使用 dump.rdb 文件恢复数据场景，可以使用 `rdb_reader` |
||||
|
||||
### sync_xxx_reader |
||||
|
||||
对于源端为单机 Redis-like 数据库时，使用 `sync_standalone_reader`；对于源端为 Redis Cluster 时，使用 `sync_cluster_reader`。 |
||||
|
||||
#### sync_standlone_reader |
||||
|
||||
```toml |
||||
[sync_standlone_reader] |
||||
address = "127.0.0.1:6379" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
#### sync_cluster_reader |
||||
|
||||
```toml |
||||
[sync_cluster_reader] |
||||
address = "127.0.0.1:6379" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
### scan_xxx_reader |
||||
|
||||
对于源端为单机 Redis-like 数据库时，使用 `scan_standalone_reader`；对于源端为 Redis Cluster 时，使用 `scan_cluster_reader`。 |
||||
|
||||
#### scan_standlone_reader |
||||
|
||||
```toml |
||||
[scan_standlone_reader] |
||||
address = "127.0.0.1:6379" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
#### scan_cluster_reader |
||||
|
||||
```toml |
||||
[scan_cluster_reader] |
||||
address = "127.0.0.1:6379" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
### rdb_reader |
||||
|
||||
```toml |
||||
[rdb_reader] |
||||
filepath = "/path/to/dump.rdb" |
||||
``` |
||||
|
||||
filepath 为 dump.rdb 文件的路径，最好使用绝对路径。 |
||||
|
||||
## writer 配置 |
||||
|
||||
根据目标端的类型，RedisShake 提供了不同的 Writer 配置，用来对接不同的目标端。 |
||||
目前 RedisShake 支持的目标端有： |
||||
* 单机 Redis-like 数据库：redis_standalone_writer |
||||
* Redis Cluster：redis_cluster_writer |
||||
|
||||
### redis_standalone_writer |
||||
|
||||
```toml |
||||
[redis_standalone_writer] |
||||
address = "127.0.0.1:6380" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
### redis_cluster_writer |
||||
|
||||
```toml |
||||
[redis_cluster_writer] |
||||
address = "127.0.0.1:6380" |
||||
username = "" # keep empty if not using ACL |
||||
password = "" # keep empty if no authentication is required |
||||
tls = false |
||||
``` |
||||
|
||||
## advanced 配置 |
||||
|
||||
```toml |
||||
[advanced] |
||||
dir = "data" |
||||
ncpu = 3 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores |
||||
|
||||
pprof_port = 0 # pprof port, 0 means disable |
||||
status_port = 0 # status port, 0 means disable |
||||
|
||||
# log |
||||
log_file = "shake.log" |
||||
log_level = "info" # debug, info or warn |
||||
log_interval = 5 # in seconds |
||||
|
||||
# redis-shake gets key and value from rdb file, and uses RESTORE command to |
||||
# create the key in target redis. Redis RESTORE will return a "Target key name |
||||
# is busy" error when key already exists. You can use this configuration item |
||||
# to change the default behavior of restore: |
||||
# panic: redis-shake will stop when meet "Target key name is busy" error. |
||||
# rewrite: redis-shake will replace the key with new value. |
||||
# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. |
||||
rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip |
||||
|
||||
# redis-shake uses pipeline to improve sending performance. |
||||
# This item limits the maximum number of commands in a pipeline. |
||||
pipeline_count_limit = 1024 |
||||
|
||||
# Client query buffers accumulate new commands. They are limited to a fixed |
||||
# amount by default. This amount is normally 1gb. |
||||
target_redis_client_max_querybuf_len = 1024_000_000 |
||||
|
||||
# In the Redis protocol, bulk requests, that are, elements representing single |
||||
# strings, are normally limited to 512 mb. |
||||
target_redis_proto_max_bulk_len = 512_000_000 |
||||
|
||||
# If the source is Elasticache or MemoryDB, you can set this item. |
||||
aws_psync = "" |
||||
``` |
||||
@ -0,0 +1,48 @@ |
||||
--- |
||||
outline: deep |
||||
--- |
||||
# 快速上手 |
||||
|
||||
## 安装 |
||||
|
||||
### 下载二进制包 |
||||
|
||||
直接在此处下载：https://github.com/tair-opensource/RedisShake/releases |
||||
|
||||
### 从源码编译 |
||||
|
||||
确保本地有 Golang 环境： |
||||
|
||||
```shell |
||||
git clone https://github.com/alibaba/RedisShake |
||||
cd RedisShake |
||||
sh build.sh |
||||
``` |
||||
|
||||
## 运行 |
||||
|
||||
假设现在有两个 Redis 实例： |
||||
|
||||
* 实例 A：127.0.0.1:6379 |
||||
* 实例 B：127.0.0.1:6380 |
||||
|
||||
新建文件 `shake.toml`： |
||||
|
||||
```toml |
||||
[sync_standalone_reader] |
||||
address = "127.0.0.1:6379" |
||||
|
||||
[redis_standalone_writer] |
||||
address = "127.0.0.1:6380" |
||||
``` |
||||
|
||||
启动 redis-shake： |
||||
|
||||
```shell |
||||
./redis-shake shake.toml |
||||
|
||||
``` |
||||
|
||||
运行日志如图： |
||||
|
||||
 |
||||
|
After Width: | Height: | Size: 1.5 MiB |
@ -0,0 +1,65 @@ |
||||
--- |
||||
outline: deep |
||||
--- |
||||
# transform 样例 |
||||
|
||||
## 阿里云兼容 |
||||
|
||||
```lua |
||||
-- Aliyun Redis 4.0: skip OPINFO command |
||||
function transform(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) |
||||
if cmd_name == "OPINFO" then |
||||
return 1, db_id -- disallow |
||||
else |
||||
return 0, db_id -- allow |
||||
end |
||||
end |
||||
``` |
||||
|
||||
## AWS 兼容 |
||||
|
||||
```lua |
||||
-- ElastiCache: skip REPLCONF command |
||||
function transform(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) |
||||
if cmd_name == "REPLCONF" then |
||||
return 1, db_id -- disallow |
||||
else |
||||
return 0, db_id -- allow |
||||
end |
||||
end |
||||
``` |
||||
|
||||
## 过滤命令 |
||||
|
||||
### 过滤所有 lua 脚本 |
||||
|
||||
``` |
||||
-- skip all scripts included LUA scripts and Redis Functions. |
||||
function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) |
||||
if group == "SCRIPTING" then |
||||
return 1, db_id -- disallow |
||||
else |
||||
return 0, db_id -- allow |
||||
end |
||||
end |
||||
``` |
||||
|
||||
## key 操作 |
||||
|
||||
### 按照前缀过滤 key |
||||
|
||||
``` |
||||
-- skip keys prefixed with ABC |
||||
function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) |
||||
if #keys ~= 1 then |
||||
return 0, db_id -- allow |
||||
end |
||||
|
||||
if string.sub(keys[1], 0, 3) == "ABC" then |
||||
return 1, db_id -- disallow |
||||
end |
||||
|
||||
return 0, db_id -- allow |
||||
end |
||||
``` |
||||
|
||||
@ -0,0 +1,6 @@ |
||||
--- |
||||
outline: deep |
||||
--- |
||||
# 上手使用 |
||||
|
||||
TODO |
||||
Loading…
Reference in new issue