From 886dce129e985909a172b363f2f2e5b8e9894559 Mon Sep 17 00:00:00 2001 From: chenby Date: Sun, 19 Aug 2018 22:32:11 +0800 Subject: [PATCH] 0.1.10 --- CHANGELOG.md | 4 + README.md | 795 ++++++++++++++++++++++++------------------------ README.zh_CN.md | 786 ++++++++++++++++++++++++----------------------- 3 files changed, 803 insertions(+), 782 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4c155a..6277eafc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### 0.1.10 + +`rmt` support cluster migration. + ### 0.1.9 Add `metric` feature. diff --git a/README.md b/README.md index fec4eab1..11247b10 100644 --- a/README.md +++ b/README.md @@ -1,395 +1,400 @@ -# redis-rdb-cli - -## A tool that can parse, filter, split, merge rdb and analyze memory usage offline. - -## Binary release - -[binary releases](https://github.com/leonchen83/redis-rdb-cli/releases) - -## Runtime requirement - -```java -jdk 1.8+ -``` - -## Install - -```java -mkdir redis-rdb-cli -cd ./redis-rdb-cli -wget https://github.com/leonchen83/redis-rdb-cli/releases/download/${version}/redis-rdb-cli.zip -unzip redis-rdb-cli.zip -sudo chmod -R 755 ./ -cd ./bin -./rct -h -``` - -## Compile requirement - -```java - -jdk 1.8+ -maven-3.3.1+ - -``` - -## Compile & run - -```java - git clone https://github.com/leonchen83/redis-rdb-cli.git - cd redis-rdb-cli - mvn clean install -Dmaven.test.skip=true - cd target/redis-rdb-cli/bin - ./rct -h -``` - -## Windows Environment Variables - -Add `/path/to/redis-rdb-cli/bin` to `Path` environment variable - -## Usage - -```java - -usage: rct -f -s -o [-d ] [-e - ] [-k ] [-t ] [-b - ] [-l ] [-r] - -options: - -b,--bytes limit memory output(--format mem) to keys - greater to or equal to this value (in bytes) - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -e,--escape escape strings to encoding: raw (default), - redis. - -f,--format format to export. valid formats are json, - dump, diff, key, keyval, count, mem and resp - -h,--help rct usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -l,--largest limit memory output(--format mem) to only the - top n keys (by size). - -o,--out output file. - -r,--replace whether the generated aof with - parameter(--format dump). if not specified, - default value is false. - -s,--source eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb. - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rct version. - -examples: - rct -f dump -s ./dump.rdb -o ./appendonly.aof -r - rct -f resp -s redis://127.0.0.1:6379 -o ./target.aof -d 0 1 - rct -f json -s ./dump.rdb -o ./target.json -k user.* product.* - rct -f mem -s ./dump.rdb -o ./target.aof -e redis -t list -l 10 -b 1024 - -``` - - -```java - -usage: rmt -s [-m | -c ] [-d ] [-k ] [-t ] [-r] - -options: - -c,--config migrate data to cluster via redis cluster's - file, if specified, no need to - specify --migrate. - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -h,--help rmt usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -m,--migrate migrate to uri. eg: - redis://host:port?authPassword=foobar. - -r,--replace replace exist key value. if not specified, - default value is false. - -s,--source eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rmt version. - -examples: - rmt -s ./dump.rdb -c ./nodes.conf -t string -r - rmt -s ./dump.rdb -m redis://127.0.0.1:6380 -t list -d 0 - rmt -s redis://120.0.0.1:6379 -m redis://127.0.0.1:6380 -d 0 - -``` - -```java - -usage: rdt [-b | -s -c | -m ] -o - [-d ] [-k ] [-t ] - -options: - -b,--backup backup to local rdb file. eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb - -c,--config redis cluster's file(--split - ). - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -h,--help rdt usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -m,--merge merge multi rdb files to one rdb file. - -o,--out if --backup or --merge specified. the is the target - file. if --split specified. the - is the target path. - -s,--split split rdb to multi rdb files via cluster's - . eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rdt version. - -examples: - rdt -b ./dump.rdb -o ./dump.rdb1 -d 0 1 - rdt -b redis://127.0.0.1:6379 -o ./dump.rdb -k user.* - rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash - rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -t hash -d 0 - rdt -s redis://127.0.0.1:6379 -c ./nodes.conf -o /path/to/folder -d 0 - -``` - -### Filter - -`rct`, `rdt` and `rmt` all these commands support data filter by `type`,`db` and `key` RegEx(Java style). -For example: - -```java - -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -d 0 -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -t string hash -rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -d 0 1 -t list -``` - -### Redis mass insertion - -```java - -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -r -cat /path/to/dump.aof | /redis/src/redis-cli -p 6379 --pipe - -``` - -### Convert rdb to dump format - -```java -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -``` - -### Convert rdb to json format - -```java -rct -f json -s /path/to/dump.rdb -o /path/to/dump.json -``` - -### Numbers of key in rdb - -```java -rct -f count -s /path/to/dump.rdb -o /path/to/dump.csv -``` - -### Find top 50 largest keys - -```java -rct -f mem -s /path/to/dump.rdb -o /path/to/dump.mem -l 50 -``` - -### Diff rdb - -```java -rct -f diff -s /path/to/dump1.rdb -o /path/to/dump1.diff -rct -f diff -s /path/to/dump2.rdb -o /path/to/dump2.diff -diff /path/to/dump1.diff /path/to/dump2.diff -``` - -### Convert rdb to RESP - -```java -rct -f resp -s /path/to/dump.rdb -o /path/to/appendonly.aof -``` - -### Migrate rdb to remote redis - -```java -rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -``` - -### Migrate rdb to remote redis cluster - -```java -rmt -s /path/to/dump.rdb -c ./nodes-30001.conf -r -``` - - -### Backup remote redis's rdb - -```java -rdt -b redis://192.168.1.105:6379 -o /path/to/dump.rdb -``` - -### Filter rdb - -```java -rdt -b /path/to/dump.rdb -o /path/to/filtered-dump.rdb -d 0 -t string -``` - -### Split rdb via cluster's nodes.conf - -```java -rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -d 0 -``` - -### Merge multi rdb to one - -```java -rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash -``` - -### Other parameter - -More configurable parameter can be modified in `/path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` - -## Dashboard - -Since `v0.1.9`, the `rct -f mem` support showing result in grafana dashboard like the following: -![img](./images/memory-dashboard.png) - -If you want to turn it on. you **MUST** install `docker` and `docker-compose` first, the installation please refer to [docker](https://docs.docker.com/install/) -Then run the following command: - -```java -cd /path/to/redis-rdb-cli/dashboard -docker-compose up -d -``` - -`cd /path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` -Then change parameter [metric_gateway](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf#L149) from `none` to `prometheus`. - -Open `http://localhost:3000` to check the `rct -f mem`'s result. - -All done! - - -## Hack rmt - -### Rmt threading model - -The `rmt` command use the following 4 parameters([redis-rdb-cli.conf](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf)) to migrate data to remote. - -```java -migrate_batch_size=4096 -migrate_threads=4 -migrate_flush=yes -migrate_retries=1 -``` - -The most important parameter is `migrate_threads=4`. this means we use the following threading model to migrate data. - -```java - -single redis ----> single redis - -+--------------+ +----------+ thread 1 +--------------+ -| | +----| Endpoint |-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 2 | | -| | |----| Endpoint |-------------------| | -| | | +----------+ | | -| Source Redis |----| | Target Redis | -| | | +----------+ thread 3 | | -| | |----| Endpoint |-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 4 | | -| | +----| Endpoint |-------------------| | -+--------------+ +----------+ +--------------+ - -``` - -```java - -single redis ----> redis cluster - -+--------------+ +----------+ thread 1 +--------------+ -| | +----| Endpoints|-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 2 | | -| | |----| Endpoints|-------------------| | -| | | +----------+ | | -| Source Redis |----| | Redis cluster| -| | | +----------+ thread 3 | | -| | |----| Endpoints|-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 4 | | -| | +----| Endpoints|-------------------| | -+--------------+ +----------+ +--------------+ - -``` - -The difference between cluster migration and single migration is `Endpoint` and `Endpoints`. In cluster migration the `Endpoints` contains multi `Endpoint` to point to every `master` instance in cluster. For example: - -3 masters 3 replicas redis cluster. if `migrate_threads=4` then we have `3 * 4 = 12` connections that connected with `master` instance. - -### Migration performance - -The following 3 parameters affect migration performance - -```java -migrate_batch_size=4096 -migrate_retries=1 -migrate_flush=yes -``` - -By default we use redis `pipeline` to migrate data to remote. the `migrate_batch_size` is the `pipeline` batch size. if `migrate_batch_size=1` then the `pipeline` devolved into 1 single command to sent and wait the response from remote. -The `migrate_retries=1` means if socket error occurred. we recreate a new socket and retry to send that failed command to target redis with `migrate_retries` times. -The `migrate_flush=yes` means we write every 1 command to socket. then we invoke `SocketOutputStream.flush()` immediately. if `migrate_flush=no` we invoke `SocketOutputStream.flush()` when write to socket every 64KB. notice that this parameter also affect `migrate_retries`. the `migrate_retries` only take effect when `migrate_flush=yes` - -### Migration principle - -```java - -+---------------+ +-------------------+ restore +---------------+ -| | | redis dump format |---------------->| | -| | |-------------------| restore | | -| | convert | redis dump format |---------------->| | -| Dump rdb |------------>|-------------------| restore | Targe Redis | -| | | redis dump format |---------------->| | -| | |-------------------| restore | | -| | | redis dump format |---------------->| | -+---------------+ +-------------------+ +---------------+ -``` - -### limitation of cluster migration - -We use cluster's `nodes.conf` to migrate data to cluster. because of we did't handle the `MOVED` `ASK` redirection. so the only limitation is that the cluster **MUST** in stable state during the migration. this means the cluster **MUST** have no `migrating`, `importing` slot and no switch slave to master. - +# redis-rdb-cli + +## A tool that can parse, filter, split, merge rdb and analyze memory usage offline. + +## Binary release + +[binary releases](https://github.com/leonchen83/redis-rdb-cli/releases) + +## Runtime requirement + +```java +jdk 1.8+ +``` + +## Install + +```java +mkdir redis-rdb-cli +cd ./redis-rdb-cli +wget https://github.com/leonchen83/redis-rdb-cli/releases/download/${version}/redis-rdb-cli.zip +unzip redis-rdb-cli.zip +sudo chmod -R 755 ./ +cd ./bin +./rct -h +``` + +## Compile requirement + +```java + +jdk 1.8+ +maven-3.3.1+ + +``` + +## Compile & run + +```java + git clone https://github.com/leonchen83/redis-rdb-cli.git + cd redis-rdb-cli + mvn clean install -Dmaven.test.skip=true + cd target/redis-rdb-cli/bin + ./rct -h +``` + +## Windows Environment Variables + +Add `/path/to/redis-rdb-cli/bin` to `Path` environment variable + +## Usage + +```java + +usage: rct -f -s -o [-d ] [-e + ] [-k ] [-t ] [-b + ] [-l ] [-r] + +options: + -b,--bytes limit memory output(--format mem) to keys + greater to or equal to this value (in bytes) + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -e,--escape escape strings to encoding: raw (default), + redis. + -f,--format format to export. valid formats are json, + dump, diff, key, keyval, count, mem and resp + -h,--help rct usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -l,--largest limit memory output(--format mem) to only the + top n keys (by size). + -o,--out output file. + -r,--replace whether the generated aof with + parameter(--format dump). if not specified, + default value is false. + -s,--source eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb. + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rct version. + +examples: + rct -f dump -s ./dump.rdb -o ./appendonly.aof -r + rct -f resp -s redis://127.0.0.1:6379 -o ./target.aof -d 0 1 + rct -f json -s ./dump.rdb -o ./target.json -k user.* product.* + rct -f mem -s ./dump.rdb -o ./target.aof -e redis -t list -l 10 -b 1024 + +``` + + +```java + +usage: rmt -s [-m | -c ] [-d ] [-k ] [-t ] [-r] + +options: + -c,--config migrate data to cluster via redis cluster's + file, if specified, no need to + specify --migrate. + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -h,--help rmt usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -m,--migrate migrate to uri. eg: + redis://host:port?authPassword=foobar. + -r,--replace replace exist key value. if not specified, + default value is false. + -s,--source eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rmt version. + +examples: + rmt -s ./dump.rdb -c ./nodes.conf -t string -r + rmt -s ./dump.rdb -m redis://127.0.0.1:6380 -t list -d 0 + rmt -s redis://120.0.0.1:6379 -m redis://127.0.0.1:6380 -d 0 + +``` + +```java + +usage: rdt [-b | -s -c | -m ] -o + [-d ] [-k ] [-t ] + +options: + -b,--backup backup to local rdb file. eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb + -c,--config redis cluster's file(--split + ). + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -h,--help rdt usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -m,--merge merge multi rdb files to one rdb file. + -o,--out if --backup or --merge specified. the is the target + file. if --split specified. the + is the target path. + -s,--split split rdb to multi rdb files via cluster's + . eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rdt version. + +examples: + rdt -b ./dump.rdb -o ./dump.rdb1 -d 0 1 + rdt -b redis://127.0.0.1:6379 -o ./dump.rdb -k user.* + rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash + rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -t hash -d 0 + rdt -s redis://127.0.0.1:6379 -c ./nodes.conf -o /path/to/folder -d 0 + +``` + +### Filter + +`rct`, `rdt` and `rmt` all these commands support data filter by `type`,`db` and `key` RegEx(Java style). +For example: + +```java + +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -d 0 +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -t string hash +rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -d 0 1 -t list +``` + +### Redis mass insertion + +```java + +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -r +cat /path/to/dump.aof | /redis/src/redis-cli -p 6379 --pipe + +``` + +### Convert rdb to dump format + +```java +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof +``` + +### Convert rdb to json format + +```java +rct -f json -s /path/to/dump.rdb -o /path/to/dump.json +``` + +### Numbers of key in rdb + +```java +rct -f count -s /path/to/dump.rdb -o /path/to/dump.csv +``` + +### Find top 50 largest keys + +```java +rct -f mem -s /path/to/dump.rdb -o /path/to/dump.mem -l 50 +``` + +### Diff rdb + +```java +rct -f diff -s /path/to/dump1.rdb -o /path/to/dump1.diff +rct -f diff -s /path/to/dump2.rdb -o /path/to/dump2.diff +diff /path/to/dump1.diff /path/to/dump2.diff +``` + +### Convert rdb to RESP + +```java +rct -f resp -s /path/to/dump.rdb -o /path/to/appendonly.aof +``` + +### Migrate rdb to remote redis + +```java +rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r +``` + +### Migrate rdb to remote redis cluster + +```java +rmt -s /path/to/dump.rdb -c ./nodes-30001.conf -r +``` + +or simply use following cmd without `nodes-30001.conf` + +```java +rmt -s /path/to/dump.rdb -m redis://127.0.0.1:30001 -r +``` + +### Backup remote redis's rdb + +```java +rdt -b redis://192.168.1.105:6379 -o /path/to/dump.rdb +``` + +### Filter rdb + +```java +rdt -b /path/to/dump.rdb -o /path/to/filtered-dump.rdb -d 0 -t string +``` + +### Split rdb via cluster's nodes.conf + +```java +rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -d 0 +``` + +### Merge multi rdb to one + +```java +rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash +``` + +### Other parameter + +More configurable parameter can be modified in `/path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` + +## Dashboard + +Since `v0.1.9`, the `rct -f mem` support showing result in grafana dashboard like the following: +![img](./images/memory-dashboard.png) + +If you want to turn it on. you **MUST** install `docker` and `docker-compose` first, the installation please refer to [docker](https://docs.docker.com/install/) +Then run the following command: + +```java +cd /path/to/redis-rdb-cli/dashboard +docker-compose up -d +``` + +`cd /path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` +Then change parameter [metric_gateway](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf#L149) from `none` to `prometheus`. + +Open `http://localhost:3000` to check the `rct -f mem`'s result. + +All done! + + +## Hack rmt + +### Rmt threading model + +The `rmt` command use the following 4 parameters([redis-rdb-cli.conf](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf)) to migrate data to remote. + +```java +migrate_batch_size=4096 +migrate_threads=4 +migrate_flush=yes +migrate_retries=1 +``` + +The most important parameter is `migrate_threads=4`. this means we use the following threading model to migrate data. + +```java + +single redis ----> single redis + ++--------------+ +----------+ thread 1 +--------------+ +| | +----| Endpoint |-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 2 | | +| | |----| Endpoint |-------------------| | +| | | +----------+ | | +| Source Redis |----| | Target Redis | +| | | +----------+ thread 3 | | +| | |----| Endpoint |-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 4 | | +| | +----| Endpoint |-------------------| | ++--------------+ +----------+ +--------------+ + +``` + +```java + +single redis ----> redis cluster + ++--------------+ +----------+ thread 1 +--------------+ +| | +----| Endpoints|-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 2 | | +| | |----| Endpoints|-------------------| | +| | | +----------+ | | +| Source Redis |----| | Redis cluster| +| | | +----------+ thread 3 | | +| | |----| Endpoints|-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 4 | | +| | +----| Endpoints|-------------------| | ++--------------+ +----------+ +--------------+ + +``` + +The difference between cluster migration and single migration is `Endpoint` and `Endpoints`. In cluster migration the `Endpoints` contains multi `Endpoint` to point to every `master` instance in cluster. For example: + +3 masters 3 replicas redis cluster. if `migrate_threads=4` then we have `3 * 4 = 12` connections that connected with `master` instance. + +### Migration performance + +The following 3 parameters affect migration performance + +```java +migrate_batch_size=4096 +migrate_retries=1 +migrate_flush=yes +``` + +By default we use redis `pipeline` to migrate data to remote. the `migrate_batch_size` is the `pipeline` batch size. if `migrate_batch_size=1` then the `pipeline` devolved into 1 single command to sent and wait the response from remote. +The `migrate_retries=1` means if socket error occurred. we recreate a new socket and retry to send that failed command to target redis with `migrate_retries` times. +The `migrate_flush=yes` means we write every 1 command to socket. then we invoke `SocketOutputStream.flush()` immediately. if `migrate_flush=no` we invoke `SocketOutputStream.flush()` when write to socket every 64KB. notice that this parameter also affect `migrate_retries`. the `migrate_retries` only take effect when `migrate_flush=yes` + +### Migration principle + +```java + ++---------------+ +-------------------+ restore +---------------+ +| | | redis dump format |---------------->| | +| | |-------------------| restore | | +| | convert | redis dump format |---------------->| | +| Dump rdb |------------>|-------------------| restore | Targe Redis | +| | | redis dump format |---------------->| | +| | |-------------------| restore | | +| | | redis dump format |---------------->| | ++---------------+ +-------------------+ +---------------+ +``` + +### limitation of cluster migration + +We use cluster's `nodes.conf` to migrate data to cluster. because of we did't handle the `MOVED` `ASK` redirection. so the only limitation is that the cluster **MUST** in stable state during the migration. this means the cluster **MUST** have no `migrating`, `importing` slot and no switch slave to master. + diff --git a/README.zh_CN.md b/README.zh_CN.md index cde7b3a6..b3b7bf46 100644 --- a/README.zh_CN.md +++ b/README.zh_CN.md @@ -1,387 +1,399 @@ -# redis-rdb-cli - -## 一个可以解析, 过滤, 分割, 合并 rdb 离线内存分析的工具. - -## 下载 - -[binary releases](https://github.com/leonchen83/redis-rdb-cli/releases) - -## 运行时依赖 - -```java -jdk 1.8+ -``` - -## 安装 - -```java -mkdir redis-rdb-cli -cd ./redis-rdb-cli -wget https://github.com/leonchen83/redis-rdb-cli/releases/download/${version}/redis-rdb-cli.zip -unzip redis-rdb-cli.zip -sudo chmod -R 755 ./ -cd ./bin -./rct -h -``` - -## 手动编译依赖 - -```java - -jdk 1.8+ -maven-3.3.1+ - -``` - -## 编译 & 运行 - -```java - git clone https://github.com/leonchen83/redis-rdb-cli.git - cd redis-rdb-cli - mvn clean install -Dmaven.test.skip=true - cd target/redis-rdb-cli/bin - ./rct -h -``` - -## 设置Windows环境变量 - -把 `/path/to/redis-rdb-cli/bin` 添加到 `Path` 中 - -### 使用 - -```java - -usage: rct -f -s -o [-d ] [-e - ] [-k ] [-t ] [-b - ] [-l ] [-r] - -options: - -b,--bytes limit memory output(--format mem) to keys - greater to or equal to this value (in bytes) - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -e,--escape escape strings to encoding: raw (default), - redis. - -f,--format format to export. valid formats are json, - dump, diff, key, keyval, count, mem and resp - -h,--help rct usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -l,--largest limit memory output(--format mem) to only the - top n keys (by size). - -o,--out output file. - -r,--replace whether the generated aof with - parameter(--format dump). if not specified, - default value is false. - -s,--source eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb. - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rct version. - -examples: - rct -f dump -s ./dump.rdb -o ./appendonly.aof -r - rct -f resp -s redis://127.0.0.1:6379 -o ./target.aof -d 0 1 - rct -f json -s ./dump.rdb -o ./target.json -k user.* product.* - rct -f mem -s ./dump.rdb -o ./target.aof -e redis -t list -l 10 -b 1024 - -``` - - -```java - -usage: rmt -s [-m | -c ] [-d ] [-k ] [-t ] [-r] - -options: - -c,--config migrate data to cluster via redis cluster's - file, if specified, no need to - specify --migrate. - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -h,--help rmt usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -m,--migrate migrate to uri. eg: - redis://host:port?authPassword=foobar. - -r,--replace replace exist key value. if not specified, - default value is false. - -s,--source eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rmt version. - -examples: - rmt -s ./dump.rdb -c ./nodes.conf -t string -r - rmt -s ./dump.rdb -m redis://127.0.0.1:6380 -t list -d 0 - rmt -s redis://120.0.0.1:6379 -m redis://127.0.0.1:6380 -d 0 - -``` - -```java - -usage: rdt [-b | -s -c | -m ] -o - [-d ] [-k ] [-t ] - -options: - -b,--backup backup to local rdb file. eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump.rdb - -c,--config redis cluster's file(--split - ). - -d,--db database number. multiple databases can be - provided. if not specified, all databases - will be included. - -h,--help rdt usage. - -k,--key keys to export. this can be a regex. if not - specified, all keys will be returned. - -m,--merge merge multi rdb files to one rdb file. - -o,--out if --backup or --merge specified. the is the target - file. if --split specified. the - is the target path. - -s,--split split rdb to multi rdb files via cluster's - . eg: - /path/to/dump.rdb - redis://host:port?authPassword=foobar - redis:///path/to/dump - -t,--type data type to export. possible values are - string, hash, set, sortedset, list, module, - stream. multiple types can be provided. if - not specified, all data types will be - returned. - -v,--version rdt version. - -examples: - rdt -b ./dump.rdb -o ./dump.rdb1 -d 0 1 - rdt -b redis://127.0.0.1:6379 -o ./dump.rdb -k user.* - rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash - rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -t hash -d 0 - rdt -s redis://127.0.0.1:6379 -c ./nodes.conf -o /path/to/folder -d 0 - -``` - -### 过滤 - -`rct`, `rdt` 和 `rmt` 这3个命令支持`type`,`db` 和 `key`正则表达式(Java风格)数据过滤 -举例如下: - -```java - -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -d 0 -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -t string hash -rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -d 0 1 -t list -``` - -### Redis大量数据插入 - -```java - -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -r -cat /path/to/dump.aof | /redis/src/redis-cli -p 6379 --pipe - -``` - -### 把rdb转换成dump格式 - -```java -rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -``` - -### 把rdb转换成json格式 - -```java -rct -f json -s /path/to/dump.rdb -o /path/to/dump.json -``` - -### rdb的key数量统计 - -```java -rct -f count -s /path/to/dump.rdb -o /path/to/dump.csv -``` - -### 找到占用内存最大的50个key - -```java -rct -f mem -s /path/to/dump.rdb -o /path/to/dump.mem -l 50 -``` - -### Diff rdb - -```java -rct -f diff -s /path/to/dump1.rdb -o /path/to/dump1.diff -rct -f diff -s /path/to/dump2.rdb -o /path/to/dump2.diff -diff /path/to/dump1.diff /path/to/dump2.diff -``` - -### 把rdb转换成RESP格式 - -```java -rct -f resp -s /path/to/dump.rdb -o /path/to/appendonly.aof -``` - -### 同步rdb到远端redis - -```java -rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -``` - -### 备份远端redis的rdb - -```java -rdt -b redis://192.168.1.105:6379 -o /path/to/dump.rdb -``` - -### 过滤rdb - -```java -rdt -b /path/to/dump.rdb -o /path/to/filtered-dump.rdb -d 0 -t string -``` - -### 通过集群的nodes.conf把1个rdb分割成多个rdb - -```java -rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -d 0 -``` - -### 合并多个rdb成1个 - -```java -rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash -``` - -### 其他参数 - -更多的可配置参数可以在 `/path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` 这里配置 - -## Dashboard - -从 `v0.1.9` 起, `rct -f mem` 支持在grafana上显示结果 -![img](./images/memory-dashboard.png) - -如果你想开启这项功能. **必须** 先安装 `docker` 和 `docker-compose`, 安装方法请参照 [docker](https://docs.docker.com/install/) -然后遵循如下的步骤: - -```java -cd /path/to/redis-rdb-cli/dashboard -docker-compose up -d -``` - -`cd /path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` -把 [metric_gateway](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf#L149) 这个参数从 `none` 改成 `prometheus` - -浏览器打开 `http://localhost:3000` 来查看 `rct -f mem` 命令的结果. - -结束! - -## Hack rmt - -### Rmt 线程模型 - -`rmt`使用下面四个参数([redis-rdb-cli.conf](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf))来同步数据到远端. - -```java -migrate_batch_size=4096 -migrate_threads=4 -migrate_flush=yes -migrate_retries=1 -``` - -最重要的参数是 `migrate_threads=4`. 这意味着我们用如下的线程模型同步数据 - -```java - -单 redis ----> 单 redis - -+--------------+ +----------+ thread 1 +--------------+ -| | +----| Endpoint |-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 2 | | -| | |----| Endpoint |-------------------| | -| | | +----------+ | | -| Source Redis |----| | Target Redis | -| | | +----------+ thread 3 | | -| | |----| Endpoint |-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 4 | | -| | +----| Endpoint |-------------------| | -+--------------+ +----------+ +--------------+ - -``` - -```java - -单 redis ----> redis 集群 - -+--------------+ +----------+ thread 1 +--------------+ -| | +----| Endpoints|-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 2 | | -| | |----| Endpoints|-------------------| | -| | | +----------+ | | -| Source Redis |----| | Redis cluster| -| | | +----------+ thread 3 | | -| | |----| Endpoints|-------------------| | -| | | +----------+ | | -| | | | | -| | | +----------+ thread 4 | | -| | +----| Endpoints|-------------------| | -+--------------+ +----------+ +--------------+ - -``` - -上面两张图的不同点在 `Endpoint` 和 `Endpoints`. 在集群同步中 `Endpoints` 包含多个 `Endpoint`, 每个`Endpoint` 和集群中的 `master` 链接, 举例如下: - -集群中有 3 master 3 replica. 如果 `migrate_threads=4` 那么我们有 `3 * 4 = 12` 个连接与redis集群相连. - -### 同步性能 - -下面3个参数影响同步性能 - -```java -migrate_batch_size=4096 -migrate_retries=1 -migrate_flush=yes -``` - -默认我们使用redis的 `pipeline` 来同步数据. `migrate_batch_size` 就是 `pipeline` 批处理大小. 如果 `migrate_batch_size=1` 那么 `pipeline` 的大小就退化成处理单条命令并同步等待命令结果返回. -`migrate_retries=1` 意思是如果 socket 连接错误发生. 我们重建一个新的 socket 并重试1次把上次发送失败的命令重新发送一遍. -`migrate_flush=yes` 意思是我们每写入socket一条命令之后, 立即调用一次 `SocketOutputStream.flush()`. 如果 `migrate_flush=no` 我们每写入 64KB 到 socket 才调用一次 `SocketOutputStream.flush()`. 请注意这个参数影响 `migrate_retries`. `migrate_retries` 只有在 `migrate_flush=yes` 的时候生效. - -### 同步原理 - -```java - -+---------------+ +-------------------+ restore +---------------+ -| | | redis dump format |---------------->| | -| | |-------------------| restore | | -| | convert | redis dump format |---------------->| | -| Dump rdb |------------>|-------------------| restore | Targe Redis | -| | | redis dump format |---------------->| | -| | |-------------------| restore | | -| | | redis dump format |---------------->| | -+---------------+ +-------------------+ +---------------+ -``` - -### 同步到集群的限制 - -我们通过集群的 `nodes.conf` 文件来同步数据到集群. 因为我们没有处理 `MOVED` `ASK` 重定向. 因此唯一的限制是集群在同步期间 **必须** 是稳定的状态. 这意味着集群 **必须** 不存在 `migrating`, `importing` 这样的slot. 而且没有主从切换. - +# redis-rdb-cli + +## 一个可以解析, 过滤, 分割, 合并 rdb 离线内存分析的工具. + +## 下载 + +[binary releases](https://github.com/leonchen83/redis-rdb-cli/releases) + +## 运行时依赖 + +```java +jdk 1.8+ +``` + +## 安装 + +```java +mkdir redis-rdb-cli +cd ./redis-rdb-cli +wget https://github.com/leonchen83/redis-rdb-cli/releases/download/${version}/redis-rdb-cli.zip +unzip redis-rdb-cli.zip +sudo chmod -R 755 ./ +cd ./bin +./rct -h +``` + +## 手动编译依赖 + +```java + +jdk 1.8+ +maven-3.3.1+ + +``` + +## 编译 & 运行 + +```java + git clone https://github.com/leonchen83/redis-rdb-cli.git + cd redis-rdb-cli + mvn clean install -Dmaven.test.skip=true + cd target/redis-rdb-cli/bin + ./rct -h +``` + +## 设置Windows环境变量 + +把 `/path/to/redis-rdb-cli/bin` 添加到 `Path` 中 + +### 使用 + +```java + +usage: rct -f -s -o [-d ] [-e + ] [-k ] [-t ] [-b + ] [-l ] [-r] + +options: + -b,--bytes limit memory output(--format mem) to keys + greater to or equal to this value (in bytes) + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -e,--escape escape strings to encoding: raw (default), + redis. + -f,--format format to export. valid formats are json, + dump, diff, key, keyval, count, mem and resp + -h,--help rct usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -l,--largest limit memory output(--format mem) to only the + top n keys (by size). + -o,--out output file. + -r,--replace whether the generated aof with + parameter(--format dump). if not specified, + default value is false. + -s,--source eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb. + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rct version. + +examples: + rct -f dump -s ./dump.rdb -o ./appendonly.aof -r + rct -f resp -s redis://127.0.0.1:6379 -o ./target.aof -d 0 1 + rct -f json -s ./dump.rdb -o ./target.json -k user.* product.* + rct -f mem -s ./dump.rdb -o ./target.aof -e redis -t list -l 10 -b 1024 + +``` + + +```java + +usage: rmt -s [-m | -c ] [-d ] [-k ] [-t ] [-r] + +options: + -c,--config migrate data to cluster via redis cluster's + file, if specified, no need to + specify --migrate. + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -h,--help rmt usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -m,--migrate migrate to uri. eg: + redis://host:port?authPassword=foobar. + -r,--replace replace exist key value. if not specified, + default value is false. + -s,--source eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rmt version. + +examples: + rmt -s ./dump.rdb -c ./nodes.conf -t string -r + rmt -s ./dump.rdb -m redis://127.0.0.1:6380 -t list -d 0 + rmt -s redis://120.0.0.1:6379 -m redis://127.0.0.1:6380 -d 0 + +``` + +```java + +usage: rdt [-b | -s -c | -m ] -o + [-d ] [-k ] [-t ] + +options: + -b,--backup backup to local rdb file. eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump.rdb + -c,--config redis cluster's file(--split + ). + -d,--db database number. multiple databases can be + provided. if not specified, all databases + will be included. + -h,--help rdt usage. + -k,--key keys to export. this can be a regex. if not + specified, all keys will be returned. + -m,--merge merge multi rdb files to one rdb file. + -o,--out if --backup or --merge specified. the is the target + file. if --split specified. the + is the target path. + -s,--split split rdb to multi rdb files via cluster's + . eg: + /path/to/dump.rdb + redis://host:port?authPassword=foobar + redis:///path/to/dump + -t,--type data type to export. possible values are + string, hash, set, sortedset, list, module, + stream. multiple types can be provided. if + not specified, all data types will be + returned. + -v,--version rdt version. + +examples: + rdt -b ./dump.rdb -o ./dump.rdb1 -d 0 1 + rdt -b redis://127.0.0.1:6379 -o ./dump.rdb -k user.* + rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash + rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -t hash -d 0 + rdt -s redis://127.0.0.1:6379 -c ./nodes.conf -o /path/to/folder -d 0 + +``` + +### 过滤 + +`rct`, `rdt` 和 `rmt` 这3个命令支持`type`,`db` 和 `key`正则表达式(Java风格)数据过滤 +举例如下: + +```java + +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -d 0 +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -t string hash +rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r -d 0 1 -t list +``` + +### Redis大量数据插入 + +```java + +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof -r +cat /path/to/dump.aof | /redis/src/redis-cli -p 6379 --pipe + +``` + +### 把rdb转换成dump格式 + +```java +rct -f dump -s /path/to/dump.rdb -o /path/to/dump.aof +``` + +### 把rdb转换成json格式 + +```java +rct -f json -s /path/to/dump.rdb -o /path/to/dump.json +``` + +### rdb的key数量统计 + +```java +rct -f count -s /path/to/dump.rdb -o /path/to/dump.csv +``` + +### 找到占用内存最大的50个key + +```java +rct -f mem -s /path/to/dump.rdb -o /path/to/dump.mem -l 50 +``` + +### Diff rdb + +```java +rct -f diff -s /path/to/dump1.rdb -o /path/to/dump1.diff +rct -f diff -s /path/to/dump2.rdb -o /path/to/dump2.diff +diff /path/to/dump1.diff /path/to/dump2.diff +``` + +### 把rdb转换成RESP格式 + +```java +rct -f resp -s /path/to/dump.rdb -o /path/to/appendonly.aof +``` + +### 同步rdb到远端redis + +```java +rmt -s /path/to/dump.rdb -m redis://192.168.1.105:6379 -r +``` + +### 同步rdb到远端redis集群 + +```java +rmt -s /path/to/dump.rdb -c ./nodes-30001.conf -r +``` + +或者不用 `nodes-30001.conf` 这个配置文件, 直接使用如下命令 + +```java +rmt -s /path/to/dump.rdb -m redis://127.0.0.1:30001 -r +``` + +### 备份远端redis的rdb + +```java +rdt -b redis://192.168.1.105:6379 -o /path/to/dump.rdb +``` + +### 过滤rdb + +```java +rdt -b /path/to/dump.rdb -o /path/to/filtered-dump.rdb -d 0 -t string +``` + +### 通过集群的nodes.conf把1个rdb分割成多个rdb + +```java +rdt -s ./dump.rdb -c ./nodes.conf -o /path/to/folder -d 0 +``` + +### 合并多个rdb成1个 + +```java +rdt -m ./dump1.rdb ./dump2.rdb -o ./dump.rdb -t hash +``` + +### 其他参数 + +更多的可配置参数可以在 `/path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` 这里配置 + +## Dashboard + +从 `v0.1.9` 起, `rct -f mem` 支持在grafana上显示结果 +![img](./images/memory-dashboard.png) + +如果你想开启这项功能. **必须** 先安装 `docker` 和 `docker-compose`, 安装方法请参照 [docker](https://docs.docker.com/install/) +然后遵循如下的步骤: + +```java +cd /path/to/redis-rdb-cli/dashboard +docker-compose up -d +``` + +`cd /path/to/redis-rdb-cli/conf/redis-rdb-cli.conf` +把 [metric_gateway](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf#L149) 这个参数从 `none` 改成 `prometheus` + +浏览器打开 `http://localhost:3000` 来查看 `rct -f mem` 命令的结果. + +结束! + +## Hack rmt + +### Rmt 线程模型 + +`rmt`使用下面四个参数([redis-rdb-cli.conf](https://github.com/leonchen83/redis-rdb-cli/blob/master/src/main/resources/redis-rdb-cli.conf))来同步数据到远端. + +```java +migrate_batch_size=4096 +migrate_threads=4 +migrate_flush=yes +migrate_retries=1 +``` + +最重要的参数是 `migrate_threads=4`. 这意味着我们用如下的线程模型同步数据 + +```java + +单 redis ----> 单 redis + ++--------------+ +----------+ thread 1 +--------------+ +| | +----| Endpoint |-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 2 | | +| | |----| Endpoint |-------------------| | +| | | +----------+ | | +| Source Redis |----| | Target Redis | +| | | +----------+ thread 3 | | +| | |----| Endpoint |-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 4 | | +| | +----| Endpoint |-------------------| | ++--------------+ +----------+ +--------------+ + +``` + +```java + +单 redis ----> redis 集群 + ++--------------+ +----------+ thread 1 +--------------+ +| | +----| Endpoints|-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 2 | | +| | |----| Endpoints|-------------------| | +| | | +----------+ | | +| Source Redis |----| | Redis cluster| +| | | +----------+ thread 3 | | +| | |----| Endpoints|-------------------| | +| | | +----------+ | | +| | | | | +| | | +----------+ thread 4 | | +| | +----| Endpoints|-------------------| | ++--------------+ +----------+ +--------------+ + +``` + +上面两张图的不同点在 `Endpoint` 和 `Endpoints`. 在集群同步中 `Endpoints` 包含多个 `Endpoint`, 每个`Endpoint` 和集群中的 `master` 链接, 举例如下: + +集群中有 3 master 3 replica. 如果 `migrate_threads=4` 那么我们有 `3 * 4 = 12` 个连接与redis集群相连. + +### 同步性能 + +下面3个参数影响同步性能 + +```java +migrate_batch_size=4096 +migrate_retries=1 +migrate_flush=yes +``` + +默认我们使用redis的 `pipeline` 来同步数据. `migrate_batch_size` 就是 `pipeline` 批处理大小. 如果 `migrate_batch_size=1` 那么 `pipeline` 的大小就退化成处理单条命令并同步等待命令结果返回. +`migrate_retries=1` 意思是如果 socket 连接错误发生. 我们重建一个新的 socket 并重试1次把上次发送失败的命令重新发送一遍. +`migrate_flush=yes` 意思是我们每写入socket一条命令之后, 立即调用一次 `SocketOutputStream.flush()`. 如果 `migrate_flush=no` 我们每写入 64KB 到 socket 才调用一次 `SocketOutputStream.flush()`. 请注意这个参数影响 `migrate_retries`. `migrate_retries` 只有在 `migrate_flush=yes` 的时候生效. + +### 同步原理 + +```java + ++---------------+ +-------------------+ restore +---------------+ +| | | redis dump format |---------------->| | +| | |-------------------| restore | | +| | convert | redis dump format |---------------->| | +| Dump rdb |------------>|-------------------| restore | Targe Redis | +| | | redis dump format |---------------->| | +| | |-------------------| restore | | +| | | redis dump format |---------------->| | ++---------------+ +-------------------+ +---------------+ +``` + +### 同步到集群的限制 + +我们通过集群的 `nodes.conf` 文件来同步数据到集群. 因为我们没有处理 `MOVED` `ASK` 重定向. 因此唯一的限制是集群在同步期间 **必须** 是稳定的状态. 这意味着集群 **必须** 不存在 `migrating`, `importing` 这样的slot. 而且没有主从切换. +