Skip to content

Commit

Permalink
Add new handlersToPrint + expiration filter
Browse files Browse the repository at this point in the history
  • Loading branch information
@moticless
moticless committed Jul 17, 2024
1 parent 08757bf commit 2e7b1ee
Show file tree
Hide file tree
Showing 11 changed files with 561 additions and 101 deletions.
12 changes: 9 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,17 +196,23 @@ destruction, or when newer block replacing old one.

### rdb-cli usage

Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]
Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis} [FORMAT_OPTIONS]
OPTIONS:
-l, --log-file <PATH> Path to the log file or stdout (Default: './rdb-cli.log')

Multiple filters combination of keys/types/dbs can be specified:
-k, --key <REGEX> Include only keys that match REGEX
-K --no-key <REGEX> Exclude all keys that match REGEX
-t, --type <TYPE> Include only selected TYPE {str|list|set|zset|hash|module|func}
-T, --no-type <TYPE> Exclude TYPE {str|list|set|zset|hash|module|func}
-d, --dbnum <DBNUM> Include only selected db number
-D, --no-dbnum <DBNUM> Exclude DB number
-e, --expired Include only expired keys
-E, --no-expired Exclude expired keys

FORMAT_OPTIONS ('print'):
-a, --aux-val <FMT> %f=Auxiliary-Field, %v=Auxiliary-Value (Default: "")
-k, --key <FMT> %d=Db %k=Key %v=Value %t=Type %e=Expiry %r=LRU %f=LFU %i=Items
(Default: "%d,%k,%v,%t,%e,%i")
-o, --output <FILE> Specify the output file. If not specified, output to stdout

FORMAT_OPTIONS ('json'):
-i, --include <EXTRAS> To include: {aux-val|func|stream-meta}
Expand Down
1 change: 1 addition & 0 deletions api/librdb-api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ typedef struct RdbKeyInfo {
long long lruIdle; /* -1 if not set */
int lfuFreq; /* -1 if not set */
int opcode;
int dataType; /* See enum RdbDataType */
} RdbKeyInfo;

typedef struct RdbSlotInfo {
Expand Down
26 changes: 26 additions & 0 deletions api/librdb-ext-api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ typedef struct RdbxReaderFileDesc RdbxReaderFileDesc;
typedef struct RdbxFilter RdbxFilter;
typedef struct RdbxToJson RdbxToJson;
typedef struct RdbxToResp RdbxToResp;
typedef struct RdbxToPrint RdbxToPrint;
typedef struct RdbxRespToRedisLoader RdbxRespToRedisLoader;

/****************************************************************
Expand Down Expand Up @@ -93,6 +94,28 @@ _LIBRDB_API RdbxToJson *RDBX_createHandlersToJson(RdbParser *p,
const char *filename,
RdbxToJsonConf *c);

/****************************************************************
* Create PRINT Handlers
*
* auxFmt - Format string for auxiliary values, where:
* %f = Auxiliary field name
* %v = Auxiliary field value
* keyFmt - Format string for key details, where:
* %d = Database number
* %k = Key
* %v = Value (If the value is a string, it will be printed as escaped string)
* %t = Type
* %e = Expiry
* %r = LRU
* %f = LFU
* %i = Items
*
****************************************************************/
_LIBRDB_API RdbxToPrint *RDBX_createHandlersToPrint(RdbParser *p,
const char *auxFmt,
const char *keyFmt,
const char *outFilename);

/****************************************************************
* Create Filter Handlers
****************************************************************/
Expand All @@ -109,6 +132,9 @@ _LIBRDB_API RdbxFilter *RDBX_createHandlersFilterDbNum(RdbParser *p,
int dbnum,
uint32_t exclude);

_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterExpired(RdbParser *p,
uint32_t exclude);

/****************************************************************
* Create RDB to RESP Handlers
*
Expand Down
49 changes: 45 additions & 4 deletions src/cli/rdb-cli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,23 @@ static void printUsage(int shortUsage) {
return;
}
printf("[v%s] ", RDB_getLibVersion(NULL,NULL,NULL));
printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]\n");
printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {print|json|resp|redis} [FORMAT_OPTIONS]\n");
printf("OPTIONS:\n");
printf("\t-l, --log-file <PATH> Path to the log file or stdout (Default: './rdb-cli.log')\n\n");
printf("\tMultiple filters combination of keys/types/dbs can be specified:\n");
printf("\t-l, --log-file <PATH> Path to the log file or stdout (Default: './rdb-cli.log')\n");
printf("\t-k, --key <REGEX> Include only keys that match REGEX\n");
printf("\t-K --no-key <REGEX> Exclude all keys that match REGEX\n");
printf("\t-t, --type <TYPE> Include only selected TYPE {str|list|set|zset|hash|module|func}\n");
printf("\t-T, --no-type <TYPE> Exclude TYPE {str|list|set|zset|hash|module|func}\n");
printf("\t-d, --dbnum <DBNUM> Include only selected db number\n");
printf("\t-D, --no-dbnum <DBNUM> Exclude DB number\n\n");
printf("\t-D, --no-dbnum <DBNUM> Exclude DB number\n");
printf("\t-e, --expired Include only expired keys\n");
printf("\t-E, --no-expired Exclude expired keys\n\n");

printf("FORMAT_OPTIONS ('print'):\n");
printf("\t-a, --aux-val <FMT> %%f=Auxiliary-Field, %%v=Auxiliary-Value (Default: \"\") \n");
printf("\t-k, --key <FMT> %%d=Db %%k=Key %%v=Value %%t=Type %%e=Expiry %%r=LRU %%f=LFU %%i=Items\n");
printf("\t (Default: \"%%d,%%k,%%v,%%t,%%e,%%i\")\n");
printf("\t-o, --output <FILE> Specify the output file. If not specified, output to stdout\n\n");

printf("FORMAT_OPTIONS ('json'):\n");
printf("\t-i, --include <EXTRAS> To include: {aux-val|func|stream-meta|db-info}\n");
Expand Down Expand Up @@ -169,6 +176,27 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) {
return RDB_OK;
}

static RdbRes formatPrint(RdbParser *parser, int argc, char **argv) {
const char *auxFmt = NULL, *keyFmt = "%d,%k,%v,%t,%e,%i";
const char *output = NULL;/*default:stdout*/

/* parse specific command options */
for (int at = 1; at < argc; ++at) {
char *opt = argv[at];
if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue;
if (getOptArg(argc, argv, &at, "-a", "--aux-val", NULL, &auxFmt)) continue;
if (getOptArg(argc, argv, &at, "-k", "--key", NULL, &keyFmt)) continue;
loggerWrap(RDB_LOG_ERR, "Invalid 'print' [FORMAT_OPTIONS] argument: %s\n", opt);
printUsage(1);
return RDB_ERR_GENERAL;
}

if (RDBX_createHandlersToPrint(parser, auxFmt, keyFmt, output) == NULL)
return RDB_ERR_GENERAL;

return RDB_OK;
}

static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) {
const char *output = NULL;
RdbxRedisAuth auth = {0};
Expand Down Expand Up @@ -354,9 +382,22 @@ int readCommonOptions(RdbParser *p, int argc, char* argv[], Options *options, in
continue;
}

if (getOptArg(argc, argv, &at, "-e", "--expired", NULL, NULL)) {
if ((applyFilters) && (!RDBX_createHandlersFilterExpired(p, 0)))
exit(RDBX_ERR_FAILED_CREATE_FILTER);
continue;
}

if (getOptArg(argc, argv, &at, "-E", "--no-expired", NULL, NULL)) {
if ((applyFilters) && (!RDBX_createHandlersFilterExpired(p, 1)))
exit(RDBX_ERR_FAILED_CREATE_FILTER);
continue;
}

if (strcmp(opt, "json") == 0) { options->formatFunc = formatJson; break; }
else if (strcmp(opt, "resp") == 0) { options->formatFunc = formatResp; break; }
else if (strcmp(opt, "redis") == 0) { options->formatFunc = formatRedis; break; }
else if (strcmp(opt, "print") == 0) { options->formatFunc = formatPrint; break; }

loggerWrap(RDB_LOG_ERR, "At argv[%d], unexpected OPTIONS argument: %s\n", at, opt);
printUsage(1);
Expand Down
80 changes: 34 additions & 46 deletions src/ext/handlersFilter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ struct RdbxFilter {

int regexInitialized; /* for filter keys */
RdbDataType type; /* for filter types */
int isExpireFilter; /* for filter expired */
int dbnum; /* for filter db */
};

Expand All @@ -22,45 +23,6 @@ static void deleteFilterCtx(RdbParser *p, void *data) {
RDB_free(p, ctx);
}

/* mapping opcode to type */
static void initOpcodeToType(RdbxFilter *ctx) {
memset(ctx->opToType, 0, sizeof(ctx->opToType));
/*string*/
ctx->opToType[RDB_TYPE_STRING] = RDB_DATA_TYPE_STRING;
/*list*/
ctx->opToType[RDB_TYPE_LIST] = RDB_DATA_TYPE_LIST;
ctx->opToType[RDB_TYPE_LIST_ZIPLIST] = RDB_DATA_TYPE_LIST;
ctx->opToType[RDB_TYPE_LIST_QUICKLIST] = RDB_DATA_TYPE_LIST;
ctx->opToType[RDB_TYPE_LIST_QUICKLIST_2] = RDB_DATA_TYPE_LIST;
/*set*/
ctx->opToType[RDB_TYPE_SET] = RDB_DATA_TYPE_SET;
ctx->opToType[RDB_TYPE_SET_INTSET] = RDB_DATA_TYPE_SET;
ctx->opToType[RDB_TYPE_SET_LISTPACK] = RDB_DATA_TYPE_SET;
/*zset*/
ctx->opToType[RDB_TYPE_ZSET] = RDB_DATA_TYPE_ZSET;
ctx->opToType[RDB_TYPE_ZSET_2] = RDB_DATA_TYPE_ZSET;
ctx->opToType[RDB_TYPE_ZSET_ZIPLIST] = RDB_DATA_TYPE_ZSET;
ctx->opToType[RDB_TYPE_ZSET_LISTPACK] = RDB_DATA_TYPE_ZSET;
/*hash*/
ctx->opToType[RDB_TYPE_HASH] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_METADATA_PRE_GA] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_METADATA] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_ZIPMAP] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_ZIPLIST] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_LISTPACK] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_LISTPACK_EX_PRE_GA] = RDB_DATA_TYPE_HASH;
ctx->opToType[RDB_TYPE_HASH_LISTPACK_EX] = RDB_DATA_TYPE_HASH;
/*module*/
ctx->opToType[RDB_TYPE_MODULE_2] = RDB_DATA_TYPE_MODULE;
ctx->opToType[RDB_OPCODE_MODULE_AUX] = RDB_DATA_TYPE_MODULE;
/*stream*/
ctx->opToType[RDB_TYPE_STREAM_LISTPACKS] = RDB_DATA_TYPE_STREAM;
ctx->opToType[RDB_TYPE_STREAM_LISTPACKS_2] = RDB_DATA_TYPE_STREAM;
ctx->opToType[RDB_TYPE_STREAM_LISTPACKS_3] = RDB_DATA_TYPE_STREAM;
/*func*/
ctx->opToType[RDB_OPCODE_FUNCTION2] = RDB_DATA_TYPE_FUNCTION;
}

/*** filtering BY key, type or dbnum ***/

static RdbRes filterNewKeyByRegex(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) {
Expand All @@ -75,12 +37,32 @@ static RdbRes filterNewKeyByRegex(RdbParser *p, void *userData, RdbBulk key, Rdb
static RdbRes filterNewKeyByType(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) {
UNUSED(p, key);
RdbxFilter *ctx = userData;
if (ctx->opToType[info->opcode] == ctx->type) /* if match */

if (info->dataType == (int) ctx->type)
return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK_DONT_PROPAGATE : RDB_OK;
else
return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE;
}

static RdbRes filterNewKeyByExpiry(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) {
UNUSED(p, key);
RdbxFilter *ctx = userData;

/* if persistent key */
if (info->expiretime == -1)
return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE;

struct timeval te;
gettimeofday(&te, NULL);
long long milliseconds = te.tv_sec*1000LL + te.tv_usec/1000;

if (info->expiretime > milliseconds)
return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE;

return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK_DONT_PROPAGATE
: RDB_OK;
}

static RdbRes filterNewDbByNumber(RdbParser *p, void *userData, int dbnum) {
UNUSED(p);
RdbxFilter *ctx = userData;
Expand Down Expand Up @@ -356,15 +338,15 @@ static RdbxFilter *createHandlersFilterCommon(RdbParser *p,
const char *keyRegex,
RdbDataType *type,
int *dbnum,
int isExpireFilter,
uint32_t exclude) {
RdbRes (*handleNewKey)(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) = filterNewKey;
RdbRes (*handleNewDb)(RdbParser *p, void *userData, int dbnum) = filterNewDb;
RdbxFilter *ctx;

if ( (ctx = RDB_alloc(p, sizeof(RdbxFilter))) == NULL)
return NULL;

ctx->regexInitialized = 0;
memset(ctx, 0, sizeof(RdbxFilter));

/* specific if-else init to filter regex/type/dbnum */
if (keyRegex) { /* filter keys by regex */
Expand All @@ -383,7 +365,9 @@ static RdbxFilter *createHandlersFilterCommon(RdbParser *p,
} else if (type) { /* filter keys by type */
ctx->type = *type;
handleNewKey = filterNewKeyByType;
initOpcodeToType(ctx);
} else if (isExpireFilter) {
ctx->isExpireFilter = 1;
handleNewKey = filterNewKeyByExpiry;
} else { /* filter by dbnum */
ctx->dbnum = *dbnum;
handleNewDb = filterNewDbByNumber;
Expand Down Expand Up @@ -421,13 +405,17 @@ static RdbxFilter *createHandlersFilterCommon(RdbParser *p,
/*** API ***/

_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterKey(RdbParser *p, const char *keyRegex, uint32_t exclude) {
return createHandlersFilterCommon(p, keyRegex, NULL, NULL, exclude);
return createHandlersFilterCommon(p, keyRegex, NULL, NULL, 0, exclude);
}

_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterType(RdbParser *p, RdbDataType type, uint32_t exclude) {
return createHandlersFilterCommon(p, NULL, &type, NULL, exclude);
return createHandlersFilterCommon(p, NULL, &type, NULL, 0, exclude);
}

_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterDbNum(RdbParser *p, int dbnum, uint32_t exclude) {
return createHandlersFilterCommon(p, NULL, NULL, &dbnum, exclude);
return createHandlersFilterCommon(p, NULL, NULL, &dbnum, 0, exclude);
}

_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterExpired(RdbParser *p, uint32_t exclude) {
return createHandlersFilterCommon(p, NULL, NULL, NULL, 1, exclude);
}
Loading

0 comments on commit 2e7b1ee

Please sign in to comment.