Skip to content

Commit

Permalink
integrate python json parser service
Browse files Browse the repository at this point in the history
  • Loading branch information
moticless committed Jun 12, 2024
1 parent 581221f commit c444094
Show file tree
Hide file tree
Showing 50 changed files with 580 additions and 302 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ To see cool internal state printouts of the parser, set env-var `LIBRDB_DEBUG_DA

% LIBRDB_DEBUG_DATA=1 make example

To build and run tests, you need to have cmocka unit testing framework installed:
To build and run tests, you need to have cmocka unit testing and python3 installed:

% make test

Expand Down
23 changes: 15 additions & 8 deletions src/ext/handlersToJson.c
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ static RdbRes toJsonDbSize(RdbParser *p, void *userData, uint64_t db_size, uint6
}

/* output json part */
fprintf(ctx->outfile, " \"%sdbSize\": {\n", jsonMetaPrefix);
fprintf(ctx->outfile, " \"__dbsize__\": {\n");
fprintf(ctx->outfile, " \"size\": %" PRIu64 ",\n", db_size);
fprintf(ctx->outfile, " \"expires\": %" PRIu64 "\n", exp_size);
fprintf(ctx->outfile, " }%s\n", (db_size) ? "," : "");
Expand All @@ -171,7 +171,7 @@ static RdbRes toJsonSlotInfo(RdbParser *p, void *userData, RdbSlotInfo *info) {
}

/* output json part */
fprintf(ctx->outfile, " \"%sSlotInfo\": {\n", jsonMetaPrefix);
fprintf(ctx->outfile, " \"__slotinfo__\": {\n");
fprintf(ctx->outfile, " \"slotId\": %lu,\n", info->slot_id);
fprintf(ctx->outfile, " \"slotSize\": %lu,\n", info->slot_size);
fprintf(ctx->outfile, " \"slotSExpiresSize\": %lu\n", info->expires_slot_size);
Expand All @@ -184,7 +184,7 @@ static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBu

if (ctx->state == R2J_IDLE) {
ctx->state = R2J_AUX_FIELDS;
fprintf(ctx->outfile, "{\n "); /* group aux-fields with { ... } */
fprintf(ctx->outfile, "\"__aux__\" : {\n "); /* group aux-fields with { ... } */
} else if (ctx->state == R2J_AUX_FIELDS) {
fprintf(ctx->outfile, ",\n ");
} else {
Expand Down Expand Up @@ -432,14 +432,20 @@ static RdbRes toJsonSet(RdbParser *p, void *userData, RdbBulk member) {
static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double score) {
RdbxToJson *ctx = userData;

char score_str[MAX_D2STRING_CHARS];
int len = d2string(score_str, sizeof(score_str), score);
char scoreStr[MAX_D2STRING_CHARS];
int len = d2string(scoreStr, sizeof(scoreStr), score);

/* -0 is a valid double, but we want to output it as 0 */
if ((len == 2) && (scoreStr[0] == '-') && (scoreStr[1] == '0')) {
scoreStr[0] = '0';
scoreStr[1] = '\0';
}

if (ctx->state == R2J_IN_KEY) {
/* output json part */
fprintf(ctx->outfile, "{");
outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
fprintf(ctx->outfile, ":\"%.*s\"", len, score_str);
fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);

/* update new state */
ctx->state = R2J_IN_ZSET;
Expand All @@ -448,7 +454,7 @@ static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double sc
/* output json part */
fprintf(ctx->outfile, ",");
outputQuotedEscaping(ctx, member, RDB_bulkLen(p, member));
fprintf(ctx->outfile, ":\"%.*s\"", len, score_str);
fprintf(ctx->outfile, ":\"%.*s\"", len, scoreStr);

/* state unchanged */

Expand Down Expand Up @@ -494,8 +500,9 @@ static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) {

if (ctx->state == R2J_IDLE) {
ctx->state = R2J_FUNCTIONS;
fprintf(ctx->outfile, "\"__func__\": {\n");
} else if (ctx->state == R2J_AUX_FIELDS) {
fprintf(ctx->outfile, "\n},{\n");
fprintf(ctx->outfile, "\n},\n \"__func__\": {\n");
ctx->state = R2J_FUNCTIONS;
} else if (ctx->state == R2J_FUNCTIONS) {
fprintf(ctx->outfile, ",\n");
Expand Down
24 changes: 13 additions & 11 deletions src/lib/defines.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,22 @@
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */

/* Object types for encoded objects. */
#define RDB_TYPE_HASH_ZIPMAP 9
#define RDB_TYPE_LIST_ZIPLIST 10
#define RDB_TYPE_SET_INTSET 11
#define RDB_TYPE_ZSET_ZIPLIST 12
#define RDB_TYPE_HASH_ZIPLIST 13
#define RDB_TYPE_LIST_QUICKLIST 14
#define RDB_TYPE_STREAM_LISTPACKS 15
#define RDB_TYPE_HASH_LISTPACK 16
#define RDB_TYPE_ZSET_LISTPACK 17
#define RDB_TYPE_HASH_ZIPMAP 9
#define RDB_TYPE_LIST_ZIPLIST 10
#define RDB_TYPE_SET_INTSET 11
#define RDB_TYPE_ZSET_ZIPLIST 12
#define RDB_TYPE_HASH_ZIPLIST 13
#define RDB_TYPE_LIST_QUICKLIST 14
#define RDB_TYPE_STREAM_LISTPACKS 15
#define RDB_TYPE_HASH_LISTPACK 16
#define RDB_TYPE_ZSET_LISTPACK 17
#define RDB_TYPE_LIST_QUICKLIST_2 18
#define RDB_TYPE_STREAM_LISTPACKS_2 19
#define RDB_TYPE_SET_LISTPACK 20
#define RDB_TYPE_SET_LISTPACK 20
#define RDB_TYPE_STREAM_LISTPACKS_3 21
#define RDB_TYPE_MAX 22
#define RDB_TYPE_HASH_METADATA 22
#define RDB_TYPE_HASH_LISTPACK_EX 23
#define RDB_TYPE_MAX 24
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */

/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
Expand Down
2 changes: 1 addition & 1 deletion test/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ STD = -std=c99
STACK = -fstack-protector-all -Wstack-protector
WARNS = -Wall -Wextra -pedantic -Werror -Wno-unused-function

OPTIMIZATION?=-O3
OPTIMIZATION?=-O0

CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS)
DEBUG = -g3 -DDEBUG=1
Expand Down
27 changes: 14 additions & 13 deletions test/dumps/cluster_slot_info.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[{
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1713005699",
Expand All @@ -8,15 +8,16 @@
"repl-offset":"390",
"aof-base":"0"
},
{
"__dbSize": {
"size": 1,
"expires": 0
},
"__SlotInfo": {
"slotId": 7638,
"slotSize": 1,
"slotSExpiresSize": 0
},
"abc":"abc"
}]

"__dbsize__": {
"size": 1,
"expires": 0
},

"__slotinfo__": {
"slotId": 7638,
"slotSize": 1,
"slotSExpiresSize": 0
},

"abc":"abc"
6 changes: 0 additions & 6 deletions test/dumps/function.json

This file was deleted.

29 changes: 15 additions & 14 deletions test/dumps/function2.json
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
[{
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1713086666",
"used-mem":"966312",
"aof-base":"0"
},{
},

"__func__": {
"__Function_1":"#!lua name=mylib2\n\nlocal function my_hset2(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall2(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified2(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset2', my_hset2)\nredis.register_function('my_hgetall2', my_hgetall2)\nredis.register_function('my_hlastmodified2', my_hlastmodified2)\n\n",
"__Function_3":"#!lua name=mylib\n\nlocal function my_hset(keys, args)\n local hash = keys[1]\n local time = redis.call('TIME')[1]\n return redis.call('HSET', hash, '_last_modified_', time, unpack(args))\nend\n\nlocal function my_hgetall(keys, args)\n redis.setresp(3)\n local hash = keys[1]\n local res = redis.call('HGETALL', hash)\n res['map']['_last_modified_'] = nil\n return res\nend\n\nlocal function my_hlastmodified(keys, args)\n local hash = keys[1]\n return redis.call('HGET', hash, '_last_modified_')\nend\n\nredis.register_function('my_hset', my_hset)\nredis.register_function('my_hgetall', my_hgetall)\nredis.register_function('my_hlastmodified', my_hlastmodified)\n\n"
},
{
"key_97":"value_97",
"key_90":"value_90",
"key_93":"value_93",
"key_99":"value_99",
"key_96":"value_96",
"key_92":"value_92",
"key_95":"value_95",
"key_91":"value_91",
"key_94":"value_94",
"key_98":"value_98"
}]

"key_97":"value_97",
"key_90":"value_90",
"key_93":"value_93",
"key_99":"value_99",
"key_96":"value_96",
"key_92":"value_92",
"key_95":"value_95",
"key_91":"value_91",
"key_94":"value_94",
"key_98":"value_98"
13 changes: 8 additions & 5 deletions test/dumps/hash_lp_v11_data.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0"
},

"hash2":{"field7":"value7","field8":"value8","9":"value9","field10":"value10","field11":"11","12":"12.0"},
"hash1":{"field2":"2","field3":"3","field4":"value4.0","field5":"5.0","6":"6"}
13 changes: 8 additions & 5 deletions test/dumps/hash_lp_v11_raw.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0"
},

"hash2":"\u0010\u00c3@C@V\u0013V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value \u0007`\u000f\u00008\u00a0\u000f\u00038\u0007\t\u0001\u0080\u0019\u00029\u0007\u0087`\u0019\u000310\b\u0087`*@\b\u0080\u0011\f1\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff",
"hash1":"\u0010\u00c35>\u000f>\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0080\t\u00023\u0007\u0003\u00a0\t\u000b4\u0007\u0088value4.0\t\u0080\u001b\u000b5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"
13 changes: 8 additions & 5 deletions test/dumps/hash_lp_v11_struct.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1695280472",
"used-mem":"1062024",
"aof-base":"0"
},

"hash2":["V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value7\u0007\u0086field8\u0007\u0086value8\u0007\t\u0001\u0086value9\u0007\u0087field10\b\u0087value10\b\u0087field11\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff"],
"hash1":[">\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0086field3\u0007\u0003\u0001\u0086field4\u0007\u0088value4.0\t\u0086field5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"]
13 changes: 8 additions & 5 deletions test/dumps/hash_zl_v6_data.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0",
"__aux__" : {
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0"
},

"myhash":{"1":"2","3":"4","5":"6","7.0":"8.0","str1":"str2","str3":"str4"}
13 changes: 8 additions & 5 deletions test/dumps/hash_zl_v6_raw.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0",
"__aux__" : {
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0"
},

"myhash":"\r99\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"
13 changes: 8 additions & 5 deletions test/dumps/hash_zl_v6_struct.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0",
"__aux__" : {
"redis-ver":"6.0.16",
"redis-bits":"64",
"ctime":"1690533464",
"used-mem":"865216",
"aof-preamble":"0"
},

"myhash":["9\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"]
11 changes: 2 additions & 9 deletions test/dumps/multiple_dbs_data.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
[{
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1683103535",
"used-mem":"967040",
"aof-base":"0"
},
{
"x":"0"
},{
"x":"0"
},{
"y":"1"
},{
"z":"2"
Expand Down
13 changes: 8 additions & 5 deletions test/dumps/multiple_lists_strings_data.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0"
},

"string2":"Hi there!",
"mylist1":["v1"],
"mylist3":["v3","v2","v1"],
Expand Down
13 changes: 8 additions & 5 deletions test/dumps/multiple_lists_strings_raw.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0"
},

"string2":"\u0000\tHi there!",
"mylist1":"\u0012\u0001\u0002\u000b\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff",
"mylist3":"\u0012\u0001\u0002\u0013\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff",
Expand Down
13 changes: 8 additions & 5 deletions test/dumps/multiple_lists_strings_struct.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0",
"__aux__" : {
"redis-ver":"255.255.255",
"redis-bits":"64",
"ctime":"1677580558",
"used-mem":"937464",
"aof-base":"0"
},

"string2":"Hi there!",
"mylist1":["\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff"],
"mylist3":["\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff"],
Expand Down
11 changes: 7 additions & 4 deletions test/dumps/plain_hash_data.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
"redis-ver":"3.2.1",
"redis-bits":"64",
"ctime":"1689605183",
"used-mem":"821904",
"__aux__" : {
"redis-ver":"3.2.1",
"redis-bits":"64",
"ctime":"1689605183",
"used-mem":"821904"
},

"myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"}
11 changes: 7 additions & 4 deletions test/dumps/plain_hash_raw.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
"redis-ver":"3.2.1",
"redis-bits":"64",
"ctime":"1689605183",
"used-mem":"821904",
"__aux__" : {
"redis-ver": "3.2.1",
"redis-bits": "64",
"ctime": "1689605183",
"used-mem": "821904"
},

"myhash1":"\u0004\u000b\u00c0\u0005\u00c0\u0005\u00c0\u0006\u00c0\u0006\u00c0\n\u00c0\n\u00c0\u0001\u00c0\u0001\u00c0\u0002\u00c0\u0002\u00c0\b\u00c0\b\u00c0\u0004\u00c0\u0004\u00c0\t\u00c0\t\u00c0\u0003\u00c0\u0003\u00c0\u0007\u00c0\u0007\u00c0\u000b\u00c0\u000b"
Loading

0 comments on commit c444094

Please sign in to comment.