Skip to content

Commit

Permalink
[PPANTT-170] feat: Improved perf test review script
Browse files Browse the repository at this point in the history
  • Loading branch information
@svariant
svariant committed Nov 22, 2024
1 parent 5c59e9c commit 3509bf2
Showing 1 changed file with 77 additions and 71 deletions.
148 changes: 77 additions & 71 deletions performance-test/src/scripts/review_script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,90 +35,96 @@ const reviewIngestionTimeToProcess = async () => {
const notTokenizedIds = await readFromRedisWithKey(REDIS_ARRAY_IDS_NOT_TOKENIZED);
const arrNotTokenizedParsed = JSON.parse(notTokenizedIds);

for (const id of arrTokenizedParsed) {
// RETRIEVE RAW MESSAGE FROM REDIS
console.log("Retrieving from Redis message with id: " + id);
const retrievedMsg = await readFromRedisWithKey(id + REDIS_RAW_SUFFIX);
const rawMsg = JSON.parse(retrievedMsg);
if (rawMsg) {
const rawMsgValue = rawMsg.value;
console.log("Processing raw message with id: " + id);

// CALCULATE TIME TO CAPTURE
let timePsgToRaw = rawMsgValue.ts_ms - rawMsgValue.source.ts_ms;
arrayTimePsgToRaw.push(timePsgToRaw);
totalTimePsgToRaw += timePsgToRaw;
minTimePsgToRaw = minTimePsgToRaw === null || timePsgToRaw < minTimePsgToRaw ? timePsgToRaw : minTimePsgToRaw;
maxTimePsgToRaw = maxTimePsgToRaw === null || timePsgToRaw > maxTimePsgToRaw ? timePsgToRaw : maxTimePsgToRaw;

// RETRIEVE TOKENIZED MESSAGE FROM REDIS WITH RAW OBJ ID
const tokenizedMsg = await readFromRedisWithKey(id + REDIS_ING_SUFFIX);

if (tokenizedMsg) {
const tokenizedMsgValue = JSON.parse(tokenizedMsg);
console.log("Processing tokenized message with id: " + id);

// CALCULATE TIME TO TOKENIZE
let timeRawToTokenize = Number(tokenizedMsgValue.timestamp) - Number(rawMsg.timestamp);
arrayTimeRawToTokenize.push(timeRawToTokenize);
totalTimeRawToTokenize += timeRawToTokenize;
minTimeRawToTokenize = minTimeRawToTokenize === null || timeRawToTokenize < minTimeRawToTokenize ? timeRawToTokenize : minTimeRawToTokenize;
maxTimeRawToTokenize = maxTimeRawToTokenize === null || timeRawToTokenize > maxTimeRawToTokenize ? timeRawToTokenize : maxTimeRawToTokenize;
if(arrTokenizedParsed){
for (const id of arrTokenizedParsed) {
// RETRIEVE RAW MESSAGE FROM REDIS
console.log("Retrieving from Redis message with id: " + id);
const retrievedMsg = await readFromRedisWithKey(id + REDIS_RAW_SUFFIX);
const rawMsg = JSON.parse(retrievedMsg);
if (rawMsg) {
const rawMsgValue = rawMsg.value;
console.log("Processing raw message with id: " + id);

// CALCULATE TIME TO CAPTURE
let timePsgToRaw = rawMsgValue.ts_ms - rawMsgValue.source.ts_ms;
arrayTimePsgToRaw.push(timePsgToRaw);
totalTimePsgToRaw += timePsgToRaw;
minTimePsgToRaw = minTimePsgToRaw === null || timePsgToRaw < minTimePsgToRaw ? timePsgToRaw : minTimePsgToRaw;
maxTimePsgToRaw = maxTimePsgToRaw === null || timePsgToRaw > maxTimePsgToRaw ? timePsgToRaw : maxTimePsgToRaw;

// RETRIEVE TOKENIZED MESSAGE FROM REDIS WITH RAW OBJ ID
const tokenizedMsg = await readFromRedisWithKey(id + REDIS_ING_SUFFIX);

if (tokenizedMsg) {
const tokenizedMsgValue = JSON.parse(tokenizedMsg);
console.log("Processing tokenized message with id: " + id);

// CALCULATE TIME TO TOKENIZE
let timeRawToTokenize = Number(tokenizedMsgValue.timestamp) - Number(rawMsg.timestamp);
arrayTimeRawToTokenize.push(timeRawToTokenize);
totalTimeRawToTokenize += timeRawToTokenize;
minTimeRawToTokenize = minTimeRawToTokenize === null || timeRawToTokenize < minTimeRawToTokenize ? timeRawToTokenize : minTimeRawToTokenize;
maxTimeRawToTokenize = maxTimeRawToTokenize === null || timeRawToTokenize > maxTimeRawToTokenize ? timeRawToTokenize : maxTimeRawToTokenize;
} else {
console.log("Fail to tokenize message with id: " + id);
failedTokenized += 1;
}
} else {
console.log("Fail to tokenize message with id: " + id);
failedTokenized += 1;
console.log("Fail to capture message with id: " + id);
failedRaw += 1;
}
} else {
console.log("Fail to capture message with id: " + id);
failedRaw += 1;
}

}
}

for (const id of arrNotTokenizedParsed) {
// RETRIEVE RAW MESSAGE FROM REDIS
console.log("Retrieving from Redis message with id: " + id);
const retrievedMsg = await readFromRedisWithKey(id + REDIS_RAW_SUFFIX);
const rawMsg = JSON.parse(retrievedMsg);
if (rawMsg) {
const rawMsgValue = rawMsg.value;
console.log("Processing raw message with id: " + id);

// CALCULATE TIME TO CAPTURE
let timePsgToRaw = rawMsgValue.ts_ms - rawMsgValue.source.ts_ms;
arrayTimePsgToRaw.push(timePsgToRaw);
totalTimePsgToRaw += timePsgToRaw;
minTimePsgToRaw = minTimePsgToRaw === null || timePsgToRaw < minTimePsgToRaw ? timePsgToRaw : minTimePsgToRaw;
maxTimePsgToRaw = maxTimePsgToRaw === null || timePsgToRaw > maxTimePsgToRaw ? timePsgToRaw : maxTimePsgToRaw;

// RETRIEVE INGESTED MESSAGE FROM REDIS WITH RAW OBJ ID
const ingestedMsg = await readFromRedisWithKey(id + REDIS_ING_SUFFIX);

if (ingestedMsg) {
const ingestedMsgValue = JSON.parse(ingestedMsg);
console.log("Processing ingested message with id: " + id);

// CALCULATE TIME TO INGEST WITHOUT TOKENIZER
let timeRawToIngest = Number(ingestedMsgValue.timestamp) - Number(rawMsg.timestamp);
arrayTimeRawToIngest.push(timeRawToIngest);
totalTimeRawToIngest += timeRawToIngest;
minTimeRawToIngest = minTimeRawToIngest === null || timeRawToIngest < minTimeRawToIngest ? timeRawToIngest : minTimeRawToIngest;
maxTimeRawToIngest = maxTimeRawToIngest === null || timeRawToIngest > maxTimeRawToIngest ? timeRawToIngest : maxTimeRawToIngest;
if(arrNotTokenizedParsed){
for (const id of arrNotTokenizedParsed) {
// RETRIEVE RAW MESSAGE FROM REDIS
console.log("Retrieving from Redis message with id: " + id);
const retrievedMsg = await readFromRedisWithKey(id + REDIS_RAW_SUFFIX);
const rawMsg = JSON.parse(retrievedMsg);
if (rawMsg) {
const rawMsgValue = rawMsg.value;
console.log("Processing raw message with id: " + id);

// CALCULATE TIME TO CAPTURE
let timePsgToRaw = rawMsgValue.ts_ms - rawMsgValue.source.ts_ms;
arrayTimePsgToRaw.push(timePsgToRaw);
totalTimePsgToRaw += timePsgToRaw;
minTimePsgToRaw = minTimePsgToRaw === null || timePsgToRaw < minTimePsgToRaw ? timePsgToRaw : minTimePsgToRaw;
maxTimePsgToRaw = maxTimePsgToRaw === null || timePsgToRaw > maxTimePsgToRaw ? timePsgToRaw : maxTimePsgToRaw;

// RETRIEVE INGESTED MESSAGE FROM REDIS WITH RAW OBJ ID
const ingestedMsg = await readFromRedisWithKey(id + REDIS_ING_SUFFIX);

if (ingestedMsg) {
const ingestedMsgValue = JSON.parse(ingestedMsg);
console.log("Processing ingested message with id: " + id);

// CALCULATE TIME TO INGEST WITHOUT TOKENIZER
let timeRawToIngest = Number(ingestedMsgValue.timestamp) - Number(rawMsg.timestamp);
arrayTimeRawToIngest.push(timeRawToIngest);
totalTimeRawToIngest += timeRawToIngest;
minTimeRawToIngest = minTimeRawToIngest === null || timeRawToIngest < minTimeRawToIngest ? timeRawToIngest : minTimeRawToIngest;
maxTimeRawToIngest = maxTimeRawToIngest === null || timeRawToIngest > maxTimeRawToIngest ? timeRawToIngest : maxTimeRawToIngest;
} else {
console.log("Fail to ingest message with id: " + id);
failedIngested += 1;
}
} else {
console.log("Fail to ingest message with id: " + id);
failedIngested += 1;
console.log("Fail to capture message with id: " + id);
failedRaw += 1;
}
} else {
console.log("Fail to capture message with id: " + id);
failedRaw += 1;
}
}

console.log("/////////////////////////////////");
console.log("/----------- METRICS -----------/");
console.log("/////////////////////////////////");
console.log("--------------------------------");
console.log(`total messages....................: ${arrTokenizedParsed.length + arrNotTokenizedParsed.length}`);
if(arrNotTokenizedParsed && arrNotTokenizedParsed){
console.log("--------------------------------");
console.log(`total messages....................: ${arrTokenizedParsed.length + arrNotTokenizedParsed.length}`);
}
console.log("--------------------------------");
console.log(`mean time to capture..............: ${totalTimePsgToRaw ? getTimeString(Math.round(totalTimePsgToRaw / arrayTimePsgToRaw.length)) : "-"}`);
console.log(`mean time to tokenize.............: ${totalTimeRawToTokenize ? getTimeString(Math.round(totalTimeRawToTokenize / arrayTimeRawToTokenize.length)) : "-"}`);
Expand Down

0 comments on commit 3509bf2

Please sign in to comment.