Skip to content

Commit

Permalink
Reconfigure benchmarks code to separate it from regular tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@itsmesamster
itsmesamster committed Sep 9, 2024
1 parent 2228c0b commit ba94a92
Show file tree
Hide file tree
Showing 12 changed files with 463 additions and 721 deletions.
105 changes: 13 additions & 92 deletions application/apps/rustcore/ts-bindings/spec/benchmarks.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,131 +2,52 @@
{
"log_level": 1,
"tests": {
"observe": {
"regular": {
/* With numbers you can define, which tests should be executed. "execute_only": [1, 3] will run ONLY tests 1 and 3*/
/* If "execute_only" isn't empty, all performance tests will be ignored*/
"execute_only": [-1],
"list": {
"1": "Test 1. Observe and grab content (text)",
"2": "Test 2. Observe and grab content (pcapng)",
"3": "Test 3. Observe and grab content (dlt)",
"4": "Test 4. Observe and grab content (attachments)",
"5": "Test 5. Observe and grab content (someip from pcapng)",
"6": "Test 6. Observe and grab content (someip from pcapng with fibex)",
"7": "Test 7. Observe and grab content (someip from pcap)",
"8": "Test 8. Observe and grab content (someip from pcap with fibex)"
},
"files": {
}
},
"benchmark": {
"performance": {
/* In false will prevent running performance tests */
"run": true,
"tests": {
"test1": {
"alias": "Observe - grab content (text)",
"open_as": "text",
"file": "test_files/temp_readings3.txt",
"expectation_ms": 10000
},
"test2": {
"alias": "Observe - grab content (pcapng)",
"open_as": "dlt",
"alias": "Observe - grab content (dlt)",
"file": "test_files/FzgProg_SP21.dlt",
"expectation_ms": 60000
},
"test3": {
"alias": "Observe - grab content (dlt)",
"open_as": "pcapng",
"alias": "Observe - grab content (pcapng)",
"file": "test_files/someip.pcapng",
"expectation_ms": 1000
}
}
}
},
"stream": {
"regular": {
"execute_only": [-1],
"list": {
"1": "Test 1. Observe and grab",
"2": "Test 2. Life cycle",
"3": "Test 3. Invalid data source",
"4": "Test 4. Updated stream",
"5": "Test 5. Updated stream search",
"6": "Test 6. Aborting stream",
"7": "Test 7. Multiple stream & SDE test"
},
"files": {
}
},
"performance": {
"run": true,
"tests": {
"test1": {
},
"test4": {
"alias": "Stream - startup measurement",
"file": "",
"expectation_ms": 1000
},
"test2": {
"test5": {
"alias": "Stream - shutdown measurement",
"file": "",
"expectation_ms": 10000
},
"test3": {
"test6": {
"alias": "Stream - Open 50 sessions",
"file": "",
"expectation_ms": 10000
}
}
}
},
"indexes": {
"regular": {
"execute_only": [-1],
"list": {
"1": "Test 1. Switch to breadcrumb mode"
},
"files": {
}
},
"performance": {
/* In false will prevent running performance tests */
"run": true,
"tests": {
"test1": {
},
"test7": {
"alias": "Indexes - Switch to breadcrumb mode",
"file": "test_files/indexing_access_huge.log",
"expectation_ms": 15000
}
}
}
},
"search": {
"regular": {
"execute_only": [-1],
"list": {
"1": "Test 1. Assign & single search",
"2": "Test 2. Assign & multiple search",
"3": "Test 3. Assign & zero search",
"4": "Test 4. Assign & single not case sensitive search",
"5": "Test 5. Assign & single word search",
"6": "Test 6. Assign & single search with crossing terms",
"7": "Test 7. Assign & repeated search"
},
"files": {
}
},
"performance": {
/* In false will prevent running performance tests */
"run": true,
"tests": {
"test1": {
},
"test8": {
"alias": "Assign & single search",
"file": "test_files/indexing_access_huge.log",
"expectation_ms": 1000
},
"test2": {
"test9": {
"alias": "Assign & multiple search",
"file": "test_files/indexing_access_huge.log",
"expectation_ms": 10000
Expand All @@ -135,4 +56,4 @@
}
}
}
}
}
68 changes: 58 additions & 10 deletions application/apps/rustcore/ts-bindings/spec/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { Logger, getLogger } from './logger';
import { error, numToLogLevel } from 'platform/log/utils';
import { state } from 'platform/log';
import { IRegularTests } from './config';
import { IPerformanceTest } from './config_benchmarks';

import * as tmp from 'tmp';
import * as fs from 'fs';
Expand All @@ -25,7 +26,7 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 900000;
export type ScopeInjector<T> = (s: T) => T;

export function runner(
config: IRegularTests,
config: IRegularTests | IPerformanceTest,
id: string | number,
test: (
logger: Logger,
Expand All @@ -38,17 +39,33 @@ export function runner(
scope.push(obj);
return obj;
};
const name = config.list[id];

let name: string;
let shouldExecute = true;

if ('list' in config) {
// Handling IRegularTests
name = config.list[id];
shouldExecute = config.execute_only.length === 0 || config.execute_only.includes(typeof id === 'number' ? id : parseInt(id, 10));
} else if ('alias' in config) {
// Handling IPerformanceTest
name = config.alias;
shouldExecute = !config.ignore;
} else {
// Log the type of config received
console.error('Invalid configuration passed to runner. Config:', config);
return Promise.reject(new Error('Invalid configuration passed to runner'));
}

const logger = getLogger(name);
if (
config.execute_only.length > 0 &&
config.execute_only.indexOf(typeof id === 'number' ? id : parseInt(id, 10)) === -1
) {

if (!shouldExecute) {
console.log(`\nIgnored: ${name}`);
return Promise.resolve();
} else {
console.log(`\nStarted: ${name}`);
}

return new Promise((done) => {
try {
test(logger, done, injector).catch((err: Error) => {
Expand All @@ -60,6 +77,41 @@ export function runner(
});
}

export function readConfigFile<T>(filenameEnvVar: string, defaultPaths: string[]): T | Error {
const defaults = (() => {
for (const target of defaultPaths) {
if (fs.existsSync(target)) {
return target;
}
}
return undefined;
})();

let filename = (process.env as any)[filenameEnvVar];
if ((typeof filename !== 'string' || filename.trim() === '') && defaults === undefined) {
return new Error(
`To run test you should define a path to configuration file with ${filenameEnvVar}=path_to_config_json_file`,
);
} else if (typeof filename !== 'string' || filename.trim() === '') {
filename = defaults;
}

if (!fs.existsSync(filename)) {
return new Error(`Configuration file ${filename} doesn't exist`);
}

const buffer = fs.readFileSync(filename);
try {
return JSON.parse(buffer.toString().replace(/\/\*.*\*\//gi, '')) as T;
} catch (err) {
return new Error(
`Fail to parse configuration file ${filename}; error: ${
err instanceof Error ? err.message : err
}`,
);
}
}

export function finish(
sessions: Array<Session | Jobs | Tracker | undefined> | Session | Jobs | Tracker | undefined,
done: (...args: any[]) => void,
Expand Down Expand Up @@ -271,7 +323,3 @@ export function setMeasurement(): () => ITimeMeasurement {
};
};
}

export function helloWorld() {
return 'Hello, World!';
}
66 changes: 8 additions & 58 deletions application/apps/rustcore/ts-bindings/spec/config.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { readConfigFile } from './common';

export interface IPerformanceTest {
open_as: 'text' | 'dlt' | 'pcapng';
ignore: boolean;
alias: string;
expectation_ms: number;
file: string;
}
export interface ICancelTestSpec {
terms: string[];
interval_ms: number;
Expand Down Expand Up @@ -36,24 +30,12 @@ export interface IConfiguration {
tests: {
observe: {
regular: IRegularTests;
performance: {
run: boolean;
tests: { [key: string]: IPerformanceTest };
};
};
stream: {
regular: IRegularTests;
performance: {
run: boolean;
tests: { [key: string]: IPerformanceTest };
};
};
indexes: {
regular: IRegularTests;
performance: {
run: boolean;
tests: { [key: string]: IPerformanceTest };
};
};
jobs: {
regular: IRegularTests;
Expand All @@ -63,10 +45,6 @@ export interface IConfiguration {
};
search: {
regular: IRegularTests;
performance: {
run: boolean;
tests: { [key: string]: IPerformanceTest };
};
};
values: {
regular: IRegularTests;
Expand Down Expand Up @@ -99,40 +77,11 @@ export interface IConfiguration {
}

export function readConfigurationFile(): Config {
const config = (() => {
const defaults = (() => {
for (const target of [
path.resolve(path.dirname(module.filename), 'defaults.json'),
path.resolve(path.dirname(module.filename), '../../defaults.json'),
]) {
if (fs.existsSync(target)) {
return target;
}
}
return undefined;
})();
let filename = (process.env as any)['JASMIN_TEST_CONFIGURATION'];
if ((typeof filename !== 'string' || filename.trim() === '') && defaults === undefined) {
return new Error(
`To run test you should define a path to configuration file with JASMIN_TEST_CONFIGURATION=path_to_config_json_file`,
);
} else if (typeof filename !== 'string' || filename.trim() === '') {
filename = defaults;
}
if (!fs.existsSync(filename)) {
return new Error(`Configuration file ${filename} doesn't exist`);
}
const buffer = fs.readFileSync(filename);
try {
return new Config(JSON.parse(buffer.toString().replace(/\/\*.*\*\//gi, '')));
} catch (err) {
return new Error(
`Fail to parse configuration file ${filename}; error: ${
err instanceof Error ? err.message : err
}`,
);
}
})();
const config = readConfigFile<IConfiguration>('JASMIN_TEST_CONFIGURATION', [
path.resolve(path.dirname(module.filename), 'defaults.json'),
path.resolve(path.dirname(module.filename), '../../defaults.json'),
]);

if (config instanceof Error) {
console.warn(`\n`);
console.warn(`=`.repeat(81));
Expand All @@ -146,9 +95,10 @@ export function readConfigurationFile(): Config {
console.warn(`\n`);
process.exit(1);
} else {
return config;
return new Config(config);
}
}

export class Config {
private readonly _config: IConfiguration;

Expand Down
Loading

0 comments on commit ba94a92

Please sign in to comment.