From ba94a92221ca778c362a65e4a644f4cb46f65b99 Mon Sep 17 00:00:00 2001 From: Sameer Srivastava Date: Tue, 27 Aug 2024 12:50:50 +0200 Subject: [PATCH] Reconfigure benchmarks code to separate it from regular tests --- .../rustcore/ts-bindings/spec/benchmarks.json | 105 +----- .../apps/rustcore/ts-bindings/spec/common.ts | 68 +++- .../apps/rustcore/ts-bindings/spec/config.ts | 66 +--- .../ts-bindings/spec/config_benchmarks.ts | 57 ++++ .../rustcore/ts-bindings/spec/defaults.json | 25 -- .../spec/session.benchmark.spec.ts | 307 +++++++++++++++++- .../ts-bindings/spec/session.indexes.spec.ts | 114 ------- .../ts-bindings/spec/session.observe.spec.ts | 142 -------- .../ts-bindings/spec/session.search.spec.ts | 141 -------- .../ts-bindings/spec/session.stream.spec.ts | 124 ------- scripts/elements/bindings.rb | 28 +- scripts/tools/run_benchmarks.rb | 7 +- 12 files changed, 463 insertions(+), 721 deletions(-) create mode 100644 application/apps/rustcore/ts-bindings/spec/config_benchmarks.ts diff --git a/application/apps/rustcore/ts-bindings/spec/benchmarks.json b/application/apps/rustcore/ts-bindings/spec/benchmarks.json index b0f5bf4d8d..ac3ee3089c 100644 --- a/application/apps/rustcore/ts-bindings/spec/benchmarks.json +++ b/application/apps/rustcore/ts-bindings/spec/benchmarks.json @@ -2,131 +2,52 @@ { "log_level": 1, "tests": { - "observe": { - "regular": { - /* With numbers you can define, which tests should be executed. "execute_only": [1, 3] will run ONLY tests 1 and 3*/ - /* If "execute_only" isn't empty, all performance tests will be ignored*/ - "execute_only": [-1], - "list": { - "1": "Test 1. Observe and grab content (text)", - "2": "Test 2. Observe and grab content (pcapng)", - "3": "Test 3. Observe and grab content (dlt)", - "4": "Test 4. Observe and grab content (attachments)", - "5": "Test 5. Observe and grab content (someip from pcapng)", - "6": "Test 6. Observe and grab content (someip from pcapng with fibex)", - "7": "Test 7. Observe and grab content (someip from pcap)", - "8": "Test 8. Observe and grab content (someip from pcap with fibex)" - }, - "files": { - } - }, + "benchmark": { "performance": { /* In false will prevent running performance tests */ "run": true, "tests": { "test1": { "alias": "Observe - grab content (text)", - "open_as": "text", "file": "test_files/temp_readings3.txt", "expectation_ms": 10000 }, "test2": { - "alias": "Observe - grab content (pcapng)", - "open_as": "dlt", + "alias": "Observe - grab content (dlt)", "file": "test_files/FzgProg_SP21.dlt", "expectation_ms": 60000 }, "test3": { - "alias": "Observe - grab content (dlt)", - "open_as": "pcapng", + "alias": "Observe - grab content (pcapng)", "file": "test_files/someip.pcapng", "expectation_ms": 1000 - } - } - } - }, - "stream": { - "regular": { - "execute_only": [-1], - "list": { - "1": "Test 1. Observe and grab", - "2": "Test 2. Life cycle", - "3": "Test 3. Invalid data source", - "4": "Test 4. Updated stream", - "5": "Test 5. Updated stream search", - "6": "Test 6. Aborting stream", - "7": "Test 7. Multiple stream & SDE test" - }, - "files": { - } - }, - "performance": { - "run": true, - "tests": { - "test1": { + }, + "test4": { "alias": "Stream - startup measurement", "file": "", "expectation_ms": 1000 }, - "test2": { + "test5": { "alias": "Stream - shutdown measurement", "file": "", "expectation_ms": 10000 }, - "test3": { + "test6": { "alias": "Stream - Open 50 sessions", "file": "", "expectation_ms": 10000 - } - } - } - }, - "indexes": { - "regular": { - "execute_only": [-1], - "list": { - "1": "Test 1. Switch to breadcrumb mode" - }, - "files": { - } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": true, - "tests": { - "test1": { + }, + "test7": { "alias": "Indexes - Switch to breadcrumb mode", "file": "test_files/indexing_access_huge.log", "expectation_ms": 15000 - } - } - } - }, - "search": { - "regular": { - "execute_only": [-1], - "list": { - "1": "Test 1. Assign & single search", - "2": "Test 2. Assign & multiple search", - "3": "Test 3. Assign & zero search", - "4": "Test 4. Assign & single not case sensitive search", - "5": "Test 5. Assign & single word search", - "6": "Test 6. Assign & single search with crossing terms", - "7": "Test 7. Assign & repeated search" - }, - "files": { - } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": true, - "tests": { - "test1": { + }, + "test8": { "alias": "Assign & single search", "file": "test_files/indexing_access_huge.log", "expectation_ms": 1000 }, - "test2": { + "test9": { "alias": "Assign & multiple search", "file": "test_files/indexing_access_huge.log", "expectation_ms": 10000 @@ -135,4 +56,4 @@ } } } -} +} \ No newline at end of file diff --git a/application/apps/rustcore/ts-bindings/spec/common.ts b/application/apps/rustcore/ts-bindings/spec/common.ts index 50a930abf5..d59fcee6c9 100644 --- a/application/apps/rustcore/ts-bindings/spec/common.ts +++ b/application/apps/rustcore/ts-bindings/spec/common.ts @@ -9,6 +9,7 @@ import { Logger, getLogger } from './logger'; import { error, numToLogLevel } from 'platform/log/utils'; import { state } from 'platform/log'; import { IRegularTests } from './config'; +import { IPerformanceTest } from './config_benchmarks'; import * as tmp from 'tmp'; import * as fs from 'fs'; @@ -25,7 +26,7 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 900000; export type ScopeInjector = (s: T) => T; export function runner( - config: IRegularTests, + config: IRegularTests | IPerformanceTest, id: string | number, test: ( logger: Logger, @@ -38,17 +39,33 @@ export function runner( scope.push(obj); return obj; }; - const name = config.list[id]; + + let name: string; + let shouldExecute = true; + + if ('list' in config) { + // Handling IRegularTests + name = config.list[id]; + shouldExecute = config.execute_only.length === 0 || config.execute_only.includes(typeof id === 'number' ? id : parseInt(id, 10)); + } else if ('alias' in config) { + // Handling IPerformanceTest + name = config.alias; + shouldExecute = !config.ignore; + } else { + // Log the type of config received + console.error('Invalid configuration passed to runner. Config:', config); + return Promise.reject(new Error('Invalid configuration passed to runner')); + } + const logger = getLogger(name); - if ( - config.execute_only.length > 0 && - config.execute_only.indexOf(typeof id === 'number' ? id : parseInt(id, 10)) === -1 - ) { + + if (!shouldExecute) { console.log(`\nIgnored: ${name}`); return Promise.resolve(); } else { console.log(`\nStarted: ${name}`); } + return new Promise((done) => { try { test(logger, done, injector).catch((err: Error) => { @@ -60,6 +77,41 @@ export function runner( }); } +export function readConfigFile(filenameEnvVar: string, defaultPaths: string[]): T | Error { + const defaults = (() => { + for (const target of defaultPaths) { + if (fs.existsSync(target)) { + return target; + } + } + return undefined; + })(); + + let filename = (process.env as any)[filenameEnvVar]; + if ((typeof filename !== 'string' || filename.trim() === '') && defaults === undefined) { + return new Error( + `To run test you should define a path to configuration file with ${filenameEnvVar}=path_to_config_json_file`, + ); + } else if (typeof filename !== 'string' || filename.trim() === '') { + filename = defaults; + } + + if (!fs.existsSync(filename)) { + return new Error(`Configuration file ${filename} doesn't exist`); + } + + const buffer = fs.readFileSync(filename); + try { + return JSON.parse(buffer.toString().replace(/\/\*.*\*\//gi, '')) as T; + } catch (err) { + return new Error( + `Fail to parse configuration file ${filename}; error: ${ + err instanceof Error ? err.message : err + }`, + ); + } +} + export function finish( sessions: Array | Session | Jobs | Tracker | undefined, done: (...args: any[]) => void, @@ -271,7 +323,3 @@ export function setMeasurement(): () => ITimeMeasurement { }; }; } - -export function helloWorld() { - return 'Hello, World!'; -} diff --git a/application/apps/rustcore/ts-bindings/spec/config.ts b/application/apps/rustcore/ts-bindings/spec/config.ts index ebd1ed0ffa..4db723d2b7 100644 --- a/application/apps/rustcore/ts-bindings/spec/config.ts +++ b/application/apps/rustcore/ts-bindings/spec/config.ts @@ -1,13 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; +import { readConfigFile } from './common'; -export interface IPerformanceTest { - open_as: 'text' | 'dlt' | 'pcapng'; - ignore: boolean; - alias: string; - expectation_ms: number; - file: string; -} export interface ICancelTestSpec { terms: string[]; interval_ms: number; @@ -36,24 +30,12 @@ export interface IConfiguration { tests: { observe: { regular: IRegularTests; - performance: { - run: boolean; - tests: { [key: string]: IPerformanceTest }; - }; }; stream: { regular: IRegularTests; - performance: { - run: boolean; - tests: { [key: string]: IPerformanceTest }; - }; }; indexes: { regular: IRegularTests; - performance: { - run: boolean; - tests: { [key: string]: IPerformanceTest }; - }; }; jobs: { regular: IRegularTests; @@ -63,10 +45,6 @@ export interface IConfiguration { }; search: { regular: IRegularTests; - performance: { - run: boolean; - tests: { [key: string]: IPerformanceTest }; - }; }; values: { regular: IRegularTests; @@ -99,40 +77,11 @@ export interface IConfiguration { } export function readConfigurationFile(): Config { - const config = (() => { - const defaults = (() => { - for (const target of [ - path.resolve(path.dirname(module.filename), 'defaults.json'), - path.resolve(path.dirname(module.filename), '../../defaults.json'), - ]) { - if (fs.existsSync(target)) { - return target; - } - } - return undefined; - })(); - let filename = (process.env as any)['JASMIN_TEST_CONFIGURATION']; - if ((typeof filename !== 'string' || filename.trim() === '') && defaults === undefined) { - return new Error( - `To run test you should define a path to configuration file with JASMIN_TEST_CONFIGURATION=path_to_config_json_file`, - ); - } else if (typeof filename !== 'string' || filename.trim() === '') { - filename = defaults; - } - if (!fs.existsSync(filename)) { - return new Error(`Configuration file ${filename} doesn't exist`); - } - const buffer = fs.readFileSync(filename); - try { - return new Config(JSON.parse(buffer.toString().replace(/\/\*.*\*\//gi, ''))); - } catch (err) { - return new Error( - `Fail to parse configuration file ${filename}; error: ${ - err instanceof Error ? err.message : err - }`, - ); - } - })(); + const config = readConfigFile('JASMIN_TEST_CONFIGURATION', [ + path.resolve(path.dirname(module.filename), 'defaults.json'), + path.resolve(path.dirname(module.filename), '../../defaults.json'), + ]); + if (config instanceof Error) { console.warn(`\n`); console.warn(`=`.repeat(81)); @@ -146,9 +95,10 @@ export function readConfigurationFile(): Config { console.warn(`\n`); process.exit(1); } else { - return config; + return new Config(config); } } + export class Config { private readonly _config: IConfiguration; diff --git a/application/apps/rustcore/ts-bindings/spec/config_benchmarks.ts b/application/apps/rustcore/ts-bindings/spec/config_benchmarks.ts new file mode 100644 index 0000000000..ed0a53c382 --- /dev/null +++ b/application/apps/rustcore/ts-bindings/spec/config_benchmarks.ts @@ -0,0 +1,57 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { readConfigFile } from './common'; + +export interface IPerformanceTest { + open_as: 'text' | 'dlt' | 'pcapng' | ''; + ignore: boolean; + alias: string; + expectation_ms: number; + file: string; +} + +export interface IConfiguration { + log_level: number; + tests: { + benchmark: { + performance: { + tests: { [key: string]: IPerformanceTest }; + }; + }; + }; +} + +export function readBenchmarkConfigurationFile(): Config { + const config = readConfigFile('JASMIN_TEST_CONFIGURATION', [ + path.resolve(path.dirname(module.filename), 'benchmarks.json'), + path.resolve(path.dirname(module.filename), '../../benchmarks.json'), + ]); + + if (config instanceof Error) { + console.warn(`\n`); + console.warn(`=`.repeat(81)); + console.warn(`**** ERROR ${'*'.repeat(68)}`); + console.warn(`=`.repeat(81)); + console.warn(`Fail to read configuration file due error: ${config.message}`); + console.warn( + `Test will be done in the scope of tasks, which do not require configuration.`, + ); + console.warn(`=`.repeat(81)); + console.warn(`\n`); + process.exit(1); + } else { + return new Config(config); + } +} + +export class Config { + private readonly _config: IConfiguration; + + constructor(config: IConfiguration) { + this._config = config; + } + + public get(): IConfiguration { + return this._config; + } +} diff --git a/application/apps/rustcore/ts-bindings/spec/defaults.json b/application/apps/rustcore/ts-bindings/spec/defaults.json index d4ad0746bc..4e4b6e4f29 100644 --- a/application/apps/rustcore/ts-bindings/spec/defaults.json +++ b/application/apps/rustcore/ts-bindings/spec/defaults.json @@ -5,7 +5,6 @@ "observe": { "regular": { /* With numbers you can define, which tests should be executed. "execute_only": [1, 3] will run ONLY tests 1 and 3*/ - /* If "execute_only" isn't empty, all performance tests will be ignored*/ "execute_only": [1, 4, 5, 6, 7, 8], "list": { "1": "Test 1. Observe and grab content (text)", @@ -25,12 +24,6 @@ "someip-pcap": "../../../../application/developing/resources/someip.pcap", "someip-fibex": "../../../../application/developing/resources/someip.xml" } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": false, - "tests": { - } } }, "stream": { @@ -47,12 +40,6 @@ }, "files": { } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": false, - "tests": { - } } }, "indexes": { @@ -63,12 +50,6 @@ }, "files": { } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": false, - "tests": { - } } }, "jobs": { @@ -124,12 +105,6 @@ "6": "Test 6. Assign & single search with crossing terms", "7": "Test 7. Assign & repeated search" } - }, - "performance": { - /* In false will prevent running performance tests */ - "run": false, - "tests": { - } } }, "values": { diff --git a/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts index f7897034ef..49f0e82142 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.benchmark.spec.ts @@ -1,9 +1,306 @@ +// tslint:disable + +// We need to provide path to TypeScript types definitions +/// +/// import { initLogger } from './logger'; initLogger(); -import { helloWorld } from './common'; +import { Session, Factory } from '../src/api/session'; +import { IAttachmentsUpdatedUpdated } from '../src/api/session.provider'; +import { IAttachment } from 'platform/types/content'; +import { createSampleFile, finish, performanceReport, setMeasurement, runner } from './common'; +import { readBenchmarkConfigurationFile } from './config_benchmarks'; +import { IndexingMode } from 'platform/types/content'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const config = readBenchmarkConfigurationFile().get().tests.benchmark; + +describe('Benchmark Tests', function () { + Object.keys(config.performance.tests).forEach((alias: string) => { + const test = config.performance.tests[alias]; + const testName = `${test.alias}`; + if (test.ignore) { + console.log(`Test "${testName}" has been ignored`); + return; + } + + it(testName, function () { + return runner( + { + open_as: '', + ignore: false, + alias: testName, + expectation_ms: 10000, + file: '', + }, + 1, + async (logger, done, collector) => { + const measurement = setMeasurement(); + try { + const session = await Session.create(); + session.debug(true, testName); + + const stream = session.getStream(); + if (stream instanceof Error) throw stream; + + const events = session.getEvents(); + if (events instanceof Error) throw events; + + const search = session.getSearch(); + + let home_dir = (process.env as any)['SH_HOME_DIR']; + + // Handle based on the test type + switch (test.alias) { + case 'Observe - grab content (text)': + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get() + .sterilized(), + ) + .catch(finish.bind(null, session, done)); + break; + case 'Observe - grab content (dlt)': + stream + .observe( + new Factory.File() + .type(Factory.FileType.Binary) + .file(`${home_dir}/${test.file}`) + .asDlt({ + filter_config: undefined, + fibex_file_paths: [], + with_storage_header: true, + tz: undefined, + }) + .get() + .sterilized(), + ) + .catch(finish.bind(null, session, done)); + break; + case 'Observe - grab content (pcapng)': + stream + .observe( + new Factory.File() + .type(Factory.FileType.PcapNG) + .file(`${home_dir}/${test.file}`) + .asDlt({ + filter_config: undefined, + fibex_file_paths: [], + with_storage_header: false, + tz: undefined, + }) + .get() + .sterilized(), + ) + .catch(finish.bind(null, session, done)); + break; + case 'Stream - startup measurement': + const tmpobj1 = createSampleFile( + 5000, + logger, + (i: number) => `some line data: ${i}\n` + ); + + const startupSession = await Session.create(); + startupSession.debug(true, testName); + + const startupStream = startupSession.getStream(); + if (startupStream instanceof Error) throw startupStream; + + const startupEvents = session.getEvents(); + if (startupEvents instanceof Error) throw startupEvents; + startupStream.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${tmpobj1.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ); + const startupResults = measurement(); + const startupReport = performanceReport(testName, startupResults.ms, test.expectation_ms, `${home_dir}/${test.file}`); + finish(startupSession, done, startupReport ? undefined : new Error(`${testName} is fail`)); + break; + case 'Stream - shutdown measurement': + const tmpobj2 = createSampleFile( + 5000, + logger, + (i: number) => `some line data: ${i}\n` + ); + + const shutdownSession = await Session.create(); + shutdownSession.debug(true, testName); + + const shutdownStream = shutdownSession.getStream(); + if (shutdownStream instanceof Error) throw shutdownStream; + + shutdownStream.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${tmpobj2.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ); + const shutdownResults = measurement(); + const shutdownReport = performanceReport(testName, shutdownResults.ms, test.expectation_ms, `${home_dir}/${test.file}`); + finish(shutdownSession, done, shutdownReport ? undefined : new Error(`${testName} is fail`)); + break; + case 'Stream - Open 50 sessions': + const results = []; + const multiSessions = []; + for (let i = 0; i < 50; i++) { + const file = createSampleFile( + 100, + logger, + (j: number) => `file ${i} line data: ${j}\n` + ); + + const multiSession = await Session.create(); + multiSessions.push(multiSession); + + const multiStream = multiSession.getStream(); + if (multiStream instanceof Error) { + throw multiStream; + } + + let result = multiStream.observe( + new Factory.Stream() + .asText() + .process({ + command: `less ${file.name}`, + cwd: process.cwd(), + envs: process.env as { [key: string]: string }, + }) + .get() + .sterilized() + ).catch((err) => `File ${i} failed to open: ${err.message}`); + results.push(result); + } + const multiResults = measurement(); + const multiReport = performanceReport(testName, multiResults.ms, test.expectation_ms, `${home_dir}/${test.file}`); + finish(undefined, done, multiReport ? undefined : new Error(`${testName} is fail`)); + break; + case 'Indexes - Switch to breadcrumb mode': + let controlSum = 0; + let countMatches = 0; + let read: boolean = false; + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get().sterilized(), + ) + .catch(finish.bind(null, session, done)); + const updates: number[] = []; + events.IndexedMapUpdated.subscribe((event) => { + event.len > 0 && updates.push(event.len); + }); + events.StreamUpdated.subscribe(async () => { + read = true; + try { + await search.search([ + { + filter: 'HTTP', + flags: { reg: true, word: true, cases: false }, + }, + ]); + let items = await stream.grabIndexed(0, countMatches); + await stream.setIndexingMode(IndexingMode.Breadcrumbs); + finish(session, done); + } catch (err) { + finish( + undefined, + done, + new Error( + `Fail to finish test due error: ${ + err instanceof Error ? err.message : err + }`, + ), + ); + } + }); + break; + case 'Assign & single search': + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get() + .sterilized(), + ) + .on('processing', () => { + search + .search([ + { + filter: 'http', + flags: { reg: true, word: false, cases: false }, + }, + ]) + .catch(finish.bind(null, session, done)); + }) + .catch(finish.bind(null, session, done)); + break; + case 'Assign & multiple search': + stream + .observe( + new Factory.File() + .asText() + .type(Factory.FileType.Text) + .file(`${home_dir}/${test.file}`) + .get() + .sterilized(), + ) + .on('processing', () => { + search + .search([ + { + filter: 'http://www.almhuette-raith.at', + flags: { reg: true, word: false, cases: false }, + }, + { + filter: 'com.apple.hiservices-xpcservice', + flags: { reg: true, word: false, cases: false }, + }, + { + filter: 'Google Chrome Helper', + flags: { reg: true, word: false, cases: false }, + }, + ]) + .catch(finish.bind(null, session, done)); + }) + .catch(finish.bind(null, session, done)); + break; + default: + throw new Error(`Unsupported format or alias: ${test.alias}`); + } + events.FileRead.subscribe(() => { + const results = measurement(); + const reportResult = performanceReport(testName, results.ms, test.expectation_ms, `${home_dir}/${test.file}`); + finish(session, done, reportResult ? undefined : new Error(`${testName} is fail`)); + }); -describe('Hello World', () => { - it('should return "Hello, World!"', () => { - expect(helloWorld()).toEqual('Hello, World!'); - }); + } catch (err) { + finish(undefined, done, new Error(`Failed to complete test "${testName}" due to error: ${err instanceof Error ? err.message : err}`)); + } + }, + ); + }); + }); }); \ No newline at end of file diff --git a/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts index 9ae2ebf144..ef3126f036 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.indexes.spec.ts @@ -288,118 +288,4 @@ describe('Indexes', function () { }); }); }); - - - - config.performance.run && - Object.keys(config.regular.execute_only).length >= 0 && - Object.keys(config.performance.tests).forEach((alias: string, index: number) => { - const test = (config.performance.tests as any)[alias]; - const testName = `${test.alias}`; - if (test.ignore) { - console.log(`Test "${testName}" has been ignored`); - return; - } - it(testName, function () { - return runner( - { - list: { 1: testName }, - execute_only: [], - files: {}, - }, - 1, - async (logger, done, collector) => { - const measurement = setMeasurement(); - Session.create() - .then((session: Session) => { - // Set provider into debug mode - session.debug(true, testName); - const stream = session.getStream(); - if (stream instanceof Error) { - finish(session, done, stream); - return; - } - const search = session.getSearch(); - if (search instanceof Error) { - finish(session, done, search); - return; - } - const events = session.getEvents(); - if (events instanceof Error) { - finish(session, done, events); - return; - } - let controlSum = 0; - let countMatches = 0; - let read: boolean = false; - let home_dir = (process.env as any)['SH_HOME_DIR']; - stream - .observe( - new Factory.File() - .asText() - .type(Factory.FileType.Text) - .file(`${home_dir}/${test.file}`) - .get().sterilized(), - ) - .catch(finish.bind(null, session, done)); - const updates: number[] = []; - events.IndexedMapUpdated.subscribe((event) => { - event.len > 0 && updates.push(event.len); - }); - events.StreamUpdated.subscribe(async () => { - read = true; - try { - await search.search([ - { - filter: 'HTTP', - flags: { reg: true, word: true, cases: false }, - }, - ]); - let items = await stream.grabIndexed(0, countMatches); - await stream.setIndexingMode(IndexingMode.Breadcrumbs); - finish(session, done); - } catch (err) { - finish( - undefined, - done, - new Error( - `Fail to finish test due error: ${ - err instanceof Error ? err.message : err - }`, - ), - ); - } - }); - events.FileRead.subscribe(() => { - const results = measurement(); - finish( - session, - done, - performanceReport( - testName, - results.ms, - test.expectation_ms, - `${home_dir}/${test.file}`, - ) - ? undefined - : new Error(`${testName} is fail`), - ); - }); - }) - .catch((err: Error) => { - finish( - undefined, - done, - new Error( - `Fail to create session due error: ${ - err instanceof Error ? err.message : err - }`, - ), - ); - }); - }, - ); - }); - }); - }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts index df2fee5d61..917e6da118 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.observe.spec.ts @@ -795,146 +795,4 @@ describe('Observe', function () { }); }); }); - - config.performance.run && - Object.keys(config.regular.execute_only).length > 0 && - Object.keys(config.performance.tests).forEach((alias: string, index: number) => { - const test = (config.performance.tests as any)[alias]; - const testName = `${test.alias}`; - if (test.ignore) { - console.log(`Test "${testName}" has been ignored`); - return; - } - it(testName, function () { - return runner( - { - list: { 1: testName }, - execute_only: [], - files: {}, - }, - 1, - async (logger, done, collector) => { - const measurement = setMeasurement(); - Session.create() - .then((session: Session) => { - // Set provider into debug mode - session.debug(true, testName); - const stream = session.getStream(); - if (stream instanceof Error) { - finish(session, done, stream); - return; - } - const events = session.getEvents(); - if (events instanceof Error) { - finish(session, done, events); - return; - } - let home_dir = (process.env as any)['SH_HOME_DIR']; - switch (test.open_as) { - case 'text': - stream - .observe( - new Factory.File() - .asText() - .type(Factory.FileType.Text) - .file(`${home_dir}/${test.file}`) - .get() - .sterilized(), - ) - .catch(finish.bind(null, session, done)); - break; - case 'dlt': - stream - .observe( - new Factory.File() - .type(Factory.FileType.Binary) - .file(`${home_dir}/${test.file}`) - .asDlt({ - filter_config: undefined, - fibex_file_paths: [], - with_storage_header: true, - tz: undefined, - }) - .get() - .sterilized(), - ) - .catch(finish.bind(null, session, done)); - break; - case 'pcapng': - stream - .observe( - new Factory.File() - .type(Factory.FileType.PcapNG) - .file(`${home_dir}/${test.file}`) - .asDlt({ - filter_config: undefined, - fibex_file_paths: [], - with_storage_header: false, - tz: undefined, - }) - .get() - .sterilized(), - ) - .catch(finish.bind(null, session, done)); - break; - case 'startup_measurement': - const tmpobj = createSampleFile( - 5000, - logger, - (i: number) => `some line data: ${i}\n`, - ); - stream - .observe( - new Factory.Stream() - .asText() - .process({ - command: `less ${tmpobj.name}`, - cwd: process.cwd(), - envs: process.env as { [key: string]: string }, - }) - .get() - .sterilized(), - ) - .catch(finish.bind(null, session, done)); - break; - default: - finish( - undefined, - done, - new Error(`Unsupported format: ${test.open_as}`), - ); - return; - } - events.FileRead.subscribe(() => { - const results = measurement(); - finish( - session, - done, - performanceReport( - testName, - results.ms, - test.expectation_ms, - `${home_dir}/${test.file}`, - ) - ? undefined - : new Error(`${testName} is fail`), - ); - }); - }) - .catch((err: Error) => { - finish( - undefined, - done, - new Error( - `Fail to create session due error: ${ - err instanceof Error ? err.message : err - }`, - ), - ); - }); - }, - ); - }); - }); - }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts index 9d3592bc85..630fdded65 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.search.spec.ts @@ -1002,145 +1002,4 @@ describe('Search', function () { }); }); }); - - config.performance.run && - Object.keys(config.regular.execute_only).length > 0 && - Object.keys(config.performance.tests).forEach((alias: string, index: number) => { - const test = (config.performance.tests as any)[alias]; - const testName = `${test.alias}`; - if (test.ignore) { - console.log(`Test "${testName}" has been ignored`); - return; - } - it(testName, function () { - return runner( - { - list: { 1: testName }, - execute_only: [], - files: {}, - }, - 1, - async (logger, done, collector) => { - const measurement = setMeasurement(); - Session.create() - .then((session: Session) => { - // Set provider into debug mode - session.debug(true, testName); - const stream = session.getStream(); - if (stream instanceof Error) { - finish(session, done, stream); - return; - } - const events = session.getEvents(); - if (events instanceof Error) { - finish(session, done, events); - return; - } - const search = session.getSearch(); - if (search instanceof Error) { - finish(session, done, search); - return; - } - let home_dir = (process.env as any)['SH_HOME_DIR']; - switch (index+1) { - case 1: - stream - .observe( - new Factory.File() - .asText() - .type(Factory.FileType.Text) - .file(`${home_dir}/${test.file}`) - .get() - .sterilized(), - ) - .on('processing', () => { - search - .search([ - { - filter: 'http', - flags: { reg: true, word: false, cases: false }, - }, - ]) - .catch(finish.bind(null, session, done)); - }) - .catch(finish.bind(null, session, done)); - break; - case 2: - stream - .observe( - new Factory.File() - .asText() - .type(Factory.FileType.Text) - .file(`${home_dir}/${test.file}`) - .get() - .sterilized(), - ) - .on('processing', () => { - search - .search([ - { - filter: 'http://www.almhuette-raith.at', - flags: { reg: true, word: false, cases: false }, - }, - { - filter: 'com.apple.hiservices-xpcservice', - flags: { reg: true, word: false, cases: false }, - }, - { - filter: 'Google Chrome Helper', - flags: { reg: true, word: false, cases: false }, - }, - ]) - .catch(finish.bind(null, session, done)); - }) - .catch(finish.bind(null, session, done)); - break; - default: - finish( - undefined, - done, - new Error(`Unsupported format: ${test.open_as}`), - ); - return; - } - events.FileRead.subscribe(() => { - const measurement = setMeasurement(); - const search = session.getSearch(); - if (search instanceof Error) { - finish(session, done, search); - return; - } - search.search([]).then((_maches: number) => { - const results = measurement(); - finish( - session, - done, - performanceReport( - testName, - results.ms, - test.expectation_ms, - `${home_dir}/${test.file}`, - ) - ? undefined - : new Error(`${testName} is fail`), - ); - }); - }); - }) - .catch((err: Error) => { - finish( - undefined, - done, - new Error( - `Fail to create session due error: ${ - err instanceof Error ? err.message : err - }`, - ), - ); - }); - }, - ); - }); - }); - }); diff --git a/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts b/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts index 9a26f9b88f..93a1725e8e 100644 --- a/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts +++ b/application/apps/rustcore/ts-bindings/spec/session.stream.spec.ts @@ -577,128 +577,4 @@ if (process.platform === 'win32') { }); }); }); - - config.performance.run && - Object.keys(config.regular.execute_only).length > 0 && - Object.keys(config.performance.tests).forEach((alias: string, index: number) => { - const test = (config.performance.tests as any)[alias]; - const testName = `${test.alias}`; - if (test.ignore) { - console.log(`Test "${testName}" has been ignored`); - return; - } - it(testName, function () { - return runner( - { - list: { 1: testName }, - execute_only: [], - files: {}, - }, - 1, - async (logger, done, collector) => { - const measurement = setMeasurement(); - try { - switch (index + 1) { - case 1: - const tmpobj1 = createSampleFile( - 5000, - logger, - (i: number) => `some line data: ${i}\n` - ); - const session1 = await Session.create(); - session1.debug(true, testName); - const stream1 = session1.getStream(); - if (stream1 instanceof Error) { - throw stream1; - } - await stream1.observe( - new Factory.Stream() - .asText() - .process({ - command: `less ${tmpobj1.name}`, - cwd: process.cwd(), - envs: process.env as { [key: string]: string }, - }) - .get() - .sterilized() - ); - break; - case 2: - const tmpobj2 = createSampleFile( - 5000, - logger, - (i: number) => `some line data: ${i}\n` - ); - const session2 = await Session.create(); - session2.debug(true, testName); - const stream2 = session2.getStream(); - if (stream2 instanceof Error) { - throw stream2; - } - await stream2.observe( - new Factory.Stream() - .asText() - .process({ - command: `less ${tmpobj2.name}`, - cwd: process.cwd(), - envs: process.env as { [key: string]: string }, - }) - .get() - .sterilized() - ); - finish(undefined, done); - break; - case 3: - const results = []; - for (let i = 0; i < 50; i++) { - const file = createSampleFile( - 100, - logger, - (j: number) => `file ${i} line data: ${j}\n` - ); - - const session = await Session.create(); - session.debug(true, `${testName} - session ${i}`); - const stream = session.getStream(); - if (stream instanceof Error) { - throw stream; - } - let result = await stream.observe( - new Factory.Stream() - .asText() - .process({ - command: `less ${file.name}`, - cwd: process.cwd(), - envs: process.env as { [key: string]: string }, - }) - .get() - .sterilized() - ).catch((err) => `File ${i} failed to open: ${err.message}`); - results.push(result); - } - finish(undefined, done); - break; - default: - throw new Error(`Unsupported format: ${test.open_as}`); - } - const results = measurement(); - finish( - undefined, - done, - performanceReport(testName, results.ms, test.expectation_ms) - ? undefined - : new Error(`${testName} is fail`) - ); - } catch (err) { - finish( - undefined, - done, - new Error(`Fail to create session due error: ${err instanceof Error ? err.message : err}`) - ); - } - } - ); - }); - }); - } diff --git a/scripts/elements/bindings.rb b/scripts/elements/bindings.rb index 0d47eef14a..cac836e8e7 100644 --- a/scripts/elements/bindings.rb +++ b/scripts/elements/bindings.rb @@ -14,11 +14,16 @@ module Bindings TARGETS = [DIST, TS_NODE_MODULES, TARGET, DIST_RS, SPEC, TS_BINDINGS_LIB].freeze def self.run_jasmine_spec(spec) - run_benchmarks = ENV['JASMIN_TEST_CONFIGURATION'] && ENV['JASMIN_TEST_CONFIGURATION'].include?('benchmarks.json') ? true : false + run_benchmarks = spec == 'benchmark' ? true : false ENV['ELECTRON_RUN_AS_NODE'] = '1' Shell.chdir(Paths::TS_BINDINGS) do if run_benchmarks - for i in 1..6 do + iterations = 6 + if !ENV['JASMIN_TEST_CONFIGURATION'] + Bindings.set_environment_vars + iterations = 1 + end + for i in 1..iterations do begin Shell.sh "#{Paths::JASMINE} spec/build/spec/session.#{spec}.spec.js" rescue @@ -30,6 +35,21 @@ def self.run_jasmine_spec(spec) end end end + + def self.environment_vars + { + 'JASMIN_TEST_CONFIGURATION' => './spec/benchmarks.json', + 'PERFORMANCE_RESULTS_FOLDER' => 'chipmunk_performance_results', + 'PERFORMANCE_RESULTS' => 'Benchmark_PR_00', + 'SH_HOME_DIR' => "/chipmunk" + # 'SH_HOME_DIR' => "/Users/sameer.g.srivastava" + } + end + + def self.set_environment_vars + env_vars = environment_vars + env_vars.each { |key, value| ENV[key] = value } + end end namespace :bindings do @@ -75,13 +95,13 @@ def self.run_jasmine_spec(spec) ] test_specs.each do |spec| desc "run jasmine #{spec}-spec" - task spec.to_sym => ['bindings:build_spec', 'bindings:build'] do + task spec.to_sym => ['bindings:build', 'bindings:build_spec'] do Bindings.run_jasmine_spec(spec) end end desc 'run binding tests' - task all: test_specs.map { |spec| "bindings:test:#{spec}" } + task all: test_specs.select { |spec| "bindings:test:#{spec}" if spec!='benchmark'} end desc 'clean bindings' diff --git a/scripts/tools/run_benchmarks.rb b/scripts/tools/run_benchmarks.rb index 7a2ad5798c..3d13a6fc01 100644 --- a/scripts/tools/run_benchmarks.rb +++ b/scripts/tools/run_benchmarks.rb @@ -9,12 +9,7 @@ RAKE_COMMANDS = [ 'rake clobber', - 'rake bindings:build', - 'rake bindings:build_spec', - 'rake bindings:test:stream', - 'rake bindings:test:indexes', - 'rake bindings:test:search', - 'rake bindings:test:observe' + 'rake bindings:test:benchmark' ] SHELL_SCRIPT_PATH = 'application/apps/rustcore/ts-bindings/spec'