From c92c24a13695a57e8d68bcb6403e99e0d66a9969 Mon Sep 17 00:00:00 2001 From: Llorx Date: Sat, 2 Sep 2023 18:18:07 +0300 Subject: [PATCH] ADD: Removed warmUpTime in favour of self-warmup to reach the minimum time. Now will run new processes each 10% of gathered samples. getDiff module isolation to reduce deops. --- README.md | 5 +- package.json | 2 +- src/IsoBench.ts | 6 +- src/Test.ts | 67 ++++++------------- src/WorkerSetup.ts | 1 - src/getDiff.ts | 10 +++ src/processors/StreamLog/DynamicStream.ts | 9 ++- .../StreamLog/{StreamTTY.ts => TTYOutput.ts} | 2 +- src/tests/StreamTTY.ts | 4 +- 9 files changed, 44 insertions(+), 62 deletions(-) create mode 100644 src/getDiff.ts rename src/processors/StreamLog/{StreamTTY.ts => TTYOutput.ts} (98%) diff --git a/README.md b/README.md index 21b8e31..2387a23 100644 --- a/README.md +++ b/README.md @@ -124,9 +124,8 @@ Creates a new `IsoBench` instance to benchmark your code. - `name`: The name of this IsoBench instance. Defaults to `IsoBench`. - `options`: Object: - `parallel`: The amount of parallel tests to run. Defaults to **1**. - - `time`: The minimum time (in milliseconds) to invest on each test. The library will automatically increase the amount of cycles to reach a minimum of `ms` between tests to take samples. Defaults to **50**. - - `samples`: Amount of samples to get. Defaults to **100**. - - `warmUpTime`: The minimum time (in milliseconds) to pre-run the tests, so the [JavaScript engine optimizer kicks-in](https://doar-e.github.io/blog/2019/01/28/introduction-to-turbofan/#compilation-pipeline:~:text=If%20the%20function%20gets%20executed%20a%20lot%2C%20TurboFan%20will%20generate%20some%20optimized%20code) before initializing the timer. Defaults to **500**. + - `time`: The minimum time (in milliseconds) to invest on each test. The library will automatically increase the amount of cycles to reach a minimum of `ms` between tests to take samples. Defaults to **100**. + - `samples`: Amount of samples to get. Will launch a new process each 10% samples. Defaults to **50** so will launch a new process each **5** samples. --- ```typescript diff --git a/package.json b/package.json index 15e4fcf..c9ca99c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "iso-bench", - "version": "2.4.6", + "version": "2.4.7", "description": "Small benchmark library focused in avoiding optimization/deoptimization pollution between tests by isolating them.", "types": "./lib/_types/index.d.ts", "main": "./lib/", diff --git a/src/IsoBench.ts b/src/IsoBench.ts index 7b4fff9..3b08284 100644 --- a/src/IsoBench.ts +++ b/src/IsoBench.ts @@ -21,7 +21,6 @@ export type IsoBenchOptions = { parallel?:number; samples?:number; time?:number; - warmUpTime?:number; }; export class IsoBench { processors:Processor[] = []; @@ -32,9 +31,8 @@ export class IsoBench { constructor(readonly name:string = "IsoBench", options?:IsoBenchOptions) { this.options = {...{ // Set defaults parallel: 1, - samples: 100, - time: 50, - warmUpTime: 50 + samples: 50, + time: 100 }, ...options}; this.name = getUniqueName(this.name, BENCHES); BENCHES.set(this.name, this); diff --git a/src/Test.ts b/src/Test.ts index a8e9850..bf2904c 100644 --- a/src/Test.ts +++ b/src/Test.ts @@ -1,11 +1,11 @@ import STREAM from "stream"; import CHILD_PROCESS from "child_process"; -import { performance } from "perf_hooks"; import { Fork } from "./Fork"; import { IsoBenchOptions, Processor } from "."; import { SetupMessage } from "./WorkerSetup"; import { Messager, RunMessage } from "./Messager"; +import { getDiff } from "./getDiff"; export type Sample = { cycles: number; @@ -21,9 +21,8 @@ class ForkContext { const setup:SetupMessage = { testIndex: this._test.index, benchName: this._benchName, - samples: this._options.samples, - time: this._options.time, - warmUpTime: this._options.warmUpTime + samples: Math.min(Math.ceil(this._options.samples * 0.1), this._options.samples - this._test.samples.length), + time: this._options.time }; const worker = Fork.fork({ ["ISO_BENCH_SETUP"]: JSON.stringify(setup) @@ -40,7 +39,11 @@ class ForkContext { this._resolve(); } else if (msg.done) { this._ended = true; - this._resolve(); + if (this._test.samples.length >= this._options.samples) { + this._resolve(); + } else { + new ForkContext(this._test, this._processors, this._resolve, this._benchName, this._options).start(); + } } else { const sample:Sample = { cycles: msg.cycles, @@ -107,7 +110,7 @@ export class Test { totalTime = 0; samples:Sample[] = []; group = ""; - constructor(readonly name:string, readonly index:number, private _callback:(setup?:unknown)=>void, private _setup?:()=>unknown) {} + constructor(readonly name:string, readonly index:number, private _callback:(setupData?:unknown)=>void, private _setup?:()=>unknown) {} fork(benchName:string, processors:Processor[], options:Required) { return new Promise((resolve => { // Start new context for this specific fork run @@ -116,50 +119,20 @@ export class Test { })); } async run(setup:SetupMessage) { - const warmUpResult = setup.warmUpTime > 0 ? this._getResult(setup.warmUpTime, 1) : null; + getDiff(1, this._callback, this._setup); // warmup let cycles = 1; - if (warmUpResult) { - const ratio = (setup.time / setup.warmUpTime) * 1.02; - cycles = warmUpResult.cycles * ratio; - } let samples = setup.samples; - while(samples-- > 0) { - const result = this._getResult(setup.time, cycles); - cycles = result.cycles; - await Messager.send({ - diff: result.diff, - cycles: result.cycles - }); - } - } - private _getResult(targetTime:number, cycles:number) { - let diff:number; - while(true) { - diff = this._getCallbackTime(cycles); - if (diff >= targetTime) { - break; - } else { - const ratio = diff > 0 ? (targetTime / diff) * 1.02 : 1.1; // Go a 2% further, to avoid it ending just below the targetTime. Increase by 10% if zero is received (mostly in systems without nanosecond resolution) - cycles = Math.ceil(cycles * ratio); - } - } - return {cycles, diff}; - } - private _getCallbackTime(cycles:number) { - // Individual loops so the callback doesn't receive an argument if there's no setup - if (this._setup) { - const setup = this._setup(); - const startTS = performance.now(); - while(cycles-- > 0) { - this._callback(setup); - } - return performance.now() - startTS; - } else { - const startTS = performance.now(); - while(cycles-- > 0) { - this._callback(); + while(samples > 0) { + const diff = getDiff(cycles, this._callback, this._setup); + if (diff >= setup.time) { + samples--; + await Messager.send({ + diff: diff, + cycles: cycles + }); } - return performance.now() - startTS; + const ratio = diff > 0 ? (setup.time / diff) * 1.02 : 1.1; // Go a 2% further, to avoid it ending just below the targetTime. Increase by 10% if zero is received (mostly in systems without nanosecond resolution) + cycles = diff >= setup.time ? Math.round(cycles * ratio) : Math.ceil(cycles * ratio); } } } \ No newline at end of file diff --git a/src/WorkerSetup.ts b/src/WorkerSetup.ts index ff4d175..9d68e66 100644 --- a/src/WorkerSetup.ts +++ b/src/WorkerSetup.ts @@ -3,7 +3,6 @@ export type SetupMessage = { benchName:string; time:number; samples:number; - warmUpTime:number; }; export let WorkerSetup:SetupMessage|null = null; diff --git a/src/getDiff.ts b/src/getDiff.ts new file mode 100644 index 0000000..a87c803 --- /dev/null +++ b/src/getDiff.ts @@ -0,0 +1,10 @@ +import { performance } from "perf_hooks"; + +export function getDiff(cycles:number, callback:(setupData?:unknown)=>void, setup?:()=>unknown) { + const setupData = setup && setup(); + const startTS = performance.now(); + while(cycles-- > 0) { + callback(setupData); + } + return performance.now() - startTS; +} \ No newline at end of file diff --git a/src/processors/StreamLog/DynamicStream.ts b/src/processors/StreamLog/DynamicStream.ts index 964d14d..5201434 100644 --- a/src/processors/StreamLog/DynamicStream.ts +++ b/src/processors/StreamLog/DynamicStream.ts @@ -4,10 +4,10 @@ import { Processor } from "../../Processor"; import { Test, Sample } from "../../Test"; import { IsoBench } from "../../IsoBench"; import { Group, getTestLog, COLORS } from "./Utils"; -import { StreamTTY } from "./StreamTTY"; +import { TTYOutput } from "./TTYOutput"; export class TestOutput { - constructor(private _tty:StreamTTY, readonly line:number) {} + constructor(private _tty:TTYOutput, readonly line:number) {} log(data:string) { this._tty.log(data, this.line); } @@ -20,7 +20,7 @@ export class DynamicStream implements Processor { private _benchName = ""; private _groups = new Map(); constructor(protected _stream:TTY.WriteStream) { - this._tty = new StreamTTY(this._stream); + this._tty = new TTYOutput(this._stream); this._header = new TestOutput(this._tty, 0); } initialize(bench:IsoBench, tests:Test[]) { @@ -73,6 +73,7 @@ export class DynamicStream implements Processor { if (output) { const logArgs = getTestLog(this._padding, test, null, true, sample); logArgs.push(`${COLORS.YELLOW}Running...${COLORS.CLEAR}`); + //logArgs.push("Min:", test.samples.slice().sort((a,b) => a.ops - b.ops)[0].ops, "Max:", test.samples.slice().sort((a,b) => b.ops - a.ops)[0].ops); output.log(logArgs.join(" ")); } } @@ -80,6 +81,7 @@ export class DynamicStream implements Processor { const output = this._outputs.get(test.index); if (output) { const logArgs = getTestLog(this._padding, test, null, true); + //logArgs.push("Min:", test.samples.slice().sort((a,b) => a.ops - b.ops)[0].ops, "Max:", test.samples.slice().sort((a,b) => b.ops - a.ops)[0].ops); output.log(logArgs.join(" ")); } const group = this._groups.get(test.group); @@ -101,6 +103,7 @@ export class DynamicStream implements Processor { const output = this._outputs.get(test.index); if (output) { const logArgs = getTestLog(this._padding, test, { min, max }, true); + //logArgs.push("Min:", test.samples.slice().sort((a,b) => a.ops - b.ops)[0].ops, "Max:", test.samples.slice().sort((a,b) => b.ops - a.ops)[0].ops); output.log(logArgs.join(" ")); } } diff --git a/src/processors/StreamLog/StreamTTY.ts b/src/processors/StreamLog/TTYOutput.ts similarity index 98% rename from src/processors/StreamLog/StreamTTY.ts rename to src/processors/StreamLog/TTYOutput.ts index fc5b79c..d66e02a 100644 --- a/src/processors/StreamLog/StreamTTY.ts +++ b/src/processors/StreamLog/TTYOutput.ts @@ -1,6 +1,6 @@ import TTY from "tty"; -export class StreamTTY { +export class TTYOutput { logs:string[] = []; top = 0; drawheight = 0; diff --git a/src/tests/StreamTTY.ts b/src/tests/StreamTTY.ts index fde96b5..50a272d 100644 --- a/src/tests/StreamTTY.ts +++ b/src/tests/StreamTTY.ts @@ -1,6 +1,6 @@ -import { StreamTTY } from "../processors/StreamLog/StreamTTY"; +import { TTYOutput } from "../processors/StreamLog/TTYOutput"; -const tty = new StreamTTY(process.stdout); +const tty = new TTYOutput(process.stdout); for (let i = 0; i < 10; i++) { setTimeout(() => { tty.log("test" + i, i);