feat!: fold warmup step into benchmark run (#150)

jerome-benoit · web-flow · commit 4fcd45f73636 · 2024-10-21T12:35:22.000+02:00
Signed-off-by: Jérôme Benoit &lt;jerome.benoit@piment-noir.org&gt;
diff --git a/README.md b/README.md
@@ -44,7 +44,6 @@ bench
     console.log('I am slower');
   });
 
-await bench.warmup(); // make results more reliable
 await bench.run();
 
 console.log(bench.name);
@@ -111,6 +110,11 @@ export interface Options {
    */
   throws?: boolean;
 
+  /**
+   * warmup benchmark @default true
+   */
+  warmup?: boolean;
+
   /**
    * warmup time (milliseconds) @default 100
    */
@@ -136,7 +140,6 @@ export type Hook = (task: Task, mode: 'warmup' | 'run') => void | Promise<void>;
 ```
 
 - `async run()`: run the added tasks that were registered using the `add` method
-- `async warmup()`: warmup the benchmark tasks
 - `reset()`: reset each task and remove its result
 - `add(name: string, fn: Fn, opts?: FnOpts)`: add a benchmark task to the task map
   - `Fn`: `() => unknown | Promise<unknown>`
@@ -392,7 +395,6 @@ It may make your benchmarks slower, check [#42](https://github.com/tinylibs/tiny
 ```ts
 bench.threshold = 10; // The maximum number of concurrent tasks to run. Defaults to Number.POSITIVE_INFINITY.
 bench.concurrency = 'task'; // The concurrency mode to determine how tasks are run.
-await bench.warmup();
 await bench.run();
 ```
 
diff --git a/src/bench.ts b/src/bench.ts
@@ -52,6 +52,8 @@ export default class Bench extends EventTarget {
 
   throws = false;
 
+  warmup = true;
+
   warmupTime = defaultMinimumWarmupTime;
 
   warmupIterations = defaultMinimumWarmupIterations;
@@ -70,6 +72,7 @@ export default class Bench extends EventTarget {
     super();
     this.name = options.name;
     this.now = options.now ?? this.now;
+    this.warmup = options.warmup ?? this.warmup;
     this.warmupTime = options.warmupTime ?? this.warmupTime;
     this.warmupIterations = options.warmupIterations ?? this.warmupIterations;
     this.time = options.time ?? this.time;
@@ -90,6 +93,25 @@ export default class Bench extends EventTarget {
     }
   }
 
+  /**
+   * warmup the benchmark tasks.
+   */
+  private async warmupTasks(): Promise<void> {
+    this.dispatchEvent(createBenchEvent('warmup'));
+    if (this.concurrency === 'bench') {
+      const limit = pLimit(this.threshold);
+      const promises: Promise<void>[] = [];
+      for (const task of this._tasks.values()) {
+        promises.push(limit(() => task.warmup()));
+      }
+      await Promise.all(promises);
+    } else {
+      for (const task of this._tasks.values()) {
+        await task.warmup();
+      }
+    }
+  }
+
   private async runTask(task: Task): Promise<Task> {
     if (this.signal?.aborted) {
       return task;
@@ -99,9 +121,11 @@ export default class Bench extends EventTarget {
 
   /**
    * run the added tasks that were registered using the {@link add} method.
-   * Note: This method does not do any warmup. Call {@link warmup} for that.
    */
   async run(): Promise<Task[]> {
+    if (this.warmup) {
+      await this.warmupTasks();
+    }
     let values: Task[] = [];
     this.dispatchEvent(createBenchEvent('start'));
     if (this.concurrency === 'bench') {
@@ -120,26 +144,6 @@ export default class Bench extends EventTarget {
     return values;
   }
 
-  /**
-   * warmup the benchmark tasks.
-   * This is not run by default by the {@link run} method.
-   */
-  async warmup(): Promise<void> {
-    this.dispatchEvent(createBenchEvent('warmup'));
-    if (this.concurrency === 'bench') {
-      const limit = pLimit(this.threshold);
-      const promises: Promise<void>[] = [];
-      for (const task of this._tasks.values()) {
-        promises.push(limit(() => task.warmup()));
-      }
-      await Promise.all(promises);
-    } else {
-      for (const task of this._tasks.values()) {
-        await task.warmup();
-      }
-    }
-  }
-
   /**
    * reset each task and remove its result
    */
@@ -196,37 +200,24 @@ export default class Bench extends EventTarget {
   ): (Record<string, string | number> | undefined | null)[] {
     return this.tasks.map((task) => {
       if (task.result) {
-        if (task.result.error) {
-          throw task.result.error;
-        }
-        return (
-          // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
-          convert?.(task) || {
+        return task.result.error
+          ? (convert?.(task) ?? {
+            'Task name': task.name,
+            Error: task.result.error.message,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            Stack: task.result.error.stack!,
+            Samples: task.result.latency.samples.length,
+          })
+          : (convert?.(task) ?? {
             'Task name': task.name,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Throughput average (ops/s)': task.result.error
-              ? 'NaN'
-              : `${task.result.throughput.mean.toFixed(0)} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Throughput median (ops/s)': task.result.error
-              ? 'NaN'
-              : // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-              `${task.result.throughput.p50!.toFixed(0)}${Number.parseInt(task.result.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result.throughput.mad!.toFixed(0)}` : ''}`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Latency average (ns)': task.result.error
-              ? 'NaN'
-              : `${(task.result.latency.mean * 1e6).toFixed(2)} \xb1 ${task.result.latency.rme.toFixed(2)}%`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Latency median (ns)': task.result.error
-              ? 'NaN'
-              : // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-              `${(task.result.latency.p50! * 1e6).toFixed(2)}${Number.parseFloat((task.result.latency.mad! * 1e6).toFixed(2)) > 0 ? ` \xb1 ${(task.result.latency.mad! * 1e6).toFixed(2)}` : ''}`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            Samples: task.result.error
-              ? 'NaN'
-              : task.result.latency.samples.length,
-          }
-        );
+            'Throughput average (ops/s)': `${task.result.throughput.mean.toFixed(0)} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            'Throughput median (ops/s)': `${task.result.throughput.p50!.toFixed(0)}${Number.parseInt(task.result.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result.throughput.mad!.toFixed(0)}` : ''}`,
+            'Latency average (ns)': `${(task.result.latency.mean * 1e6).toFixed(2)} \xb1 ${task.result.latency.rme.toFixed(2)}%`,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            'Latency median (ns)': `${(task.result.latency.p50! * 1e6).toFixed(2)}${Number.parseFloat((task.result.latency.mad! * 1e6).toFixed(2)) > 0 ? ` \xb1 ${(task.result.latency.mad! * 1e6).toFixed(2)}` : ''}`,
+            Samples: task.result.latency.samples.length,
+          });
       }
       return null;
     });
diff --git a/src/task.ts b/src/task.ts
@@ -130,8 +130,35 @@ export default class Task extends EventTarget {
     return { samples };
   }
 
+  /**
+   * warmup the current task
+   * @internal
+   */
+  async warmup(): Promise<void> {
+    if (this.result?.error) {
+      return;
+    }
+    this.dispatchEvent(createBenchEvent('warmup', this));
+    await this.bench.setup(this, 'warmup');
+    const { error } = (await this.benchmark(
+      this.bench.warmupTime,
+      this.bench.warmupIterations,
+    )) as { error?: Error };
+    await this.bench.teardown(this, 'warmup');
+
+    if (error) {
+      this.setResult({ error });
+      if (this.bench.throws) {
+        throw error;
+      }
+      this.dispatchEvent(createBenchEvent('error', this));
+      this.bench.dispatchEvent(createBenchEvent('error', this));
+    }
+  }
+
   /**
    * run the current task and write the results in `Task.result` object property
+   * @internal
    */
   async run(): Promise<Task> {
     if (this.result?.error) {
@@ -205,29 +232,6 @@ export default class Task extends EventTarget {
     return this;
   }
 
-  /**
-   * warmup the current task
-   */
-  async warmup(): Promise<void> {
-    if (this.result?.error) {
-      return;
-    }
-    this.dispatchEvent(createBenchEvent('warmup', this));
-    await this.bench.setup(this, 'warmup');
-    const { error } = (await this.benchmark(
-      this.bench.warmupTime,
-      this.bench.warmupIterations,
-    )) as { error?: Error };
-    await this.bench.teardown(this, 'warmup');
-
-    if (error) {
-      this.setResult({ error });
-      if (this.bench.throws) {
-        throw error;
-      }
-    }
-  }
-
   addEventListener<K extends TaskEvents>(
     type: K,
     listener: TaskEventsMap[K],
diff --git a/src/types.ts b/src/types.ts
@@ -338,6 +338,11 @@ export interface Options {
    */
   throws?: boolean;
 
+  /**
+   * warmup benchmark @default true
+   */
+  warmup?: boolean;
+
   /**
    * warmup time (milliseconds) @default 100
    */
diff --git a/src/utils.ts b/src/utils.ts
@@ -1,7 +1,7 @@
 import { emptyFunction, tTable } from './constants';
 import type { Fn, Statistics } from './types';
 
-export const nanoToMs = (nano: number) => nano / 1e6;
+const nanoToMs = (nano: number) => nano / 1e6;
 
 const hrtimeBigint = process.hrtime.bigint.bind(process.hrtime);
 export const hrtimeNow = () => nanoToMs(Number(hrtimeBigint()));
diff --git a/test/index.test.ts b/test/index.test.ts
@@ -89,8 +89,8 @@ test('events order', async () => {
   const controller = new AbortController();
   const bench = new Bench({
     signal: controller.signal,
-    warmupIterations: 0,
     warmupTime: 0,
+    warmupIterations: 0,
   });
   bench
     // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -165,8 +165,6 @@ test('events order', async () => {
   bench.add('temporary', () => {});
   bench.remove('temporary');
 
-  await bench.warmup();
-
   setTimeout(() => {
     controller.abort();
     // the abort task takes 1000ms (500ms time || 10 iterations => 10 * 1000)
@@ -198,10 +196,7 @@ test('events order', async () => {
 }, 10000);
 
 test('events order at task completion', async () => {
-  const bench = new Bench({
-    warmupIterations: 0,
-    warmupTime: 0,
-  });
+  const bench = new Bench();
 
   bench
     .add('foo', async () => {
@@ -231,8 +226,8 @@ test('events order at task completion', async () => {
   expect(tasks[1]?.name).toBe('bar');
 });
 
-test('error event', async () => {
-  const bench = new Bench({ time: 50 });
+test.each(['warmup', 'run'])('%s error event', async (mode) => {
+  const bench = new Bench({ time: 100, warmup: mode === 'warmup' });
   const err = new Error();
 
   bench.add('error', () => {
@@ -250,8 +245,14 @@ test('error event', async () => {
   expect(taskErr).toBe(err);
 });
 
-test('throws', async () => {
-  const bench = new Bench({ iterations: 1, throws: true });
+test.each(['warmup', 'run'])('%s throws', async (mode) => {
+  const iterations = 1;
+  const bench = new Bench({
+    iterations,
+    throws: true,
+    warmup: mode === 'warmup',
+    warmupIterations: iterations,
+  });
   const err = new Error();
 
   bench.add('error', () => {
@@ -406,7 +407,6 @@ test('setup and teardown', async () => {
   });
   const fooTask = bench.getTask('foo');
 
-  await bench.warmup();
   await bench.run();
 
   expect(setup).toBeCalledWith(fooTask, 'warmup');
@@ -419,8 +419,8 @@ test('task beforeAll, afterAll, beforeEach, afterEach', async () => {
   const iterations = 100;
   const bench = new Bench({
     time: 0,
-    warmupTime: 0,
     iterations,
+    warmupTime: 0,
     warmupIterations: iterations,
   });
 
@@ -449,7 +449,6 @@ test('task beforeAll, afterAll, beforeEach, afterEach', async () => {
     },
   );
 
-  await bench.warmup();
   await bench.run();
 
   expect(beforeAll.mock.calls.length).toBe(2 /* warmup + run */);
diff --git a/test/sequential.test.ts b/test/sequential.test.ts
