feat: compute and display median and median absolute deviation (#89)

jerome-benoit · web-flow · commit 46fbd0ce395a · 2024-10-13T14:46:19.000+02:00
Signed-off-by: Jérôme Benoit &lt;jerome.benoit@piment-noir.org&gt;
diff --git a/README.md b/README.md
@@ -53,12 +53,12 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬────────────────────┬───────────┬─────────┐
-// │ (index) │   Task Name   │ ops/sec  │ Average Time (ns)  │  Margin   │ Samples │
-// ├─────────┼───────────────┼──────────┼────────────────────┼───────────┼─────────┤
-// │    0    │ 'faster task' │ '41,621' │ 24025.791819761525 │ '±20.50%' │  4257   │
-// │    1    │ 'slower task' │  '828'   │ 1207382.7838323202 │ '±7.07%'  │   83    │
-// └─────────┴───────────────┴──────────┴────────────────────┴───────────┴─────────┘
+// ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
+// │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
+// ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
+// │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
+// │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
+// └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
 
 console.table(bench.table((task) => ({ 'Task name': task.name })));
 
@@ -282,6 +282,16 @@ export type TaskResult = {
    */
   rme: number;
 
+  /**
+   * median absolute deviation
+   */
+  mad: number;
+
+  /**
+   * p50/median percentile
+   */
+  p50: number;
+
   /**
    * p75 percentile
    */
diff --git a/examples/src/simple.ts b/examples/src/simple.ts
@@ -18,12 +18,12 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬────────────────────┬───────────┬─────────┐
-// │ (index) │   Task Name   │ ops/sec  │ Average Time (ns)  │  Margin   │ Samples │
-// ├─────────┼───────────────┼──────────┼────────────────────┼───────────┼─────────┤
-// │    0    │ 'faster task' │ '41,621' │ 24025.791819761525 │ '±20.50%' │  4257   │
-// │    1    │ 'slower task' │  '828'   │ 1207382.7838323202 │ '±7.07%'  │   83    │
-// └─────────┴───────────────┴──────────┴────────────────────┴───────────┴─────────┘
+// ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
+// │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
+// ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
+// │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
+// │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
+// └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
 
 console.table(
   bench.todos.map(({ name }) => ({
diff --git a/src/bench.ts b/src/bench.ts
@@ -242,10 +242,11 @@ export default class Bench extends EventTarget {
           throw task.result.error;
         }
         return convert?.(task) || {
-          'Task Name': task.name,
+          'Task name': task.name,
           'ops/sec': task.result.error ? 'NaN' : parseInt(task.result.hz.toString(), 10).toLocaleString(),
-          'Average Time (ns)': task.result.error ? 'NaN' : task.result.mean * 1000 * 1000,
+          'Average time/op (ns)': task.result.error ? 'NaN' : task.result.mean * 1e6,
           Margin: task.result.error ? 'NaN' : `\xb1${task.result.rme.toFixed(2)}%`,
+          'Median time/op (ns)': task.result.error ? 'NaN' : `${task.result.p50 * 1e6}${task.result.mad * 1e6 > 0 ? `\xb1${(task.result.mad * 1e6).toFixed(10)}` : ''}`,
           Samples: task.result.error ? 'NaN' : task.result.samples.length,
         };
       }
diff --git a/src/task.ts b/src/task.ts
@@ -10,7 +10,13 @@ import Bench from './bench';
 import tTable from './constants';
 import { createBenchEvent } from './event';
 import { AddEventListenerOptionsArgument, RemoveEventListenerOptionsArgument } from './types';
-import { getVariance, isAsyncTask, quantileSorted } from './utils';
+import {
+  absoluteDeviation,
+  getVariance,
+  isAsyncTask,
+  medianSorted,
+  quantileSorted,
+} from './utils';
 
 /**
  * A class that represents each benchmark task in Tinybench. It keeps track of the
@@ -152,6 +158,9 @@ export default class Task extends EventTarget {
       const moe = sem * critical;
       const rme = (moe / mean) * 100;
 
+      const mad = absoluteDeviation(samples, medianSorted);
+
+      const p50 = medianSorted(samples);
       const p75 = quantileSorted(samples, 0.75);
       const p99 = quantileSorted(samples, 0.99);
       const p995 = quantileSorted(samples, 0.995);
@@ -176,6 +185,8 @@ export default class Task extends EventTarget {
         critical,
         moe,
         rme,
+        mad,
+        p50,
         p75,
         p99,
         p995,
diff --git a/src/types.ts b/src/types.ts
@@ -105,6 +105,16 @@ export type TaskResult = {
    */
   rme: number;
 
+  /**
+   * median absolute deviation
+   */
+  mad: number;
+
+  /**
+   * p50/median percentile
+   */
+  p50: number;
+
   /**
    * p75 percentile
    */
diff --git a/src/utils.ts b/src/utils.ts
@@ -19,7 +19,7 @@ function isPromiseLike<T>(maybePromiseLike: any): maybePromiseLike is PromiseLik
 const AsyncFunctionConstructor = (async () => {}).constructor;
 
 /**
- * an async function check method only consider runtime support async syntax
+ * An async function check method only consider runtime support async syntax
  */
 export const isAsyncFunction = (fn: Fn) => fn.constructor === AsyncFunctionConstructor;
 
@@ -113,3 +113,29 @@ export const quantileSorted = (samples: number[], q: number) => {
   }
   return samples[baseIndex];
 };
+
+/**
+ * Computes the median of a sorted sample.
+ *
+ * @param samples the sorted sample
+ * @returns the median of the sample
+ */
+export const medianSorted = (samples: number[]) => quantileSorted(samples, 0.5);
+
+/**
+ * Computes the absolute deviation of a sample given an aggregation.
+ *
+ * @param samples the sample
+ * @param aggFn the aggregation function to use
+ * @returns the absolute deviation of the sample given the aggregation
+ */
+export const absoluteDeviation = (samples: number[], aggFn: (arr: number[]) => number | undefined) => {
+  const value = aggFn(samples);
+  const absoluteDeviations: number[] = [];
+
+  for (const sample of samples) {
+    absoluteDeviations.push(Math.abs(sample - value!));
+  }
+
+  return aggFn(absoluteDeviations);
+};

Original file line number	Diff line number	Diff line change
`@@ -242,10 +242,11 @@ export default class Bench extends EventTarget {`
`242`	`242`	`throw task.result.error;`
`243`	`243`	`}`
`244`	`244`	`return convert?.(task) \|\| {`
`245`		`- 'Task Name': task.name,`
	`245`	`+ 'Task name': task.name,`
`246`	`246`	`'ops/sec': task.result.error ? 'NaN' : parseInt(task.result.hz.toString(), 10).toLocaleString(),`
`247`		`- 'Average Time (ns)': task.result.error ? 'NaN' : task.result.mean * 1000 * 1000,`
	`247`	`+ 'Average time/op (ns)': task.result.error ? 'NaN' : task.result.mean * 1e6,`
`248`	`248`	Margin: task.result.error ? 'NaN' : `\xb1${task.result.rme.toFixed(2)}%`,
	`249`	+ 'Median time/op (ns)': task.result.error ? 'NaN' : `${task.result.p50 * 1e6}${task.result.mad * 1e6 > 0 ? `\xb1${(task.result.mad * 1e6).toFixed(10)}` : ''}`,
`249`	`250`	`Samples: task.result.error ? 'NaN' : task.result.samples.length,`
`250`	`251`	`};`
`251`	`252`	`}`