chore: add benchmark

Author: KuznetsovRoman

.npmignore

@@ -1,3 +1,4 @@
 **/.*
 test/
 coverage/
+benchmark

README.md

@@ -7,6 +7,13 @@ perception. It is created specially for the needs of visual regression testing
 for [`testplane`](http://github.com/gemini-testing/testplane) utility, but can be used
 for other purposes.
 
+## Benchmark
+
+Benchmark is presented in the corresponding directory.
+
+- [Benchmark description](./benchmark/README.md)
+- [Benchmark results](./benchmark/results.md)
+
 ## Supported image formats
 
 JPEG, PNG, WebP, GIF, AVIF, TIFF and SVG images are supported.

benchmark/README.md

@@ -0,0 +1,71 @@
+# looks-same Benchmark
+
+This benchmark evaluates the performance of four npm packages for image comparison: [`looks-same`](https://github.com/gemini-testing/looks-same), [`pixelmatch`](https://github.com/mapbox/pixelmatch), [`resemblejs`](https://github.com/rsmbl/Resemble.js), and [`blink-diff`](https://github.com/yahoo/blink-diff).  
+It focuses on execution speed across diverse test cases, including real-world web comparisons and synthetic examples.
+
+---
+
+## Test Cases
+
+### Web-Based Averages
+
+- **Web Avg Diff (672x623)**:  
+  Aggregates **failed web page comparisons** from our production projects using [Testplane](https://testplane.io/) for e2e testing.  
+  Represents 103,142 real-world UI mismatches (e.g., layout shifts, rendering errors).
+- **Web Avg Success (656x547)**:  
+  Derived from 6,282,752 successful comparisons in production.  
+  Tests speed for "no difference" scenarios common in regression testing workflows.
+
+### Synthetic Comparisons
+
+- **Equal Images (1000x1000)**: Baseline test with identical images.
+- **1% Invisible Diff (1000x1000)**:  
+  Differences are **visually imperceptible** (ΔE < 2.3 in [CIEDE2000](https://en.wikipedia.org/wiki/Color_difference#CIEDE2000)), simulating subtle changes like anti-aliasing variations.
+- **1% Visible Diff (1000x1000)**:  
+  Human-noticeable differences (ΔE ≥ 2.3) in RGB values.
+- **5% Invisible/Visible Diff (1000x1000)**:  
+  Larger-scale variations with the same perceptual thresholds.
+- **Full Max Diff (1000x1000)**:  
+  All pixels altered (maximum difference) to stress-test worst-case performance.
+
+### Realistic Example
+
+- **Demonstrative Example (896×784)**:  
+  Real-world UI comparison with an 8% visible difference: a button intentionally recolored to a human-noticeable shade.  
+  Provides visual intuition for typical failure scenarios.  
+  <details>
+    <summary>Uses the following images (from ./fixtures)</summary>
+    
+    <table>
+      <tr>
+        <td><img src="./fixtures/reference.png" alt="Image 1" style="width: 100%;"/></td>
+        <td><img src="./fixtures/actual.png" alt="Image 2" style="width: 100%;"/></td>
+      </tr>
+    </table>
+  </details>
+  
+
+---
+
+## Results
+
+See [**results.md**](./results.md) for execution times across all test cases.  
+
+## Running the benchmark
+
+1. Clone the repository:
+   ```bash
+   git clone [email protected]:gemini-testing/looks-same.git
+   ```
+2. Navigate to "benchmark" directory:
+   ```bash
+   cd looks-same/benchmark
+   ```
+3. Install dependencies:
+   ```bash
+   npm ci
+   ```
+4. Run the benchmark:
+   ```bash
+   npm start
+   ```

benchmark/constants.js

@@ -0,0 +1,18 @@
+exports.PACKAGES = {
+    LOOKS_SAME: 'looks-same',
+    PIXELMATCH: 'pixelmatch',
+    RESEMBLE_JS: 'resemblejs',
+    BLINK_DIFF: 'blink-diff'
+};
+
+exports.CASES = {
+    WEB_AVG_DIFF: 'web avg diff (672x623)',
+    WEB_AVG_SUCCESS: 'web avg success (656x547)',
+    EQUAL_IMAGES: 'equal images (1000x1000)',
+    ONE_PERCENT_INVISIBLE_DIFF: '1% invisible diff (1000x1000)',
+    ONE_PERCENT_VISIBLE_DIFF: '1% visible diff (1000x1000)',
+    FIVE_PERCENTS_INVISIBLE_DIFF: '5% invisible diff (1000x1000)',
+    FIVE_PERCENTS_VISIBLE_DIFF: '5% visible diff (1000x1000)',
+    FULL_MAX_DIFF: 'full max diff (1000x1000)',
+    DEMONSTRATIVE_EXAMPLE: 'demonstrative example visible 8% diff (896×784)'
+};

benchmark/fixtures/actual.png

@@ -0,0 +1,51 @@
+exports.CHANNELS_COUNT = 3;
+exports.JND = 2.3; // CIEDE2000 just noticeable difference
+
+exports.REFERENCE_COLOR = [100, -38, -38]; // CIE-L*ab
+exports.CONSTRAST_REFERENCE_COLOR = [0, 128, 128]; // CIE-L*ab
+
+exports.AVERAGE_FAILED_HIST_BUCKETS = [
+    // in ["a", "b", "c"] tuples, "c" part of the pixels have ciede2000 diff befween "a" (inc.) and "b" (exc.)
+    [0, 5e-324, 0.9163583710631669],
+    [5e-324, 0.2, 0.0006867712475951489],
+    [0.2, 0.5, 0.0021610446606488607],
+    [0.5, 1, 0.005987159508767972],
+    [1, 1.6, 0.011063733707154652],
+    [1.6, 1.73, 0.0015331092837601974],
+    [1.73, 1.8765, 0.0006625396213279764],
+    [1.8765, 2, 0.0018387428611124819],
+    [2, 2.3, 0.0013118358358049037],
+    [2.3, 2.4, 0.00022329552469749336],
+    [2.4, 2.5, 0.00036489135695552325],
+    [2.5, 2.7, 0.008824595373243259],
+    [2.7, 4, 0.003404259594068377],
+    [4, 6, 0.0042124825492642895],
+    [6, 8, 0.0030169690455660605],
+    [8, 10, 0.0015209692289468576],
+    [10, 12, 0.001571573898168484],
+    [12, 14.26, 0.0011032068046126686],
+    [14.26, 15.5, 0.000604997805480533],
+    [15.5, 16.7, 0.0006418838778879667],
+    [16.7, 25, 0.0032761913664942494],
+    [25, 30, 0.004046992111565813],
+    [30, 40, 0.008427973959907584],
+    [40, 50, 0.004921414959461223],
+    [50, 60, 0.0019655582362449707],
+    [60, 70, 0.0021172710822109477],
+    [70, 80, 0.0011398025313718967],
+    [80, 90, 0.0027750423615150576],
+    [90, 125, 0.004237320542997875]
+].sort((a, b) => b[2] - a[2]);
+
+exports.AVERAGE_SUCCESS_HIST_BUCKETS = [
+    // in ["a", "b", "c"] tuples, "c" part of the pixels have ciede2000 diff befween "a" (inc.) and "b" (exc.)
+    [0, 5e-324, 0.9731894137187758],
+    [5e-324, 0.2, 0.0007293636735490286],
+    [0.2, 0.5, 0.0022950691047618353],
+    [0.5, 1, 0.006358473318051954],
+    [1, 1.6, 0.01174988831881833],
+    [1.6, 1.73, 0.0016281902060853782],
+    [1.73, 1.8765, 0.0007036292415788792],
+    [1.8765, 2, 0.0019527786764358484],
+    [2, 2.3, 0.0013931937419429665]
+].sort((a, b) => b[2] - a[2]);

benchmark/fixtures/reference.png

@@ -0,0 +1,11 @@
+const {labToRgb} = require('./utils');
+const {CONSTRAST_REFERENCE_COLOR} = require('./constants');
+const {ImageGenerator} = require('./image-generator');
+
+exports.ContrastToReferenceGenerator = class ContrastToReferenceGenerator extends ImageGenerator {
+    constructor(width, height) {
+        const colors = new Array(width * height).fill(labToRgb(CONSTRAST_REFERENCE_COLOR));
+
+        super(width, height, colors);
+    }
+};

benchmark/image-generator/constants.js

@@ -0,0 +1,15 @@
+const {JND} = require('./constants');
+const {HistCieDiffColorsGenerator} = require('./hist-cie-diff-colors-generator');
+const {ImageGenerator} = require('./image-generator');
+
+exports.FixedInvisibleDiffAmountGenerator = class FixedInvisibleDiffAmountGenerator extends ImageGenerator {
+    constructor(width, height, diffPercent) {
+        const colorsGenerator = new HistCieDiffColorsGenerator([
+            [0, Number.MIN_VALUE, 1 - diffPercent],
+            [Number.MIN_VALUE, JND, diffPercent]
+        ]);
+        const colors = new Array(width * height).fill(null).map(() => colorsGenerator.getRandomRgbColor());
+
+        super(width, height, colors);
+    }
+};

benchmark/image-generator/contrast-to-reference-generator.js

@@ -0,0 +1,15 @@
+const {JND} = require('./constants');
+const {HistCieDiffColorsGenerator} = require('./hist-cie-diff-colors-generator');
+const {ImageGenerator} = require('./image-generator');
+
+exports.FixedVisibleDiffAmountGenerator = class FixedVisibleDiffAmountGenerator extends ImageGenerator {
+    constructor(width, height, diffPercent) {
+        const colorsGenerator = new HistCieDiffColorsGenerator([
+            [0, Number.MIN_VALUE, 1 - diffPercent],
+            [JND, 100, diffPercent]
+        ]);
+        const colors = new Array(width * height).fill(null).map(() => colorsGenerator.getRandomRgbColor());
+
+        super(width, height, colors);
+    }
+};