[BUG] Diff between screenshots is not present in the report

Context:

• Playwright Version: 1.23.0
• Operating System: Mac
• Node.js version: 16.14.0
• Browser: Chromium

Code Snippet

Playwright config:

const config: PlaywrightTestConfig = {
  testDir: 'tests',
  fullyParallel: true,
  reporter: process.env.CI ? [['github'], ['html']] : [['list'], ['html', { open: 'on-failure' }]],
  forbidOnly: !!process.env.CI,
  retries: process.env.CI ? 1 : 0,
  use: {
    headless: true,
    viewport: { width: 1280, height: 720 },
    screenshot: 'only-on-failure',
    video: 'retain-on-failure',
    trace: 'retain-on-failure',
  },
  expect: {
    toHaveScreenshot: { 
      threshold: 0.2,
      maxDiffPixelRatio: 0.05
     },
  },
  projects: [
    {
      name: 'chromium',
      use: { ...devices['Desktop Chrome'] },
    },
  ],
};
export default config;

Method that compares screenshots:

async matchesScreenshot(
    locator: Locator,
    options?: LocatorScreenshotOptions,
  ) {
    expect(locator).toHaveScreenshot({timeout: 5000, ...options});
  }

Describe the bug

I’m doing visual regression testing. I want to see the difference between screenshots in the report. However, the difference is not present there. Also, in the test-results folder the difference is missing too.

[dgozman]

@muhammad-saleh Thank you for the explanation, this makes sense. So, this is happening because you use full-page screenshots. I’ll reopen this issue for now.

[anetrebsky]

If it helps someone, I have prepared an NPM package patch using https://www.npmjs.com/package/patch-package, that ignores difference in height but still requires the same width of snapshots.

diff --git a/node_modules/@playwright/test/node_modules/playwright-core/lib/utils/comparators.js b/node_modules/@playwright/test/node_modules/playwright-core/lib/utils/comparators.js
index a0a36b1..eaad391 100644
--- a/node_modules/@playwright/test/node_modules/playwright-core/lib/utils/comparators.js
+++ b/node_modules/@playwright/test/node_modules/playwright-core/lib/utils/comparators.js
@@ -10,6 +10,7 @@ var _utilsBundle = require("../utilsBundle");
 var _pixelmatch = _interopRequireDefault(require("../third_party/pixelmatch"));
 
 var _diff_match_patch = require("../third_party/diff_match_patch");
+const exp = require("constants");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
@@ -55,31 +56,48 @@ function compareImages(mimeType, actualBuffer, expectedBuffer, options = {}) {
   if (!actualBuffer || !(actualBuffer instanceof Buffer)) return {
     errorMessage: 'Actual result should be a Buffer.'
   };
-  const actual = mimeType === 'image/png' ? _utilsBundle.PNG.sync.read(actualBuffer) : _utilsBundle.jpegjs.decode(actualBuffer, {
+  var actual = mimeType === 'image/png' ? _utilsBundle.PNG.sync.read(actualBuffer) : _utilsBundle.jpegjs.decode(actualBuffer, {
     maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB
   });
-  const expected = mimeType === 'image/png' ? _utilsBundle.PNG.sync.read(expectedBuffer) : _utilsBundle.jpegjs.decode(expectedBuffer, {
+  var expected = mimeType === 'image/png' ? _utilsBundle.PNG.sync.read(expectedBuffer) : _utilsBundle.jpegjs.decode(expectedBuffer, {
     maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB
   });
 
-  if (expected.width !== actual.width || expected.height !== actual.height) {
-    return {
-      errorMessage: `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `
-    };
+  if (expected.width !== actual.width) {
+     return {
+       errorMessage: `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `
+     };
+   }
+
+  var expectedImageData = expected.data;
+  var actualImageData = actual.data;
+  var imageWidth = expected.width;
+  var imageHeight = Math.max(expected.height, actual.height);
+
+  if (actualImageData.length < expectedImageData.length) {
+    var tmp = Buffer.alloc(expectedImageData.length);
+    actualImageData.copy(tmp)
+    actualImageData = tmp;
+  }
+  else if (expectedImageData.length < actualImageData.length) {
+    var tmp = Buffer.alloc(actualImageData.length);
+    expectedImageData.copy(tmp)
+    expectedImageData = tmp;
   }
 
   const diff = new _utilsBundle.PNG({
-    width: expected.width,
-    height: expected.height
+    width: imageWidth,
+    height: imageHeight
   });
-  const count = (0, _pixelmatch.default)(expected.data, actual.data, diff.data, expected.width, expected.height, {
+  
+  const count = (0, _pixelmatch.default)(expectedImageData, actualImageData, diff.data, imageWidth, imageHeight, {
     threshold: (_options$threshold = options.threshold) !== null && _options$threshold !== void 0 ? _options$threshold : 0.2
   });
   const maxDiffPixels1 = options.maxDiffPixels;
-  const maxDiffPixels2 = options.maxDiffPixelRatio !== undefined ? expected.width * expected.height * options.maxDiffPixelRatio : undefined;
+  const maxDiffPixels2 = options.maxDiffPixelRatio !== undefined ? imageWidth * imageHeight * options.maxDiffPixelRatio : undefined;
   let maxDiffPixels;
   if (maxDiffPixels1 !== undefined && maxDiffPixels2 !== undefined) maxDiffPixels = Math.min(maxDiffPixels1, maxDiffPixels2);else maxDiffPixels = (_ref = maxDiffPixels1 !== null && maxDiffPixels1 !== void 0 ? maxDiffPixels1 : maxDiffPixels2) !== null && _ref !== void 0 ? _ref : 0;
-  const ratio = Math.ceil(count / (expected.width * expected.height) * 100) / 100;
+  const ratio = Math.ceil(count / (imageWidth * imageHeight) * 100) / 100;
   return count > maxDiffPixels ? {
     errorMessage: `${count} pixels (ratio ${ratio.toFixed(2)} of all image pixels) are different`,
     diff: _utilsBundle.PNG.sync.write(diff)