Added playwright for visual regression test

.github/workflows/playwright.yml

@@ -0,0 +1,60 @@
+name: Testing for visual regression on old theme
+
+# Run the workflow when code is pushed or when a pull request is created
+on:
+    push:
+      branches:
+        - '**'
+    pull_request:
+      branches:
+        - main
+
+jobs:
+    playwright:
+        name: Run Playwright
+        runs-on: ubuntu-latest
+        steps: 
+          # Checkout the repository so the workflow has access to the code
+          - name: Checkout code
+            uses: actions/checkout@v3
+          - name: Setup Hugo
+            uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
+            with:
+                hugo-version: "0.134.2"
+                extended: true
+          - name: Install dependencies and playwright browsers
+            run: cd tests && npm ci && npx playwright install --with-deps
+          - name: Run Playwright tests
+            id: test-visual
+            run: |
+              make tests | tee output.log
+              if grep -q -e "Error: A snapshot doesn't exist at" -e "Screenshot comparison failed" output.log; then
+                echo "Playwright tests failed due to a snapshot issue."
+                exit 1
+              elif grep -q "failed" output.log; then
+                echo "Playwright tests failed due to a non-snapshot issue."
+                exit 1
+              fi
+          - uses: actions/upload-artifact@v4
+            id: artifact-upload
+            if: ${{ !cancelled() && failure() && steps.test-visual.conclusion == 'failure' }}
+            with:
+              name: playwright-report
+              path: tests/playwright-report/
+              retention-days: 3
+          - name: Comment on PR with Playwright report
+            uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+            if: ${{ failure() }}
+            with:
+              script: |
+                const body = `### <span aria-hidden="true">❌</span> Playwright visual snapshot differences were detected.
+
+                View the [Playwright report](${{ steps.artifact-upload.outputs.artifact-url }}) 
+                **To approve the snapshot changes and update the snapshots, please comment:** /approve-snapshots`;
+
+                await github.rest.issues.createComment({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: body,
+                });

.github/workflows/update-screenshot.yml

@@ -0,0 +1,58 @@
+name: Update screenshot on comment
+on:
+    issue_comment:
+       types: [created]
+jobs:
+  update-screenshots:
+    name: Update Screenshot
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/approve-snapshots')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+        - uses: xt0rted/pull-request-comment-branch@v3
+          id: comment-branch
+        - uses: actions/checkout@v4
+          if: success()
+          with:
+            ref: ${{ steps.comment-branch.outputs.head_ref }}
+        - name: Comment on PR with Playwright updates
+          uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+          with:
+            script: |
+                const body = `### Updating snapshots. Click [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) to see the status.`;
+
+                await github.rest.issues.createComment({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    body: body,
+                });
+        - name: Setup Hugo
+          uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
+          with:
+              hugo-version: "0.134.2"
+              extended: true
+        - name: Install  dependencies and Playwright browsers
+          run: cd tests && npm ci && npx playwright install --with-deps
+        - name: Run Playwright update snapshots
+          id: test-visual
+          run: make tests-update-screenshots
+        - uses: actions/upload-artifact@v4
+          id: artifact-upload
+          with:
+            name: screenshots
+            path: tests/src/__screenshots__
+        - name: Comment on PR with success
+          uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+          with:
+            script: |
+                const body = `### Please download the artifacts [here](${{ steps.artifact-upload.outputs.artifact-url }}) and commit your updated screenshots.`;
+
+                await github.rest.issues.createComment({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    body: body,
+                });

.gitignore

@@ -15,7 +15,7 @@ Thumbs.db
 ########
 *.log
 
-node_modules/*
+*/node_modules
 .markdownlint.json
 resources/*
 
@@ -27,4 +27,9 @@ public/*
 exampleSite/public
 
 # Python
-.venv
+.venv
+
+# Playwright
+/coverage
+*/test-results
+*/playwright-report

Makefile

@@ -6,16 +6,20 @@ WRITE_FLAG := --write
 list help::
 	$(info Available Make targets:)
 	@echo "<COMMON>"
-	@echo "		list | help:  		Print these available make targets"
+	@echo "		  list | help:  		Print these available make targets"
 	@echo "<LINTING AND FORMATTING>"
 	@echo "		 biome-format: 		Runs the biome formatter."
 	@echo "		   biome-lint: 		Runs the biome linter."
 	@echo "		    biome-all:		Runs both the lint and formatting commands."
-	@echo "build-example-site:		Builds hugo exampleSite."
+	@echo "	   build-example-site:		Builds hugo exampleSite."
 	@echo "  (Set BIOME_ARGS to add additional arguments to biome command (ex: make biome-all BIOME_ARGS=write))"
+	@echo "<PRE-COMMIT>"
+	@echo "	     setup-pre-commit:		Sets up pre-commit (assuming it is installed)"
+	@echo "<PLAYWRIGHT TESTS>"
+	@echo "		 	 test: 		Runs playwright against the old theme."
+	@echo "     tests-update-screenshots:		Runs playwright against the old theme."
 
-.PHONY: biome-format biome-lint biome-all setup-pre-commit build-example-site
-BIOME_ARGS ?= 
+.PHONY: biome-format biome-lint biome-all setup-pre-commit tests build-example-site
 FLAG :=
 ifeq ($(BIOME_ARGS), write)
 	FLAG := $(WRITE_FLAG)
@@ -40,3 +44,7 @@ setup-pre-commit:
 
 build-example-site:
 	cd exampleSite && hugo mod get && hugo build --gc -e production
+tests:
+	cd tests && npx playwright test
+tests-update-screenshots:
+	cd tests && npx playwright test --update-snapshots

tests/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "nginx-docs-theme-test",
+  "version": "1.0.0",
+  "private": "true",
+  "devDependencies": {
+    "@playwright/test": "^1.48.0"
+  }
+}

tests/playwright.config.js

@@ -0,0 +1,45 @@
+import { defineConfig, devices } from '@playwright/test';
+
+const BASE_URL = 'http://127.0.0.1';
+const PORT = 1313;
+export default defineConfig({
+  testDir: './src',
+  fullyParallel: true,
+  workers: 1,
+  outputDir: './test-results',
+  snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}',
+  reporter: [['html', { outputFolder: './playwright-report' }]],
+  use: {
+    baseURL: `${BASE_URL}:${PORT}`,
+    screenshots: 'only-on-failure',
+    video: 'retain-on-failure',
+    trace: 'on-first-retry',
+    timezoneId: 'America/Los_Angeles',
+  },
+  projects: [
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+    {
+      name: 'Mobile Chrome',
+      use: { ...devices['Pixel 5'] },
+    },
+  ],
+  webServer: {
+    command: `cd ../exampleSite && hugo mod get && hugo --gc -e production && hugo serve --port ${PORT}`,
+    url: `${BASE_URL}:${PORT}`,
+    stdout: 'ignore',
+  },
+  expect: {
+    toHaveScreenshot: {
+      pathTemplate:
+        '{testDir}/__screenshots__{/projectName}/{testFilePath}/{arg}{ext}',
+      maxDiffPixels: 10,
+    },
+  },
+});

tests/src/visual-regression.spec.js

@@ -0,0 +1,14 @@
+import { expect, test } from '@playwright/test';
+
+test.describe('Testing old theme', () => {
+  test('should ensure the old theme does not visually regress', async ({
+    page,
+  }) => {
+    const currentPlus = 'nginx/installing-nginx-open-source/';
+    await page.goto(`/${currentPlus}`, { waitUntil: 'networkidle' });
+    await page.waitForFunction(() => window.scrollY === 0);
+    await expect(page).toHaveScreenshot('example-site-screenshot.png', {
+      fullPage: true,
+    });
+  });
+});