diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 0000000..9a8684b --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,60 @@ +name: Testing for visual regression on old theme + +# Run the workflow when code is pushed or when a pull request is created +on: + push: + branches: + - '**' + pull_request: + branches: + - main + +jobs: + playwright: + name: Run Playwright + runs-on: ubuntu-latest + steps: + # Checkout the repository so the workflow has access to the code + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Hugo + uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0 + with: + hugo-version: "0.134.2" + extended: true + - name: Install dependencies and playwright browsers + run: cd tests && npm ci && npx playwright install --with-deps + - name: Run Playwright tests + id: test-visual + run: | + make tests | tee output.log + if grep -q -e "Error: A snapshot doesn't exist at" -e "Screenshot comparison failed" output.log; then + echo "Playwright tests failed due to a snapshot issue." + exit 1 + elif grep -q "failed" output.log; then + echo "Playwright tests failed due to a non-snapshot issue." + exit 1 + fi + - uses: actions/upload-artifact@v4 + id: artifact-upload + if: ${{ !cancelled() && failure() && steps.test-visual.conclusion == 'failure' }} + with: + name: playwright-report + path: tests/playwright-report/ + retention-days: 3 + - name: Comment on PR with Playwright report + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ failure() }} + with: + script: | + const body = `### Playwright visual snapshot differences were detected. + + View the [Playwright report](${{ steps.artifact-upload.outputs.artifact-url }}) + **To approve the snapshot changes and update the snapshots, please comment:** /approve-snapshots`; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body, + }); \ No newline at end of file diff --git a/.github/workflows/update-screenshot.yml b/.github/workflows/update-screenshot.yml new file mode 100644 index 0000000..bdfa4dd --- /dev/null +++ b/.github/workflows/update-screenshot.yml @@ -0,0 +1,58 @@ +name: Update screenshot on comment +on: + issue_comment: + types: [created] +jobs: + update-screenshots: + name: Update Screenshot + if: github.event.issue.pull_request && contains(github.event.comment.body, '/approve-snapshots') + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: xt0rted/pull-request-comment-branch@v3 + id: comment-branch + - uses: actions/checkout@v4 + if: success() + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + - name: Comment on PR with Playwright updates + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const body = `### Updating snapshots. Click [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) to see the status.`; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body, + }); + - name: Setup Hugo + uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0 + with: + hugo-version: "0.134.2" + extended: true + - name: Install dependencies and Playwright browsers + run: cd tests && npm ci && npx playwright install --with-deps + - name: Run Playwright update snapshots + id: test-visual + run: make tests-update-screenshots + - uses: actions/upload-artifact@v4 + id: artifact-upload + with: + name: screenshots + path: tests/src/__screenshots__ + - name: Comment on PR with success + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const body = `### Please download the artifacts [here](${{ steps.artifact-upload.outputs.artifact-url }}) and commit your updated screenshots.`; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body, + }); \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1c776a8..9c609be 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ Thumbs.db ######## *.log -node_modules/* +*/node_modules .markdownlint.json resources/* @@ -27,4 +27,9 @@ public/* exampleSite/public # Python -.venv \ No newline at end of file +.venv + +# Playwright +/coverage +*/test-results +*/playwright-report \ No newline at end of file diff --git a/Makefile b/Makefile index 89460d1..4c39071 100644 --- a/Makefile +++ b/Makefile @@ -6,16 +6,20 @@ WRITE_FLAG := --write list help:: $(info Available Make targets:) @echo "" - @echo " list | help: Print these available make targets" + @echo " list | help: Print these available make targets" @echo "" @echo " biome-format: Runs the biome formatter." @echo " biome-lint: Runs the biome linter." @echo " biome-all: Runs both the lint and formatting commands." - @echo "build-example-site: Builds hugo exampleSite." + @echo " build-example-site: Builds hugo exampleSite." @echo " (Set BIOME_ARGS to add additional arguments to biome command (ex: make biome-all BIOME_ARGS=write))" + @echo "" + @echo " setup-pre-commit: Sets up pre-commit (assuming it is installed)" + @echo "" + @echo " test: Runs playwright against the old theme." + @echo " tests-update-screenshots: Runs playwright against the old theme." -.PHONY: biome-format biome-lint biome-all setup-pre-commit build-example-site -BIOME_ARGS ?= +.PHONY: biome-format biome-lint biome-all setup-pre-commit tests build-example-site FLAG := ifeq ($(BIOME_ARGS), write) FLAG := $(WRITE_FLAG) @@ -40,3 +44,7 @@ setup-pre-commit: build-example-site: cd exampleSite && hugo mod get && hugo build --gc -e production +tests: + cd tests && npx playwright test +tests-update-screenshots: + cd tests && npx playwright test --update-snapshots diff --git a/tests/package-lock.json b/tests/package-lock.json new file mode 100644 index 0000000..ed364f1 --- /dev/null +++ b/tests/package-lock.json @@ -0,0 +1,78 @@ +{ + "name": "nginx-docs-theme-test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nginx-docs-theme-test", + "version": "1.0.0", + "devDependencies": { + "@playwright/test": "^1.48.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.50.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.50.1.tgz", + "integrity": "sha512-Jii3aBg+CEDpgnuDxEp/h7BimHcUTDlpEtce89xEumlJ5ef2hqepZ+PWp1DDpYC/VO9fmWVI1IlEaoI5fK9FXQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.50.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.50.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.1.tgz", + "integrity": "sha512-G8rwsOQJ63XG6BbKj2w5rHeavFjy5zynBA9zsJMMtBoe/Uf757oG12NXz6e6OirF7RCrTVAKFXbLmn1RbL7Qaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.50.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.50.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.1.tgz", + "integrity": "sha512-ra9fsNWayuYumt+NiM069M6OkcRb1FZSK8bgi66AtpFoWkg2+y0bJSNmkFrWhMbEBbVKC/EruAHH3g0zmtwGmQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/tests/package.json b/tests/package.json new file mode 100644 index 0000000..c4e3dbf --- /dev/null +++ b/tests/package.json @@ -0,0 +1,8 @@ +{ + "name": "nginx-docs-theme-test", + "version": "1.0.0", + "private": "true", + "devDependencies": { + "@playwright/test": "^1.48.0" + } +} diff --git a/tests/playwright.config.js b/tests/playwright.config.js new file mode 100644 index 0000000..7ff6f26 --- /dev/null +++ b/tests/playwright.config.js @@ -0,0 +1,45 @@ +import { defineConfig, devices } from '@playwright/test'; + +const BASE_URL = 'http://127.0.0.1'; +const PORT = 1313; +export default defineConfig({ + testDir: './src', + fullyParallel: true, + workers: 1, + outputDir: './test-results', + snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', + reporter: [['html', { outputFolder: './playwright-report' }]], + use: { + baseURL: `${BASE_URL}:${PORT}`, + screenshots: 'only-on-failure', + video: 'retain-on-failure', + trace: 'on-first-retry', + timezoneId: 'America/Los_Angeles', + }, + projects: [ + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + { + name: 'Mobile Chrome', + use: { ...devices['Pixel 5'] }, + }, + ], + webServer: { + command: `cd ../exampleSite && hugo mod get && hugo --gc -e production && hugo serve --port ${PORT}`, + url: `${BASE_URL}:${PORT}`, + stdout: 'ignore', + }, + expect: { + toHaveScreenshot: { + pathTemplate: + '{testDir}/__screenshots__{/projectName}/{testFilePath}/{arg}{ext}', + maxDiffPixels: 10, + }, + }, +}); diff --git a/tests/src/__screenshots__/Mobile-Chrome/visual-regression.spec.js/example-site-screenshot.png b/tests/src/__screenshots__/Mobile-Chrome/visual-regression.spec.js/example-site-screenshot.png new file mode 100644 index 0000000..90dd3e3 Binary files /dev/null and b/tests/src/__screenshots__/Mobile-Chrome/visual-regression.spec.js/example-site-screenshot.png differ diff --git a/tests/src/__screenshots__/firefox/visual-regression.spec.js/example-site-screenshot.png b/tests/src/__screenshots__/firefox/visual-regression.spec.js/example-site-screenshot.png new file mode 100644 index 0000000..5e6eee0 Binary files /dev/null and b/tests/src/__screenshots__/firefox/visual-regression.spec.js/example-site-screenshot.png differ diff --git a/tests/src/__screenshots__/webkit/visual-regression.spec.js/example-site-screenshot.png b/tests/src/__screenshots__/webkit/visual-regression.spec.js/example-site-screenshot.png new file mode 100644 index 0000000..ac91dd7 Binary files /dev/null and b/tests/src/__screenshots__/webkit/visual-regression.spec.js/example-site-screenshot.png differ diff --git a/tests/src/visual-regression.spec.js b/tests/src/visual-regression.spec.js new file mode 100644 index 0000000..13bb7b5 --- /dev/null +++ b/tests/src/visual-regression.spec.js @@ -0,0 +1,14 @@ +import { expect, test } from '@playwright/test'; + +test.describe('Testing old theme', () => { + test('should ensure the old theme does not visually regress', async ({ + page, + }) => { + const currentPlus = 'nginx/installing-nginx-open-source/'; + await page.goto(`/${currentPlus}`, { waitUntil: 'networkidle' }); + await page.waitForFunction(() => window.scrollY === 0); + await expect(page).toHaveScreenshot('example-site-screenshot.png', { + fullPage: true, + }); + }); +});