Playwright CI CD Pipeline

Playwright CI/CD Pipeline

When to Use

• Setting up automated Playwright runs on every PR
• Scheduling nightly regression suites
• Publishing test reports as CI artifacts
• Running cross-browser parallel tests in CI
• Triggering smoke tests on deployment

Procedure

1. GitHub Actions Pipeline

.github/workflows/playwright.yml

name: Playwright Tests

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main, develop]
  schedule:
    - cron: "0 2 * * *"   # Nightly at 2 AM

jobs:

  # ── Smoke Tests — runs on every PR push ─────────────────────
  smoke:
    name: Smoke Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          playwright install --with-deps chromium

      - name: Run Smoke Tests
        run: pytest tests/ -m smoke --html=reports/smoke_report.html --self-contained-html
        env:
          BASE_URL: ${{ vars.BASE_URL }}
          HEADLESS: "True"

      - name: Upload Smoke Report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: smoke-report
          path: reports/smoke_report.html

  # ── Cross-Browser Regression — runs on merge to main ────────
  regression:
    name: Regression (${{ matrix.browser }})
    runs-on: ubuntu-latest
    needs: smoke
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          playwright install --with-deps ${{ matrix.browser }}

      - name: Run Regression Tests
        run: |
          pytest tests/ -m regression \
            --html=reports/regression_${{ matrix.browser }}.html \
            --self-contained-html
        env:
          BROWSER: ${{ matrix.browser }}
          BASE_URL: ${{ vars.BASE_URL }}
          HEADLESS: "True"

      - name: Upload Regression Report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: regression-${{ matrix.browser }}
          path: reports/regression_${{ matrix.browser }}.html

  # ── Visual Regression — runs nightly ────────────────────────
  visual:
    name: Visual Regression Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule'
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          playwright install --with-deps chromium

      - name: Run Visual Tests
        run: pytest tests/visual/ -m visual --html=reports/visual_report.html
        env:
          BASE_URL: ${{ vars.BASE_URL }}
          HEADLESS: "True"

      - name: Upload Visual Report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: visual-regression-report
          path: |
            reports/visual_report.html
            reports/visual/

  # ── Accessibility Tests — runs nightly ──────────────────────
  accessibility:
    name: Accessibility Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule'
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          playwright install --with-deps chromium

      - name: Run A11y Tests
        run: pytest tests/accessibility/ -m accessibility --html=reports/a11y_report.html
        env:
          BASE_URL: ${{ vars.BASE_URL }}
          HEADLESS: "True"

      - name: Upload A11y Report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: accessibility-report
          path: |
            reports/a11y_report.html
            reports/accessibility/

  # ── API Tests — runs on every PR ────────────────────────────
  api:
    name: API Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Run API Tests
        run: pytest tests/api/ -m api --html=reports/api_report.html
        env:
          API_BASE_URL: ${{ vars.API_BASE_URL }}

2. Azure DevOps Pipeline

azure-pipelines.yml

trigger:
  branches:
    include:
      - main
      - develop

pr:
  branches:
    include:
      - main

schedules:
  - cron: "0 2 * * *"
    displayName: Nightly Regression
    branches:
      include:
        - main

variables:
  pythonVersion: "3.11"
  BASE_URL: $(BASE_URL)

stages:

  # ── Stage 1: Smoke Tests ─────────────────────────────────────
  - stage: Smoke
    displayName: Smoke Tests
    jobs:
      - job: SmokeTests
        displayName: Run Smoke Tests
        pool:
          vmImage: ubuntu-latest
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: $(pythonVersion)

          - script: |
              pip install -r requirements.txt
              playwright install --with-deps chromium
            displayName: Install Dependencies

          - script: |
              pytest tests/ -m smoke \
                --html=$(Build.ArtifactStagingDirectory)/smoke_report.html \
                --self-contained-html
            displayName: Run Smoke Tests
            env:
              HEADLESS: "True"
              BASE_URL: $(BASE_URL)

          - task: PublishBuildArtifacts@1
            condition: always()
            inputs:
              PathtoPublish: $(Build.ArtifactStagingDirectory)
              ArtifactName: smoke-report

  # ── Stage 2: Cross-Browser Regression ────────────────────────
  - stage: Regression
    displayName: Cross-Browser Regression
    dependsOn: Smoke
    condition: succeeded()
    jobs:
      - job: Chromium
        displayName: Chromium
        pool:
          vmImage: ubuntu-latest
        steps:
          - script: pip install -r requirements.txt ; playwright install --with-deps chromium
            displayName: Install
          - script: pytest tests/ -m regression --html=$(Build.ArtifactStagingDirectory)/chromium_report.html
            displayName: Test
            env:
              BROWSER: chromium
              HEADLESS: "True"
          - task: PublishBuildArtifacts@1
            condition: always()
            inputs:
              PathtoPublish: $(Build.ArtifactStagingDirectory)
              ArtifactName: regression-chromium

      - job: Firefox
        displayName: Firefox
        pool:
          vmImage: ubuntu-latest
        steps:
          - script: pip install -r requirements.txt ; playwright install --with-deps firefox
            displayName: Install
          - script: pytest tests/ -m regression --html=$(Build.ArtifactStagingDirectory)/firefox_report.html
            displayName: Test
            env:
              BROWSER: firefox
              HEADLESS: "True"
          - task: PublishBuildArtifacts@1
            condition: always()
            inputs:
              PathtoPublish: $(Build.ArtifactStagingDirectory)
              ArtifactName: regression-firefox

  # ── Stage 3: Nightly Full Suite ───────────────────────────────
  - stage: Nightly
    displayName: Nightly — Visual + A11y
    dependsOn: []
    condition: eq(variables['Build.Reason'], 'Schedule')
    jobs:
      - job: Visual
        displayName: Visual Regression
        pool:
          vmImage: ubuntu-latest
        steps:
          - script: pip install -r requirements.txt ; playwright install --with-deps chromium
            displayName: Install
          - script: pytest tests/visual/ -m visual --html=$(Build.ArtifactStagingDirectory)/visual_report.html
            displayName: Visual Tests
          - task: PublishBuildArtifacts@1
            condition: always()
            inputs:
              PathtoPublish: $(Build.ArtifactStagingDirectory)
              ArtifactName: visual-report

      - job: Accessibility
        displayName: Accessibility
        pool:
          vmImage: ubuntu-latest
        steps:
          - script: pip install -r requirements.txt ; playwright install --with-deps chromium
            displayName: Install
          - script: pytest tests/accessibility/ -m accessibility --html=$(Build.ArtifactStagingDirectory)/a11y_report.html
            displayName: A11y Tests
          - task: PublishBuildArtifacts@1
            condition: always()
            inputs:
              PathtoPublish: $(Build.ArtifactStagingDirectory)
              ArtifactName: accessibility-report

3. Pipeline Summary

Best Practices

• Run smoke first, gate regression on smoke passing
• Use fail-fast: false for cross-browser — see all browser results
• Always upload reports as artifacts — even on failure
• Store BASE_URL as a pipeline variable, never hardcode
• Use playwright install --with-deps in CI — includes system libs