diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..d3603cbbf9 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,98 @@ +name: E2E Testing +on: + workflow_dispatch: + pull_request: + branches: [develop] + issue_comment: + types: [created] +jobs: + test-e2e: + # temporary while new e2e environment is not yet created: + # only trigger on workflow_dispatch, maintainer comment, or if PR is from GSOC project mentor or mentee + if: > + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'pull_request' && + contains(fromJSON('["Geethegreat", "clairep94"]'), github.event.pull_request.user.login) + ) || + ( + github.event.issue.pull_request && + github.event.comment.body == '/e2e' && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) + timeout-minutes: 60 + runs-on: ubuntu-latest + + steps: + - name: Resolve PR head ref + if: github.event_name == 'issue_comment' + id: pr + uses: actions/github-script@v7 + with: + script: | + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput('sha', pr.head.sha); + + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && steps.pr.outputs.sha || github.ref }} + + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: '18.20.x' + cache: 'npm' + + - name: Create .env file + run: cp .env.example .env + + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.10.0 + with: + mongodb-version: '6.0' + + - name: Install dependencies + run: npm ci + + - name: Cache Playwright browsers + uses: actions/cache@v4 + id: playwright-cache + with: + path: ~/.cache/ms-playwright + key: playwright-${{ hashFiles('package-lock.json') }} + + - name: Install Playwright Browsers + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps chromium + + - name: Install Playwright deps (OS libraries) + if: steps.playwright-cache.outputs.cache-hit == 'true' + run: npx playwright install-deps chromium + + - name: Start app + run: npm start > app.log 2>&1 & + + - name: Wait for app to be ready + run: | + npx wait-on@7 http://localhost:8000 --timeout 180000 || { + echo "::error::App failed to become ready at http://localhost:8000 within 180s" + echo "----- app.log -----" + cat app.log + exit 1 + } + + - name: Run Playwright tests + run: npm run e2e:ci + + - name: Upload artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index bfedbe9b96..3aedaa900d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,11 @@ duplicates.json coverage -*.tsbuildinfo \ No newline at end of file +*.tsbuildinfo + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +/playwright/.auth/ diff --git a/e2e/editor.spec.ts b/e2e/editor.spec.ts new file mode 100644 index 0000000000..99eb31123d --- /dev/null +++ b/e2e/editor.spec.ts @@ -0,0 +1,60 @@ +import { test, expect } from '@playwright/test'; + +test.describe('editor page', () => { + test.beforeEach(async ({ page }) => { + await page.goto('/'); + + // initial pageload check to fail fast: + await expect(page.locator('a.skip_link[href="#play-sketch"]')).toHaveText( + 'Skip to Play Sketch' + ); + + // Dismiss cookie banner if it appears + // Note: we do document.querySelectorAll instead of page.locator as workaround due to the buttons on the banner being beyond the viewport + await page.evaluate(() => { + const btn = Array.from(document.querySelectorAll('button')).find((b) => + /allow essential|allow all/i.test(b.textContent ?? '') + ) as HTMLElement | undefined; + btn?.click(); + }); + + // wait for page to fully load with all main IDE components: + await expect(page.locator('#play-sketch')).toBeVisible(); // play button + await expect(page.locator('iframe[title="sketch preview"]')).toBeVisible(); // sketch preview + await expect(page.locator('.preview-console')).toBeVisible(); // editor console + await expect(page.locator('.editor-holder')).toBeVisible(); // editor -- NOTE: .editor-holder .CodeMirror cannot be found on CI for some reason, so we are using .editor-holder instead. + }); + + test('can run sketch code written in the editor', async ({ page }) => { + const newCode = [ + 'function setup() {', + ' createCanvas(400, 400);', + '}', + '', + 'function draw() {', + ' background(220);', + " console.log('hi from sketch');", + ' noLoop();', + '}' + ].join(''); // Purposely joining without '\n' to avoid triggering the autocomplete with keyboard.type & creating extra brackets + + // Find editor text area, clear default code & type in the new code + const editor = page.locator('.editor-holder'); + await editor.click(); + await page.keyboard.press('ControlOrMeta+A'); + await page.keyboard.type(newCode, { delay: 5 }); // Purposely using .type instead of .insert (.insert does not work with the redux state management) + + // Click Play + await page.locator('#play-sketch').click({ force: true }); + + // Wait for the sketch iframe src to confirm the sketch actually started + await expect( + page.locator('iframe[title="sketch preview"]') + ).toHaveAttribute('src', /8002/, { timeout: 10_000 }); + + // Assert console output + await expect( + page.locator('.preview-console__messages') + ).toContainText('hi from sketch', { timeout: 15_000 }); + }); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 0000000000..7a2e174acb --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "lib": ["ES2022", "dom"], + "types": ["node"] + }, + "include": ["./**/*", "../playwright.config.ts"], + "exclude": ["../node_modules"] +} diff --git a/package-lock.json b/package-lock.json index eac9859017..043700f10b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -146,6 +146,7 @@ "@babel/preset-env": "^7.14.7", "@babel/preset-react": "^7.14.5", "@babel/preset-typescript": "^7.27.1", + "@playwright/test": "^1.61.1", "@storybook/addon-actions": "^7.6.8", "@storybook/addon-docs": "^7.6.8", "@storybook/addon-essentials": "^7.6.8", @@ -10100,6 +10101,22 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.11", "resolved": "https://registry.npmjs.org/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz", @@ -15744,7 +15761,8 @@ "node_modules/@types/node": { "version": "16.18.126", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz", - "integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==" + "integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==", + "license": "MIT" }, "node_modules/@types/node-fetch": { "version": "2.6.4", @@ -32260,6 +32278,38 @@ "node": ">=6" } }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/please-upgrade-node": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", @@ -46344,6 +46394,15 @@ "dev": true, "optional": true }, + "@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "requires": { + "playwright": "1.61.1" + } + }, "@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.11", "resolved": "https://registry.npmjs.org/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz", @@ -62479,6 +62538,22 @@ "find-up": "^3.0.0" } }, + "playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.61.1" + } + }, + "playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true + }, "please-upgrade-node": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", diff --git a/package.json b/package.json index da0b9fd4da..0f4581b338 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "build:client": "cross-env NODE_ENV=production webpack --config webpack/config.prod.js", "build:server": "cross-env NODE_ENV=production webpack --config webpack/config.server.js", "build:examples": "cross-env NODE_ENV=production webpack --config webpack/config.examples.js", + "e2e": "npx playwright test --ui", + "e2e:headed": "npx playwright test --headed", + "e2e:ci": "npx playwright test", "test": "NODE_ENV=test jest", "test:watch": "NODE_ENV=test jest --watch", "test:ci": "npm run lint && npm run test", @@ -121,6 +124,7 @@ "@babel/preset-env": "^7.14.7", "@babel/preset-react": "^7.14.5", "@babel/preset-typescript": "^7.27.1", + "@playwright/test": "^1.61.1", "@storybook/addon-actions": "^7.6.8", "@storybook/addon-docs": "^7.6.8", "@storybook/addon-essentials": "^7.6.8", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000000..379f33091c --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,81 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './e2e', + /* Timeout per individual test (w/ before & after hooks). Make CI longer than default 30s to accomodate */ + timeout: process.env.CI ? 60_000 : 30_000, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + + retries: 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('')`. */ + baseURL: 'http://localhost:8000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry' + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] } + } + + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] } + // }, + + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] } + // } + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ] + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://localhost:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/tsconfig.json b/tsconfig.json index 97fe81b722..c903c74cf2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,7 @@ "files": [], "references": [ { "path": "./client" }, - { "path": "./server" } + { "path": "./server" }, + { "path": "./e2e" } ], } \ No newline at end of file