Merge branch '4.x' of github.com:codeceptjs/CodeceptJS into 4.x

Author: DavertMik

README.md

@@ -251,20 +251,22 @@ Add the `testomatio` plugin to your `codecept.conf.js`:
 plugins: {
   testomatio: {
     enabled: true,
-    require: '@testomatio/reporter/lib/adapter/codecept',
+    require: '@testomatio/reporter/codecept',
+    html: true,
+    reportDir: 'output/report',
   },
 }
 ```
 
 ### Usage
 
-Generate HTML reports by setting the `TESTOMATIO_HTML_REPORT_SAVE` environment variable:
+Run tests normally:
 
 ```sh
-TESTOMATIO_HTML_REPORT_SAVE=1 npx codeceptjs run
+npx codeceptjs run
 ```
 
-The report will be saved to `html-report/testomatio-report.html`.
+The report will be saved to `output/report/testomatio-report.html` by default. You can also keep using `TESTOMATIO_HTML_REPORT_SAVE=1` and related environment variables if you prefer env-based setup.
 
 ### Features
 
@@ -278,14 +280,11 @@ The report will be saved to `html-report/testomatio-report.html`.
 ### Customization
 
 ```sh
-# Custom output folder
-TESTOMATIO_HTML_REPORT_SAVE=1 TESTOMATIO_HTML_REPORT_FOLDER=./reports npx codeceptjs run
-
-# Custom filename
-TESTOMATIO_HTML_REPORT_SAVE=1 TESTOMATIO_HTML_FILENAME=my-report.html npx codeceptjs run
+# Custom output folder in codecept.conf.js
+# reportDir: './reports'
 
 # Integrate with Testomat.io cloud
-TESTOMATIO_HTML_REPORT_SAVE=1 TESTOMATIO=your_api_key npx codeceptjs run
+TESTOMATIO=your_api_key npx codeceptjs run
 ```
 
 ## PageObjects

docs/images/testomatio-html-report.png

@@ -171,16 +171,17 @@ npm install --save-dev @testomatio/reporter
 plugins: {
   testomatio: {
     enabled: true,
-    require: '@testomatio/reporter/lib/adapter/codecept',
+    require: '@testomatio/reporter/codecept',
+    html: true,
   },
 }
 ```
 
 ```bash
-TESTOMATIO_DISABLE_UPLOAD=1 npx codeceptjs run
+npx codeceptjs run
 ```
 
-The HTML report is written to `output/reports/`. See [Reports → HTML](/reports) for pipe options.
+The HTML report is written to `output/report/` by default. See [Reports > HTML](/reports) for pipe options.
 
 Reporting notes:
 

docs/images/testomatio-markdown-report.png

@@ -28,10 +28,9 @@ Run tests with plugin enabled:
 #### Configuration:
 
 *   `retries` - number of retries (by default 3),
-*   `when` - function, when to perform a retry (accepts error as parameter)
 *   `factor` - The exponential factor to use. Default is 1.5.
-*   `minTimeout` - The number of milliseconds before starting the first retry. Default is 1000.
-*   `maxTimeout` - The maximum number of milliseconds between two retries. Default is Infinity.
+*   `minTimeout` - The number of milliseconds before starting the first retry. Default is 150.
+*   `maxTimeout` - The maximum number of milliseconds between two retries. Default is 10000.
 *   `randomize` - Randomizes the timeouts by multiplying with a factor from 1 to 2. Default is false.
 *   `defaultIgnoredSteps` - an array of steps to be ignored for retry. Includes:
     *   `amOnPage`

docs/migration-4.md

@@ -21,13 +21,19 @@ plugins: {
   testomatio: {
     enabled: true,
     require: '@testomatio/reporter/codecept',
+    html: true,
+    markdown: true,
+    csv: true,
+    reportDir: 'output/report',
   },
 }
 ```
 
+The local reports above are enabled directly from CodeceptJS config. If `reportDir` is omitted, reports are written to `output/report` using the CodeceptJS `output` directory.
+
 ### Enable an output
 
-Each output turns on when you set its environment variable. Run your tests as usual — one run feeds every output you enabled.
+Each output can also be enabled with environment variables. Run your tests as usual and one run feeds every output you enabled.
 
 | To get… | Set | Details |
 | --- | --- | --- |
@@ -63,7 +69,9 @@ The GitHub pipe also needs the job to grant `permissions: pull-requests: write`.
 
 A single self-contained HTML file with the run summary and, per test, its steps, screenshots, logs, and error. It needs no API key and no service, so it works anywhere — open it locally or attach it to a CI build.
 
-![HTML report](https://raw.githubusercontent.com/testomatio/reporter/master/docs/pipes/images/html-pipe.png)
+![HTML report](./images/testomatio-html-report.png)
+
+- Preferred in CodeceptJS 4: enable `html: true` in `plugins.testomatio` and run `npx codeceptjs run`
 
 - `TESTOMATIO_HTML_REPORT_SAVE=1` — enable the report
 - `TESTOMATIO_HTML_REPORT_FOLDER=output/reports` — keep it inside CodeceptJS's `output/` dir (default folder is `html-report`)
@@ -118,6 +126,10 @@ Posts a comment to the Pull Request with the same summary. Comments are created
 
 ### Markdown Report
 
+- Preferred in CodeceptJS 4: enable `markdown: true` in `plugins.testomatio` and run `npx codeceptjs run`
+
+![Markdown report](./images/testomatio-markdown-report.png)
+
 A single self-contained Markdown file — renders in PR comments, CI job summaries, and Slack, and is convenient for AI agents reading test results. Needs no API key.
 
 - `TESTOMATIO_MARKDOWN_REPORT_SAVE=1` — enable the report

docs/plugins/retryFailedStep.md

@@ -20,24 +20,36 @@ CodeceptJS provides flexible retry mechanisms to handle flaky tests. Use retries
 
 ## Helper Retries
 
-Browser automation helpers (Playwright, Puppeteer, WebDriver) have **built-in retry mechanisms** for element interactions. When you call `I.click('Button')`, Playwright automatically waits for the element to exist, be visible, stable, and enabled — retrying for up to 5 seconds.
+Plawright has a built-in retry mechanism for element interactions. When you call `I.click('Button')`, after the element is located Playwright keeps retrying until it is actionable — up to `timeout` (default 5s).
 
-Configure the timeout in your helper settings:
+> WebDriver has a different auto-retry option: [smartWait](/webdriver#smartwait)
+
+Even though the handle exists (from `.all()`), Playwright still waits for it to become visible, stable (not mid-animation), enabled, not covered by an overlay/modal, and not rerendering.
 
 ```js
 helpers: {
   Playwright: {
-    timeout: 5000,      // retry actions for up to 5 seconds
-    waitForAction: 100  // wait 100ms before each action
+    timeout: 5000,      // retry the action until the element is actionable
+    waitForAction: 100  // fixed pause AFTER click/doubleClick/pressKey
   }
 }
 ```
 
-**Learn more:** [Playwright Helper](/helpers/Playwright), [Timeouts](/timeouts)
+What each setting does:
+
+```
+find element (no wait — fails instantly if locator matches nothing)
+  → wait up to `timeout` for it to become actionable   ← timeout
+  → perform action
+  → sleep `waitForAction` ms                            ← waitForAction (settle pause, not a wait)
+```
+
+`timeout` covers the action. If the locator matches nothing yet, the step fails immediately. Use [Failed Step Retries](#failed-step-retries) to cover that gap.
+
 
 ## Failed Step Retries
 
-Automatically retry all failed steps without modifying test code:
+CodeceptJS retries all failed steps by default by using the `retryFailedStep` plugin.
 
 ```js
 plugins: {
@@ -66,18 +78,36 @@ Scenario('manual retries only', { disableRetryFailedStep: true }, ({ I }) => {
 })
 ```
 
-Full plugin options:
-
-| Option | Default | Description |
-|--------|---------|-------------|
-| `retries` | — | Retries per step |
-| `minTimeout` | — | Milliseconds before first retry |
-| `maxTimeout` | `Infinity` | Max milliseconds between retries |
-| `factor` | — | Exponential backoff multiplier |
-| `randomize` | `false` | Randomize timeout intervals |
-| `ignoredSteps` | `[]` | Patterns/regex of steps to never retry |
-| `deferToScenarioRetries` | `true` | Disable step retries when scenario retries exist |
-| `when` | `() => true` | Function receiving error; return `true` to retry |
+Defaults: `minTimeout: 150`, `factor: 1.5`, `maxTimeout: 10000`.
+
+
+> See [plugin reference](/plugins/retry-failed-step) for more options
+
+Retries are calculated via this formula:
+
+```
+gap(N) = min(minTimeout × factor^(N-1), maxTimeout)
+```
+
+Practically if step fails it will trigger a retry with increasing delay until `maxTimeout` is reached:
+
+```
+retries: 2                              => 0.15s-0.4s (150,225ms)
+retries: 3                              => 0.15s-0.7s (150,225,338ms)
+retries: 3, minTimeout: 1000            => 1s-4.75s   (1s,1.5s,2.25s)
+retries: 3, minTimeout: 1000, factor: 2 => 1s-7s      (1s,2s,4s)
+retries: 5, minTimeout: 1000, factor: 2 => 1s-25s     (1s,2s,4s,8s,10s)
+```
+
+Playwright `timeout` adds to each attempt only when the element is found:
+
+- `Playwright.timeout: 5000`
+- `retries: 2, minTimeout: 1000`
+
+```
+element not found => 0 + (1s+1s)    = 2s
+element found but not interactable  => 3×5s + (1s+1s) = 17s
+```
 
 ## Manual Step Retries
 

docs/reports.md

@@ -8,7 +8,10 @@ const debug = debugModule('codeceptjs:retryFailedStep')
 const defaultConfig = {
   retries: 3,
   defaultIgnoredSteps: ['amOnPage', 'wait*', 'send*', 'execute*', 'run*', 'have*'],
+  minTimeout: 150,
+  maxTimeout: 10000,
   factor: 1.5,
+  randomize: false,
   ignoredSteps: [],
   deferToScenarioRetries: true,
 }
@@ -44,10 +47,9 @@ const RETRY_PRIORITIES = {
  * #### Configuration:
  *
  * * `retries` - number of retries (by default 3),
- * * `when` - function, when to perform a retry (accepts error as parameter)
  * * `factor` - The exponential factor to use. Default is 1.5.
- * * `minTimeout` - The number of milliseconds before starting the first retry. Default is 1000.
- * * `maxTimeout` - The maximum number of milliseconds between two retries. Default is Infinity.
+ * * `minTimeout` - The number of milliseconds before starting the first retry. Default is 150.
+ * * `maxTimeout` - The maximum number of milliseconds between two retries. Default is 10000.
  * * `randomize` - Randomizes the timeouts by multiplying with a factor from 1 to 2. Default is false.
  * * `defaultIgnoredSteps` - an array of steps to be ignored for retry. Includes:
  *     * `amOnPage`
@@ -89,9 +91,8 @@ const RETRY_PRIORITIES = {
  *
  */
 export default function (config) {
-  config = Object.assign(defaultConfig, config)
+  config = Object.assign({}, defaultConfig, config)
   config.ignoredSteps = config.ignoredSteps.concat(config.defaultIgnoredSteps)
-  const customWhen = config.when
 
   let enableRetry = false
 
@@ -101,7 +102,6 @@ export default function (config) {
     if (!store.autoRetries) return false
     if (err && err.isTerminal) return false
     if (err && err.message && (err.message.includes('ERR_ABORTED') || err.message.includes('frame was detached') || err.message.includes('Target page, context or browser has been closed'))) return false
-    if (customWhen) return customWhen(err)
     return true
   }
   config.when = when

docs/retry.md

@@ -76,6 +76,35 @@ describe('retryFailedStep', () => {
     expect(counter).to.equal(2)
   })
 
+  describe('config', () => {
+    it('applies default retry timing', () => {
+      retryFailedStep({})
+      const cfg = recorder.retries.find(r => r.deferToScenarioRetries !== undefined)
+      expect(cfg.retries).to.equal(3)
+      expect(cfg.minTimeout).to.equal(150)
+      expect(cfg.maxTimeout).to.equal(10000)
+      expect(cfg.factor).to.equal(1.5)
+    })
+
+    it('overrides retry timing from config', () => {
+      retryFailedStep({ retries: 5, minTimeout: 1000, maxTimeout: 3000, factor: 2 })
+      const cfg = recorder.retries.find(r => r.deferToScenarioRetries !== undefined)
+      expect(cfg.retries).to.equal(5)
+      expect(cfg.minTimeout).to.equal(1000)
+      expect(cfg.maxTimeout).to.equal(3000)
+      expect(cfg.factor).to.equal(2)
+    })
+
+    it('does not leak config between instances', () => {
+      retryFailedStep({ retries: 5, minTimeout: 1000 })
+      recorder.retries = []
+      retryFailedStep({})
+      const cfg = recorder.retries.find(r => r.deferToScenarioRetries !== undefined)
+      expect(cfg.retries).to.equal(3)
+      expect(cfg.minTimeout).to.equal(150)
+    })
+  })
+
   it('should not retry steps with wait*', async () => {
     retryFailedStep({ retries: 2, minTimeout: 1 })
     event.dispatcher.emit(event.test.before, createTest('test'))