From 5f383219b2e9a204cd3d9e8614ea25716e3fada5 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 12:56:48 +0200 Subject: [PATCH 01/10] chore: Add test:stale script for running state_test.js --- package.json | 1 + test/data/sandbox/codecept.stale.js | 17 ++++++++++ test/data/sandbox/stale_test.stale.js | 47 +++++++++++++++++++++++++++ test/runner/stale_test.js | 25 ++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 test/data/sandbox/codecept.stale.js create mode 100644 test/data/sandbox/stale_test.stale.js create mode 100644 test/runner/stale_test.js diff --git a/package.json b/package.json index 2c56b255b..29ab53926 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "docs": "./runok.js docs", "test:unit": "mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", + "test:stale": "mocha test/runner/stale_test.js", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", "test:appium-other": "mocha test/helper/AppiumV2_test.js --grep 'second'", diff --git a/test/data/sandbox/codecept.stale.js b/test/data/sandbox/codecept.stale.js new file mode 100644 index 000000000..f9a26ec2c --- /dev/null +++ b/test/data/sandbox/codecept.stale.js @@ -0,0 +1,17 @@ +exports.config = { + tests: './*_test.stale.js', + // timeout: 10000, + retry: { + Scenario: 2, + After: 0, + Before: 1, + }, + output: './output', + helpers: { + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'sandbox', + }; + \ No newline at end of file diff --git a/test/data/sandbox/stale_test.stale.js b/test/data/sandbox/stale_test.stale.js new file mode 100644 index 000000000..2ffef0d62 --- /dev/null +++ b/test/data/sandbox/stale_test.stale.js @@ -0,0 +1,47 @@ +const { I } = inject(); + +Feature("Stale test"); + +/** + * Retry on config is not the same at retry on test + */ + +let tries = 0; + +Scenario("Try to stale flaky test", async () => { + tries++; + throw new Error("@@@Flaky error"); + // await new Promise((resolve) => setTimeout(() => { + // }, 500)); +}) + +Scenario.skip("2 - Try to stale flaky test", async () => { + tries++; + setTimeout(() => { + throw new Error("@@@Flaky error"); + }, 500); + await new Promise((resolve) => setTimeout(resolve, 300)); + throw new Error("@@@Flaky error"); +}).retry(2); + +After(() => { + console.log(`[T] Retries: ${tries}`); +}); + +// Works but we don't know why on retry(1) +Scenario.skip("Try to stale flaky test", async () => { + tries++; + setTimeout(() => { + throw new Error("@@@Flaky error"); + }, 500); + await new Promise((resolve) => setTimeout(resolve, 701)); +}).retry(1); + +// Lead to error "done() multiple time" because error is throw at the same time that test is over +Scenario.skip("Try to stale flaky test", async () => { + tries++; + setTimeout(() => { + throw new Error("@@@Flaky error"); + }, 500); + await new Promise((resolve) => setTimeout(resolve, 500)); +}); \ No newline at end of file diff --git a/test/runner/stale_test.js b/test/runner/stale_test.js new file mode 100644 index 000000000..b33e0105e --- /dev/null +++ b/test/runner/stale_test.js @@ -0,0 +1,25 @@ +const { expect } = require('expect'); +const path = require('path'); +const exec = require('child_process').exec; + +const runner = path.join(__dirname, '/../../bin/codecept.js'); +const codecept_dir = path.join(__dirname, '/../data/sandbox'); +const codecept_run = `${runner} run`; +const config_run_config = config => `${codecept_run} --config ${codecept_dir}/${config}`; + +describe('CodeceptJS Interface', () => { + before(() => { + process.chdir(codecept_dir); + }); + + it('Should always failed and terminate', (done) => { + exec(config_run_config('codecept.stale.js'), (err, stdout) => { + console.log('err', err); + console.log('stdout',stdout) + expect(stdout).toContain('@@@Flaky error'); // feature + expect(err).toBeTruthy(); + done(); + }); + }); + +}); From 769b8d6ab4de9d4085483c07906983cb0d333ad1 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 15:56:12 +0200 Subject: [PATCH 02/10] fix: handle async function better to prevent stale the process --- lib/scenario.js | 116 +++++++++++++------------- package.json | 2 +- test/acceptance/retryTo_test.js | 10 +++ test/data/sandbox/codecept.stale.js | 9 +- test/data/sandbox/stale_test.stale.js | 26 ++++-- test/runner/stale_test.js | 2 - 6 files changed, 91 insertions(+), 74 deletions(-) diff --git a/lib/scenario.js b/lib/scenario.js index 40f5759a1..bc85409f2 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -1,9 +1,9 @@ -const promiseRetry = require('promise-retry'); -const event = require('./event'); -const recorder = require('./recorder'); -const assertThrown = require('./assert/throws'); -const { ucfirst, isAsyncFunction } = require('./utils'); -const parser = require('./parser'); +const promiseRetry = require("promise-retry"); +const event = require("./event"); +const recorder = require("./recorder"); +const assertThrown = require("./assert/throws"); +const { ucfirst, isAsyncFunction } = require("./utils"); +const parser = require("./parser"); const injectHook = function (inject, suite) { try { @@ -35,9 +35,10 @@ module.exports.test = (test) => { test.fn = function (done) { recorder.errHandler((err) => { - recorder.session.start('teardown'); + recorder.session.start("teardown"); recorder.cleanAsyncErr(); - if (test.throws) { // check that test should actually fail + if (test.throws) { + // check that test should actually fail try { assertThrown(err, test.throws); event.emit(event.test.passed, test); @@ -55,36 +56,22 @@ module.exports.test = (test) => { if (isAsyncFunction(testFn)) { event.emit(event.test.started, test); - - const catchError = e => { - recorder.throw(e); - recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); - recorder.session.start('teardown'); - recorder.cleanAsyncErr(); - event.emit(event.test.failed, test, err); - event.emit(event.test.finished, test); - recorder.add(() => done(err)); + testFn + .call(test, getInjectedArguments(testFn, test)) + .then(() => { + recorder.add("fire test.passed", () => { + event.emit(event.test.passed, test); + event.emit(event.test.finished, test); + }); + recorder.add("finish test", () => done()); + }) + .catch((err) => { + recorder.throw(err); + }) + .finally(() => { + recorder.catch(); }); - }; - - let injectedArguments; - try { - injectedArguments = getInjectedArguments(testFn, test); - } catch (e) { - catchError(e); return; - } - - testFn.call(test, injectedArguments).then(() => { - recorder.add('fire test.passed', () => { - event.emit(event.test.passed, test); - event.emit(event.test.finished, test); - }); - recorder.add('finish test', () => done()); - recorder.catch(); - }).catch(catchError); - return; } try { @@ -93,11 +80,11 @@ module.exports.test = (test) => { } catch (err) { recorder.throw(err); } finally { - recorder.add('fire test.passed', () => { + recorder.add("fire test.passed", () => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add('finish test', () => done()); + recorder.add("finish test", () => done()); recorder.catch(); } }; @@ -110,11 +97,11 @@ module.exports.test = (test) => { module.exports.injected = function (fn, suite, hookName) { return function (done) { const errHandler = (err) => { - recorder.session.start('teardown'); + recorder.session.start("teardown"); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); - if (hookName === 'after') event.emit(event.test.after, suite); - if (hookName === 'afterSuite') event.emit(event.suite.after, suite); + if (hookName === "after") event.emit(event.test.after, suite); + if (hookName === "afterSuite") event.emit(event.suite.after, suite); recorder.add(() => done(err)); }; @@ -122,7 +109,7 @@ module.exports.injected = function (fn, suite, hookName) { errHandler(err); }); - if (!fn) throw new Error('fn is not defined'); + if (!fn) throw new Error("fn is not defined"); event.emit(event.hook.started, suite); @@ -137,31 +124,40 @@ module.exports.injected = function (fn, suite, hookName) { const opts = suite.opts || {}; const retries = opts[`retry${ucfirst(hookName)}`] || 0; - promiseRetry(async (retry, number) => { - try { - recorder.startUnlessRunning(); - await fn.call(this, getInjectedArguments(fn)); - await recorder.promise().catch(err => retry(err)); - } catch (err) { - retry(err); - } finally { - if (number < retries) { - recorder.stop(); - recorder.start(); + promiseRetry( + async (retry, number) => { + try { + recorder.startUnlessRunning(); + await fn.call(this, getInjectedArguments(fn)); + await recorder.promise().catch((err) => retry(err)); + } catch (err) { + retry(err); + } finally { + if (number < retries) { + recorder.stop(); + recorder.start(); + } } - } - }, { retries }) + }, + { retries } + ) .then(() => { - recorder.add('fire hook.passed', () => event.emit(event.hook.passed, suite)); + recorder.add("fire hook.passed", () => + event.emit(event.hook.passed, suite) + ); recorder.add(`finish ${hookName} hook`, () => done()); recorder.catch(); - }).catch((e) => { + }) + .catch((e) => { recorder.throw(e); recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); + const err = + recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); - recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); + recorder.add("fire hook.failed", () => + event.emit(event.hook.failed, suite, e) + ); }); }; }; @@ -198,7 +194,7 @@ module.exports.suiteTeardown = function (suite) { }; const getInjectedArguments = (fn, test) => { - const container = require('./container'); + const container = require("./container"); const testArgs = {}; const params = parser.getParams(fn) || []; const objects = container.support(); diff --git a/package.json b/package.json index 29ab53926..abe576da3 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "docs": "./runok.js docs", "test:unit": "mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", - "test:stale": "mocha test/runner/stale_test.js", + "test:stale": "mocha test/runner/stale_test.js --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", "test:appium-other": "mocha test/helper/AppiumV2_test.js --grep 'second'", diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index b568e8607..8fcbd77b5 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -14,3 +14,13 @@ Scenario('retryTo works with non await steps @plugin', async () => { if (tryNum < 3) I.waitForVisible('.nothing', 1); }, 4); }); + + + +Scenario('Should be succeed @plugin', async ({ I }) => { + I.amOnPage('http://example.org') + I.waitForVisible('.nothing', 1); // should fail here but it won't terminate + await retryTo( (tryNum) => { + I.see(".doesNotMatter"); + }, 10); +}) \ No newline at end of file diff --git a/test/data/sandbox/codecept.stale.js b/test/data/sandbox/codecept.stale.js index f9a26ec2c..00e9c454f 100644 --- a/test/data/sandbox/codecept.stale.js +++ b/test/data/sandbox/codecept.stale.js @@ -1,6 +1,6 @@ exports.config = { tests: './*_test.stale.js', - // timeout: 10000, + timeout: 10000, retry: { Scenario: 2, After: 0, @@ -8,10 +8,15 @@ exports.config = { }, output: './output', helpers: { + Playwright: { + url: 'http://localhost', + show: false, + browser: 'chromium', + }, }, include: {}, bootstrap: false, mocha: {}, - name: 'sandbox', + name: 'sandbox' }; \ No newline at end of file diff --git a/test/data/sandbox/stale_test.stale.js b/test/data/sandbox/stale_test.stale.js index 2ffef0d62..420e5016b 100644 --- a/test/data/sandbox/stale_test.stale.js +++ b/test/data/sandbox/stale_test.stale.js @@ -8,6 +8,12 @@ Feature("Stale test"); let tries = 0; + +Scenario('Rejected promise stale the process', async ({ I }) => { + // I.say('hey') + await new Promise((resolve, reject) => setTimeout(reject("@@@Flaky error"), 500)); +}) + Scenario("Try to stale flaky test", async () => { tries++; throw new Error("@@@Flaky error"); @@ -15,7 +21,7 @@ Scenario("Try to stale flaky test", async () => { // }, 500)); }) -Scenario.skip("2 - Try to stale flaky test", async () => { +Scenario("2 - Try to stale flaky test", async () => { tries++; setTimeout(() => { throw new Error("@@@Flaky error"); @@ -24,12 +30,9 @@ Scenario.skip("2 - Try to stale flaky test", async () => { throw new Error("@@@Flaky error"); }).retry(2); -After(() => { - console.log(`[T] Retries: ${tries}`); -}); -// Works but we don't know why on retry(1) -Scenario.skip("Try to stale flaky test", async () => { +// // Works but we don't know why on retry(1) +Scenario("3 - Try to stale flaky test", async () => { tries++; setTimeout(() => { throw new Error("@@@Flaky error"); @@ -37,11 +40,16 @@ Scenario.skip("Try to stale flaky test", async () => { await new Promise((resolve) => setTimeout(resolve, 701)); }).retry(1); -// Lead to error "done() multiple time" because error is throw at the same time that test is over -Scenario.skip("Try to stale flaky test", async () => { +// // Lead to error "done() multiple time" because error is throw at the same time that test is over +Scenario("4 - Try to stale flaky test", async () => { tries++; setTimeout(() => { throw new Error("@@@Flaky error"); }, 500); await new Promise((resolve) => setTimeout(resolve, 500)); -}); \ No newline at end of file +}); + + +After(() => { + console.log(`[T] Retries: ${tries}`); +}); diff --git a/test/runner/stale_test.js b/test/runner/stale_test.js index b33e0105e..0f842386e 100644 --- a/test/runner/stale_test.js +++ b/test/runner/stale_test.js @@ -14,8 +14,6 @@ describe('CodeceptJS Interface', () => { it('Should always failed and terminate', (done) => { exec(config_run_config('codecept.stale.js'), (err, stdout) => { - console.log('err', err); - console.log('stdout',stdout) expect(stdout).toContain('@@@Flaky error'); // feature expect(err).toBeTruthy(); done(); From cfc0be589201909fff0cf36b390a46d258c46515 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 16:27:18 +0200 Subject: [PATCH 03/10] fix: done() call multiple times --- lib/scenario.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/scenario.js b/lib/scenario.js index bc85409f2..61edca663 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -18,6 +18,16 @@ const injectHook = function (inject, suite) { return recorder.promise(); }; +function makeDoneCallableOnce(done) { + let called = false; + return function (err) { + if (called) { + return; + } + called = true; + return done(err); + }; +} /** * Wraps test function, injects support objects from container, * starts promise chain with recorder, performs before/after hooks @@ -34,6 +44,7 @@ module.exports.test = (test) => { test.async = true; test.fn = function (done) { + const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { recorder.session.start("teardown"); recorder.cleanAsyncErr(); @@ -43,7 +54,7 @@ module.exports.test = (test) => { assertThrown(err, test.throws); event.emit(event.test.passed, test); event.emit(event.test.finished, test); - recorder.add(() => done()); + recorder.add(doneFn); return; } catch (newErr) { err = newErr; @@ -51,7 +62,7 @@ module.exports.test = (test) => { } event.emit(event.test.failed, test, err); event.emit(event.test.finished, test); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }); if (isAsyncFunction(testFn)) { @@ -63,7 +74,7 @@ module.exports.test = (test) => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add("finish test", () => done()); + recorder.add("finish test", doneFn); }) .catch((err) => { recorder.throw(err); @@ -84,7 +95,7 @@ module.exports.test = (test) => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add("finish test", () => done()); + recorder.add("finish test", doneFn); recorder.catch(); } }; @@ -96,13 +107,14 @@ module.exports.test = (test) => { */ module.exports.injected = function (fn, suite, hookName) { return function (done) { + const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { recorder.session.start("teardown"); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); if (hookName === "after") event.emit(event.test.after, suite); if (hookName === "afterSuite") event.emit(event.suite.after, suite); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }; recorder.errHandler((err) => { @@ -145,7 +157,7 @@ module.exports.injected = function (fn, suite, hookName) { recorder.add("fire hook.passed", () => event.emit(event.hook.passed, suite) ); - recorder.add(`finish ${hookName} hook`, () => done()); + recorder.add(`finish ${hookName} hook`, doneFn); recorder.catch(); }) .catch((e) => { From de24891783b8f96f2f3c3a664583078509cd8477 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 16:45:10 +0200 Subject: [PATCH 04/10] chore: rename files and test description --- package.json | 1 - ...pt.stale.js => codecept.scenario-stale.js} | 2 +- test/data/sandbox/stale_test.stale.js | 55 ------------------- test/data/sandbox/test.scenario-stale.js | 24 ++++++++ .../{stale_test.js => scenario_stale_test.js} | 8 +-- test/runner/timeout_test.js | 2 +- 6 files changed, 30 insertions(+), 62 deletions(-) rename test/data/sandbox/{codecept.stale.js => codecept.scenario-stale.js} (89%) delete mode 100644 test/data/sandbox/stale_test.stale.js create mode 100644 test/data/sandbox/test.scenario-stale.js rename test/runner/{stale_test.js => scenario_stale_test.js} (65%) diff --git a/package.json b/package.json index abe576da3..2c56b255b 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "docs": "./runok.js docs", "test:unit": "mocha test/unit --recursive --timeout 10000", "test:runner": "mocha test/runner --recursive --timeout 10000", - "test:stale": "mocha test/runner/stale_test.js --timeout 10000", "test": "npm run test:unit && npm run test:runner", "test:appium-quick": "mocha test/helper/AppiumV2_test.js --grep 'quick'", "test:appium-other": "mocha test/helper/AppiumV2_test.js --grep 'second'", diff --git a/test/data/sandbox/codecept.stale.js b/test/data/sandbox/codecept.scenario-stale.js similarity index 89% rename from test/data/sandbox/codecept.stale.js rename to test/data/sandbox/codecept.scenario-stale.js index 00e9c454f..c40f98a79 100644 --- a/test/data/sandbox/codecept.stale.js +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -1,5 +1,5 @@ exports.config = { - tests: './*_test.stale.js', + tests: './test.scenario-stale.js', timeout: 10000, retry: { Scenario: 2, diff --git a/test/data/sandbox/stale_test.stale.js b/test/data/sandbox/stale_test.stale.js deleted file mode 100644 index 420e5016b..000000000 --- a/test/data/sandbox/stale_test.stale.js +++ /dev/null @@ -1,55 +0,0 @@ -const { I } = inject(); - -Feature("Stale test"); - -/** - * Retry on config is not the same at retry on test - */ - -let tries = 0; - - -Scenario('Rejected promise stale the process', async ({ I }) => { - // I.say('hey') - await new Promise((resolve, reject) => setTimeout(reject("@@@Flaky error"), 500)); -}) - -Scenario("Try to stale flaky test", async () => { - tries++; - throw new Error("@@@Flaky error"); - // await new Promise((resolve) => setTimeout(() => { - // }, 500)); -}) - -Scenario("2 - Try to stale flaky test", async () => { - tries++; - setTimeout(() => { - throw new Error("@@@Flaky error"); - }, 500); - await new Promise((resolve) => setTimeout(resolve, 300)); - throw new Error("@@@Flaky error"); -}).retry(2); - - -// // Works but we don't know why on retry(1) -Scenario("3 - Try to stale flaky test", async () => { - tries++; - setTimeout(() => { - throw new Error("@@@Flaky error"); - }, 500); - await new Promise((resolve) => setTimeout(resolve, 701)); -}).retry(1); - -// // Lead to error "done() multiple time" because error is throw at the same time that test is over -Scenario("4 - Try to stale flaky test", async () => { - tries++; - setTimeout(() => { - throw new Error("@@@Flaky error"); - }, 500); - await new Promise((resolve) => setTimeout(resolve, 500)); -}); - - -After(() => { - console.log(`[T] Retries: ${tries}`); -}); diff --git a/test/data/sandbox/test.scenario-stale.js b/test/data/sandbox/test.scenario-stale.js new file mode 100644 index 000000000..c5419a9ee --- /dev/null +++ b/test/data/sandbox/test.scenario-stale.js @@ -0,0 +1,24 @@ +const { I } = inject(); + +Feature("Scenario should not be staling"); + +const SHOULD_NOT_STALE = "should not stale scenario error"; + +Scenario('Rejected promise should not stale the process', async () => { + await new Promise((_resolve, reject) => setTimeout(reject(new Error(SHOULD_NOT_STALE)), 500)); +}) + +Scenario("Should handle throw inside synchronous and terminate gracefully", () => { + throw new Error(SHOULD_NOT_STALE); +}) +Scenario("Should handle throw inside async and terminate gracefully", async () => { + throw new Error(SHOULD_NOT_STALE); +}) + +Scenario("Should throw, retry and keep failing", async () => { + setTimeout(() => { + throw new Error(SHOULD_NOT_STALE); + }, 500); + await new Promise((resolve) => setTimeout(resolve, 300)); + throw new Error(SHOULD_NOT_STALE); +}).retry(2); diff --git a/test/runner/stale_test.js b/test/runner/scenario_stale_test.js similarity index 65% rename from test/runner/stale_test.js rename to test/runner/scenario_stale_test.js index 0f842386e..75b9690a9 100644 --- a/test/runner/stale_test.js +++ b/test/runner/scenario_stale_test.js @@ -7,14 +7,14 @@ const codecept_dir = path.join(__dirname, '/../data/sandbox'); const codecept_run = `${runner} run`; const config_run_config = config => `${codecept_run} --config ${codecept_dir}/${config}`; -describe('CodeceptJS Interface', () => { +describe('Scenario termination check', () => { before(() => { process.chdir(codecept_dir); }); - it('Should always failed and terminate', (done) => { - exec(config_run_config('codecept.stale.js'), (err, stdout) => { - expect(stdout).toContain('@@@Flaky error'); // feature + it('Should always fail and terminate', (done) => { + exec(config_run_config('codecept.scenario-stale.js'), (err, stdout) => { + expect(stdout).toContain('should not stale scenario error'); // feature expect(err).toBeTruthy(); done(); }); diff --git a/test/runner/timeout_test.js b/test/runner/timeout_test.js index 79d74f054..8e14efc09 100644 --- a/test/runner/timeout_test.js +++ b/test/runner/timeout_test.js @@ -11,7 +11,7 @@ describe('CodeceptJS Timeouts', function () { it('should stop test when timeout exceeded', (done) => { exec(config_run_config('codecept.conf.js', 'timed out'), (err, stdout) => { - console.log(stdout); + debug_this_test && console.log(stdout); expect(stdout).toContain('Timeout 2s exceeded'); expect(stdout).toContain('Timeout 1s exceeded'); expect(err).toBeTruthy(); From 5eb28d879541858057ac864e83850cac76443c97 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 17:20:07 +0200 Subject: [PATCH 05/10] fix: playwright error on test --- test/data/sandbox/codecept.scenario-stale.js | 13 +------------ test/data/sandbox/test.scenario-stale.js | 2 -- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/test/data/sandbox/codecept.scenario-stale.js b/test/data/sandbox/codecept.scenario-stale.js index c40f98a79..b579130d4 100644 --- a/test/data/sandbox/codecept.scenario-stale.js +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -1,19 +1,8 @@ exports.config = { tests: './test.scenario-stale.js', timeout: 10000, - retry: { - Scenario: 2, - After: 0, - Before: 1, - }, + retry: 2, output: './output', - helpers: { - Playwright: { - url: 'http://localhost', - show: false, - browser: 'chromium', - }, - }, include: {}, bootstrap: false, mocha: {}, diff --git a/test/data/sandbox/test.scenario-stale.js b/test/data/sandbox/test.scenario-stale.js index c5419a9ee..418806e11 100644 --- a/test/data/sandbox/test.scenario-stale.js +++ b/test/data/sandbox/test.scenario-stale.js @@ -1,5 +1,3 @@ -const { I } = inject(); - Feature("Scenario should not be staling"); const SHOULD_NOT_STALE = "should not stale scenario error"; From 5212c61ff524e264d9349852bda708f45974d21e Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Thu, 6 Jun 2024 00:24:02 +0200 Subject: [PATCH 06/10] fix: stale process due to retryTo error handler --- lib/plugin/retryTo.js | 3 +++ test/acceptance/retryTo_test.js | 2 +- test/plugin/plugin_test.js | 9 +++++++++ test/runner/run_workers_test.js | 1 - 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/plugin/retryTo.js b/lib/plugin/retryTo.js index 9929944f1..a3c7a934f 100644 --- a/lib/plugin/retryTo.js +++ b/lib/plugin/retryTo.js @@ -113,6 +113,9 @@ module.exports = function (config) { recorder.add('retryTo', async () => { tryBlock(); + }).catch(err => { + console.error('An error occurred:', err); + done(null); }); }).then(() => { if (err) recorder.throw(err); diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index 8fcbd77b5..610022cf6 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -17,7 +17,7 @@ Scenario('retryTo works with non await steps @plugin', async () => { -Scenario('Should be succeed @plugin', async ({ I }) => { +Scenario('Should be succeed', async ({ I }) => { I.amOnPage('http://example.org') I.waitForVisible('.nothing', 1); // should fail here but it won't terminate await retryTo( (tryNum) => { diff --git a/test/plugin/plugin_test.js b/test/plugin/plugin_test.js index 2f615d1d5..fea440177 100644 --- a/test/plugin/plugin_test.js +++ b/test/plugin/plugin_test.js @@ -32,6 +32,15 @@ describe('CodeceptJS plugin', function () { }); }); + it('should failed before the retryTo instruction', (done) => { + exec(`${config_run_config('codecept.Playwright.retryTo.js', 'Should be succeed')} --verbose`, (err, stdout) => { + expect(stdout).toContain('locator.waitFor: Timeout 1000ms exceeded.'), + expect(stdout).toContain('[1] Error | Error: element (.nothing) still not visible after 1 sec'), + expect(err).toBeTruthy(); + done(); + }); + }); + it('should generate the coverage report', (done) => { exec(`${config_run_config('codecept.Playwright.coverage.js', '@coverage')} --debug`, (err, stdout) => { const lines = stdout.split('\n'); diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 0cd0f3b46..ccd1afe99 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -19,7 +19,6 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version'); console.log(`${codecept_run} 3 --debug`); exec(`${codecept_run} 3 --debug`, (err, stdout) => { - console.log('aaaaaaaaaaaaa', stdout); expect(stdout).toContain('CodeceptJS'); // feature expect(stdout).toContain('glob current dir'); expect(stdout).toContain('From worker @1_grep print message 1'); From 89275b2c847f72a0588d87d082fb0fec5644d7dc Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Thu, 6 Jun 2024 12:21:52 +0200 Subject: [PATCH 07/10] fix: lint previous code --- lib/scenario.js | 47 +++++++++----------- test/acceptance/retryTo_test.js | 10 ++--- test/data/sandbox/codecept.scenario-stale.js | 19 ++++---- test/data/sandbox/test.scenario-stale.js | 16 +++---- test/runner/scenario_stale_test.js | 3 +- 5 files changed, 43 insertions(+), 52 deletions(-) diff --git a/lib/scenario.js b/lib/scenario.js index 61edca663..ab816a1ab 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -1,9 +1,9 @@ -const promiseRetry = require("promise-retry"); -const event = require("./event"); -const recorder = require("./recorder"); -const assertThrown = require("./assert/throws"); -const { ucfirst, isAsyncFunction } = require("./utils"); -const parser = require("./parser"); +const promiseRetry = require('promise-retry'); +const event = require('./event'); +const recorder = require('./recorder'); +const assertThrown = require('./assert/throws'); +const { ucfirst, isAsyncFunction } = require('./utils'); +const parser = require('./parser'); const injectHook = function (inject, suite) { try { @@ -46,7 +46,7 @@ module.exports.test = (test) => { test.fn = function (done) { const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { - recorder.session.start("teardown"); + recorder.session.start('teardown'); recorder.cleanAsyncErr(); if (test.throws) { // check that test should actually fail @@ -70,11 +70,11 @@ module.exports.test = (test) => { testFn .call(test, getInjectedArguments(testFn, test)) .then(() => { - recorder.add("fire test.passed", () => { + recorder.add('fire test.passed', () => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add("finish test", doneFn); + recorder.add('finish test', doneFn); }) .catch((err) => { recorder.throw(err); @@ -82,7 +82,7 @@ module.exports.test = (test) => { .finally(() => { recorder.catch(); }); - return; + return; } try { @@ -91,11 +91,11 @@ module.exports.test = (test) => { } catch (err) { recorder.throw(err); } finally { - recorder.add("fire test.passed", () => { + recorder.add('fire test.passed', () => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add("finish test", doneFn); + recorder.add('finish test', doneFn); recorder.catch(); } }; @@ -109,11 +109,11 @@ module.exports.injected = function (fn, suite, hookName) { return function (done) { const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { - recorder.session.start("teardown"); + recorder.session.start('teardown'); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); - if (hookName === "after") event.emit(event.test.after, suite); - if (hookName === "afterSuite") event.emit(event.suite.after, suite); + if (hookName === 'after') event.emit(event.test.after, suite); + if (hookName === 'afterSuite') event.emit(event.suite.after, suite); recorder.add(() => doneFn(err)); }; @@ -121,7 +121,7 @@ module.exports.injected = function (fn, suite, hookName) { errHandler(err); }); - if (!fn) throw new Error("fn is not defined"); + if (!fn) throw new Error('fn is not defined'); event.emit(event.hook.started, suite); @@ -151,25 +151,20 @@ module.exports.injected = function (fn, suite, hookName) { } } }, - { retries } + { retries }, ) .then(() => { - recorder.add("fire hook.passed", () => - event.emit(event.hook.passed, suite) - ); + recorder.add('fire hook.passed', () => event.emit(event.hook.passed, suite)); recorder.add(`finish ${hookName} hook`, doneFn); recorder.catch(); }) .catch((e) => { recorder.throw(e); recorder.catch((e) => { - const err = - recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); + const err = recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); - recorder.add("fire hook.failed", () => - event.emit(event.hook.failed, suite, e) - ); + recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); }); }; }; @@ -206,7 +201,7 @@ module.exports.suiteTeardown = function (suite) { }; const getInjectedArguments = (fn, test) => { - const container = require("./container"); + const container = require('./container'); const testArgs = {}; const params = parser.getParams(fn) || []; const objects = container.support(); diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index 610022cf6..1e6836f55 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -15,12 +15,10 @@ Scenario('retryTo works with non await steps @plugin', async () => { }, 4); }); - - Scenario('Should be succeed', async ({ I }) => { - I.amOnPage('http://example.org') + I.amOnPage('http://example.org'); I.waitForVisible('.nothing', 1); // should fail here but it won't terminate - await retryTo( (tryNum) => { - I.see(".doesNotMatter"); + await retryTo((tryNum) => { + I.see('.doesNotMatter'); }, 10); -}) \ No newline at end of file +}); diff --git a/test/data/sandbox/codecept.scenario-stale.js b/test/data/sandbox/codecept.scenario-stale.js index b579130d4..f07399b38 100644 --- a/test/data/sandbox/codecept.scenario-stale.js +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -1,11 +1,10 @@ exports.config = { - tests: './test.scenario-stale.js', - timeout: 10000, - retry: 2, - output: './output', - include: {}, - bootstrap: false, - mocha: {}, - name: 'sandbox' - }; - \ No newline at end of file + tests: './test.scenario-stale.js', + timeout: 10000, + retry: 2, + output: './output', + include: {}, + bootstrap: false, + mocha: {}, + name: 'sandbox', +}; diff --git a/test/data/sandbox/test.scenario-stale.js b/test/data/sandbox/test.scenario-stale.js index 418806e11..30ec3c9b0 100644 --- a/test/data/sandbox/test.scenario-stale.js +++ b/test/data/sandbox/test.scenario-stale.js @@ -1,19 +1,19 @@ -Feature("Scenario should not be staling"); +Feature('Scenario should not be staling'); -const SHOULD_NOT_STALE = "should not stale scenario error"; +const SHOULD_NOT_STALE = 'should not stale scenario error'; Scenario('Rejected promise should not stale the process', async () => { await new Promise((_resolve, reject) => setTimeout(reject(new Error(SHOULD_NOT_STALE)), 500)); -}) +}); -Scenario("Should handle throw inside synchronous and terminate gracefully", () => { +Scenario('Should handle throw inside synchronous and terminate gracefully', () => { throw new Error(SHOULD_NOT_STALE); -}) -Scenario("Should handle throw inside async and terminate gracefully", async () => { +}); +Scenario('Should handle throw inside async and terminate gracefully', async () => { throw new Error(SHOULD_NOT_STALE); -}) +}); -Scenario("Should throw, retry and keep failing", async () => { +Scenario('Should throw, retry and keep failing', async () => { setTimeout(() => { throw new Error(SHOULD_NOT_STALE); }, 500); diff --git a/test/runner/scenario_stale_test.js b/test/runner/scenario_stale_test.js index 75b9690a9..927d92acd 100644 --- a/test/runner/scenario_stale_test.js +++ b/test/runner/scenario_stale_test.js @@ -1,6 +1,6 @@ const { expect } = require('expect'); const path = require('path'); -const exec = require('child_process').exec; +const { exec } = require('child_process'); const runner = path.join(__dirname, '/../../bin/codecept.js'); const codecept_dir = path.join(__dirname, '/../data/sandbox'); @@ -19,5 +19,4 @@ describe('Scenario termination check', () => { done(); }); }); - }); From 8dd36d3e058eaba7c23317074d57c85426eb66d7 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Fri, 7 Jun 2024 10:15:45 +0200 Subject: [PATCH 08/10] fix: error due to the previous merge --- lib/plugin/retryTo.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/plugin/retryTo.js b/lib/plugin/retryTo.js index 7882f4674..4a2940176 100644 --- a/lib/plugin/retryTo.js +++ b/lib/plugin/retryTo.js @@ -114,14 +114,10 @@ module.exports = function (config) { }); }; - recorder.add('retryTo', async () => { - tryBlock(); - }).catch(err => { + recorder.add('retryTo', tryBlock).catch(err => { console.error('An error occurred:', err); done(null); }); - }).then(() => { - if (err) recorder.throw(err); }); } From 2a7d68e2bf3ea1c78d8c1b2fcef0b1cb9ea8e292 Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Wed, 5 Jun 2024 15:56:12 +0200 Subject: [PATCH 09/10] fix: handle async function better to prevent stale the process --- lib/scenario.js | 26 ++++++++++++++------------ test/acceptance/retryTo_test.js | 9 +++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/scenario.js b/lib/scenario.js index ab816a1ab..4b4baa574 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -1,9 +1,9 @@ -const promiseRetry = require('promise-retry'); -const event = require('./event'); -const recorder = require('./recorder'); -const assertThrown = require('./assert/throws'); -const { ucfirst, isAsyncFunction } = require('./utils'); -const parser = require('./parser'); +const promiseRetry = require("promise-retry"); +const event = require("./event"); +const recorder = require("./recorder"); +const assertThrown = require("./assert/throws"); +const { ucfirst, isAsyncFunction } = require("./utils"); +const parser = require("./parser"); const injectHook = function (inject, suite) { try { @@ -46,7 +46,7 @@ module.exports.test = (test) => { test.fn = function (done) { const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { - recorder.session.start('teardown'); + recorder.session.start("teardown"); recorder.cleanAsyncErr(); if (test.throws) { // check that test should actually fail @@ -91,7 +91,7 @@ module.exports.test = (test) => { } catch (err) { recorder.throw(err); } finally { - recorder.add('fire test.passed', () => { + recorder.add("fire test.passed", () => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); @@ -109,7 +109,7 @@ module.exports.injected = function (fn, suite, hookName) { return function (done) { const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { - recorder.session.start('teardown'); + recorder.session.start("teardown"); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); if (hookName === 'after') event.emit(event.test.after, suite); @@ -121,7 +121,7 @@ module.exports.injected = function (fn, suite, hookName) { errHandler(err); }); - if (!fn) throw new Error('fn is not defined'); + if (!fn) throw new Error("fn is not defined"); event.emit(event.hook.started, suite); @@ -164,7 +164,9 @@ module.exports.injected = function (fn, suite, hookName) { const err = recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); - recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); + recorder.add("fire hook.failed", () => + event.emit(event.hook.failed, suite, e) + ); }); }; }; @@ -201,7 +203,7 @@ module.exports.suiteTeardown = function (suite) { }; const getInjectedArguments = (fn, test) => { - const container = require('./container'); + const container = require("./container"); const testArgs = {}; const params = parser.getParams(fn) || []; const objects = container.support(); diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index 6253fdb80..016cfd671 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -34,3 +34,12 @@ Scenario('Should succeed at the third attempt @plugin', async () => { if (tryNum < 2) throw new Error('Custom pluginRetryTo Error'); }, 3); }); + + +Scenario('Should be succeed', async ({ I }) => { + I.amOnPage('http://example.org') + I.waitForVisible('.nothing', 1); // should fail here but it won't terminate + await retryTo( (tryNum) => { + I.see(".doesNotMatter"); + }, 10); +}) From 09db07efc1fe2e81a6717e023d1bed944e6ad92d Mon Sep 17 00:00:00 2001 From: Cyril Hue Date: Thu, 6 Jun 2024 12:21:52 +0200 Subject: [PATCH 10/10] fix: lint previous code --- lib/scenario.js | 26 ++++++++++++-------------- test/acceptance/retryTo_test.js | 6 +++--- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/scenario.js b/lib/scenario.js index 4b4baa574..ab816a1ab 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -1,9 +1,9 @@ -const promiseRetry = require("promise-retry"); -const event = require("./event"); -const recorder = require("./recorder"); -const assertThrown = require("./assert/throws"); -const { ucfirst, isAsyncFunction } = require("./utils"); -const parser = require("./parser"); +const promiseRetry = require('promise-retry'); +const event = require('./event'); +const recorder = require('./recorder'); +const assertThrown = require('./assert/throws'); +const { ucfirst, isAsyncFunction } = require('./utils'); +const parser = require('./parser'); const injectHook = function (inject, suite) { try { @@ -46,7 +46,7 @@ module.exports.test = (test) => { test.fn = function (done) { const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { - recorder.session.start("teardown"); + recorder.session.start('teardown'); recorder.cleanAsyncErr(); if (test.throws) { // check that test should actually fail @@ -91,7 +91,7 @@ module.exports.test = (test) => { } catch (err) { recorder.throw(err); } finally { - recorder.add("fire test.passed", () => { + recorder.add('fire test.passed', () => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); @@ -109,7 +109,7 @@ module.exports.injected = function (fn, suite, hookName) { return function (done) { const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { - recorder.session.start("teardown"); + recorder.session.start('teardown'); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); if (hookName === 'after') event.emit(event.test.after, suite); @@ -121,7 +121,7 @@ module.exports.injected = function (fn, suite, hookName) { errHandler(err); }); - if (!fn) throw new Error("fn is not defined"); + if (!fn) throw new Error('fn is not defined'); event.emit(event.hook.started, suite); @@ -164,9 +164,7 @@ module.exports.injected = function (fn, suite, hookName) { const err = recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); - recorder.add("fire hook.failed", () => - event.emit(event.hook.failed, suite, e) - ); + recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); }); }; }; @@ -203,7 +201,7 @@ module.exports.suiteTeardown = function (suite) { }; const getInjectedArguments = (fn, test) => { - const container = require("./container"); + const container = require('./container'); const testArgs = {}; const params = parser.getParams(fn) || []; const objects = container.support(); diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index 016cfd671..af8864175 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -37,9 +37,9 @@ Scenario('Should succeed at the third attempt @plugin', async () => { Scenario('Should be succeed', async ({ I }) => { - I.amOnPage('http://example.org') + I.amOnPage('http://example.org'); I.waitForVisible('.nothing', 1); // should fail here but it won't terminate - await retryTo( (tryNum) => { - I.see(".doesNotMatter"); + await retryTo((tryNum) => { + I.see('.doesNotMatter'); }, 10); })