diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index afe1d485fe9..8d29a25bee4 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,12 +34,9 @@ blocks: - npm -v - node -v - npm run-script test_ci_chrome_consumer - - name: consumer mock - flaky test group 1 + - name: consumer mock - flaky test group commands: - - npm run-script test_ci_chrome_consumer_flaky_1 - - name: consumer mock - flaky test group 2 - commands: - - npm run-script test_ci_chrome_consumer_flaky_2 + - npm run-script test_ci_chrome_consumer_flaky - name: content script tests commands: - npm run-script test_ci_chrome_content_scripts diff --git a/package.json b/package.json index 5f90759ed6f..184507104b8 100644 --- a/package.json +++ b/package.json @@ -87,10 +87,8 @@ "test_ci_chrome_consumer_live_gmail": "npx ava --timeout=45m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-LIVE-GMAIL STANDARD-GROUP | npx tap-xunit > report.xml", "test_ci_chrome_consumer": "npx ava --timeout=30m --verbose --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK STANDARD-GROUP", "test_ci_chrome_enterprise": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- ENTERPRISE-MOCK STANDARD-GROUP | npx tap-xunit > report.xml", - "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml", - "test_ci_chrome_consumer_flaky_1": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-1 | npx tap-xunit > report.xml", - "test_ci_chrome_consumer_flaky_2": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-2 | npx tap-xunit > report.xml", - "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS > report.xml", + "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP --pool-size=3 | npx tap-xunit > report.xml", + "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS --pool-size=1 > report.xml", "dev_start_gmail_mock_api": "./scripts/build.sh && cd ./conf && node ../build/tooling/tsc-compiler --project tsconfig.test.json && cd .. && node ./build/test/test/source/mock.js", "run_firefox": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile ~/.mozilla/firefox/flowcrypt-dev --keep-profile-changes", "run_firefox_windows": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile %userprofile%/AppData/Local/Mozilla/Firefox/Profiles/flowcrypt-dev --keep-profile-changes", diff --git a/test/source/test.ts b/test/source/test.ts index 5b55fcd5bad..a389ed7356d 100644 --- a/test/source/test.ts +++ b/test/source/test.ts @@ -24,7 +24,7 @@ import { defineSetupTests } from './tests/setup'; import { defineUnitBrowserTests } from './tests/unit-browser'; import { defineUnitNodeTests } from './tests/unit-node'; -export const { testVariant, testGroup, oneIfNotPooled, buildDir, isMock } = getParsedCliParams(); +export const { testVariant, testGroup, oneIfNotPooled, poolSize, buildDir, isMock } = getParsedCliParams(); export const internalTestState = { expectIntentionalErrReport: false }; // updated when a particular test that causes an error is run const DEBUG_BROWSER_LOG = false; // set to true to print / export information from browser const DEBUG_MOCK_LOG = false; // set to true to print mock server logs @@ -38,7 +38,7 @@ const consts = { TIMEOUT_EACH_RETRY: minutes(4), TIMEOUT_ALL_RETRIES: minutes(55), ATTEMPTS: testGroup === 'STANDARD-GROUP' ? oneIfNotPooled(3) : process.argv.includes('--retry=false') ? 1 : 3, - POOL_SIZE: oneIfNotPooled(isMock ? 20 : 3), + POOL_SIZE: poolSize ?? (isMock ? 20 : 3), IS_LOCAL_DEBUG: process.argv.includes('--debug') ? true : false, // run locally by developer, not in ci }; @@ -236,10 +236,6 @@ if (testGroup === 'UNIT-TESTS') { defineUnitBrowserTests(testVariant, testWithBrowser); } else if (testGroup === 'FLAKY-GROUP') { defineFlakyTests(testVariant, testWithBrowser); -} else if (testGroup === 'FLAKY-GROUP-1') { - defineFlakyTests(testVariant, testWithBrowser, 0, 2); -} else if (testGroup === 'FLAKY-GROUP-2') { - defineFlakyTests(testVariant, testWithBrowser, 1, 2); } else if (testGroup === 'CONTENT-SCRIPT-TESTS') { defineContentScriptTests(testWithBrowser); } else { diff --git a/test/source/tests/flaky.ts b/test/source/tests/flaky.ts index e53ce9fa6f7..2207475db1a 100644 --- a/test/source/tests/flaky.ts +++ b/test/source/tests/flaky.ts @@ -29,12 +29,9 @@ import { minutes } from './tooling'; // these tests are run serially, one after another, because they are somewhat more sensitive to parallel testing // eg if they are very cpu-sensitive (create key tests) -export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser, shardIndex = 0, totalShards = 1) => { - let testCounter = 0; +export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => { const test = (title: string, impl: Implementation) => { - if (testCounter++ % totalShards === shardIndex) { - avaTest(title, impl); - } + avaTest(title, impl); }; test.skip = avaTest.skip; diff --git a/test/source/util/index.ts b/test/source/util/index.ts index 1ce7ec50e9c..7164508450c 100644 --- a/test/source/util/index.ts +++ b/test/source/util/index.ts @@ -14,7 +14,7 @@ const ROOT_DIR = process.cwd(); export const getParsedCliParams = () => { let testVariant: TestVariant; - let testGroup: 'FLAKY-GROUP' | 'FLAKY-GROUP-1' | 'FLAKY-GROUP-2' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined; + let testGroup: 'FLAKY-GROUP' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined; if (process.argv.includes('CONTENT-SCRIPT-TESTS')) { testVariant = 'CONSUMER-CONTENT-SCRIPT-TESTS-MOCK'; testGroup = 'CONTENT-SCRIPT-TESTS'; @@ -28,13 +28,15 @@ export const getParsedCliParams = () => { throw new Error('Unknown test type: specify CONSUMER-MOCK or ENTERPRISE-MOCK CONSUMER-LIVE-GMAIL'); } if (!testGroup) { - testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : process.argv.includes('FLAKY-GROUP-1') ? 'FLAKY-GROUP-1' : process.argv.includes('FLAKY-GROUP-2') ? 'FLAKY-GROUP-2' : 'STANDARD-GROUP'; + testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : 'STANDARD-GROUP'; } const buildDir = join(ROOT_DIR, `build/chrome-${(testVariant === 'CONSUMER-LIVE-GMAIL' ? 'CONSUMER' : testVariant).toLowerCase()}`); - const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'FLAKY-GROUP-1', 'FLAKY-GROUP-2', 'CONTENT-SCRIPT-TESTS'].includes(testGroup); - const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? Math.min(1, suggestedPoolSize) : suggestedPoolSize); - console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSizeOne: ${poolSizeOne})`); - return { testVariant, testGroup, oneIfNotPooled, buildDir, isMock: testVariant.includes('-MOCK') }; + const poolSizeArg = process.argv.find(a => a.startsWith('--pool-size=')); + const poolSize = poolSizeArg ? parseInt(poolSizeArg.split('=')[1], 10) : undefined; + const poolSizeOne = poolSize === 1; + const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? 1 : suggestedPoolSize); + console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSize: ${poolSize ?? 'default'})`); + return { testVariant, testGroup, oneIfNotPooled, poolSize, buildDir, isMock: testVariant.includes('-MOCK') }; }; export type TestMessage = {