#4955 Use concurrency param for browser pool

.semaphore/semaphore.yml

@@ -34,12 +34,9 @@ blocks:
             - npm -v
             - node -v
             - npm run-script test_ci_chrome_consumer
-        - name: consumer mock - flaky test group 1
+        - name: consumer mock - flaky test group
           commands:
-            - npm run-script test_ci_chrome_consumer_flaky_1
-        - name: consumer mock - flaky test group 2
-          commands:
-            - npm run-script test_ci_chrome_consumer_flaky_2
+            - npm run-script test_ci_chrome_consumer_flaky
         - name: content script tests
           commands:
             - npm run-script test_ci_chrome_content_scripts

package.json

@@ -87,10 +87,8 @@
     "test_ci_chrome_consumer_live_gmail": "npx ava --timeout=45m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-LIVE-GMAIL STANDARD-GROUP | npx tap-xunit > report.xml",
     "test_ci_chrome_consumer": "npx ava --timeout=30m --verbose --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK STANDARD-GROUP",
     "test_ci_chrome_enterprise": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- ENTERPRISE-MOCK STANDARD-GROUP | npx tap-xunit > report.xml",
-    "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml",
-    "test_ci_chrome_consumer_flaky_1": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-1 | npx tap-xunit > report.xml",
-    "test_ci_chrome_consumer_flaky_2": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-2 | npx tap-xunit > report.xml",
-    "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS > report.xml",
+    "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP --pool-size=3 | npx tap-xunit > report.xml",
+    "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS --pool-size=1 > report.xml",
     "dev_start_gmail_mock_api": "./scripts/build.sh && cd ./conf && node ../build/tooling/tsc-compiler --project tsconfig.test.json && cd .. && node ./build/test/test/source/mock.js",
     "run_firefox": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile ~/.mozilla/firefox/flowcrypt-dev --keep-profile-changes",
     "run_firefox_windows": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile %userprofile%/AppData/Local/Mozilla/Firefox/Profiles/flowcrypt-dev --keep-profile-changes",

test/source/test.ts

@@ -24,7 +24,7 @@ import { defineSetupTests } from './tests/setup';
 import { defineUnitBrowserTests } from './tests/unit-browser';
 import { defineUnitNodeTests } from './tests/unit-node';
 
-export const { testVariant, testGroup, oneIfNotPooled, buildDir, isMock } = getParsedCliParams();
+export const { testVariant, testGroup, oneIfNotPooled, poolSize, buildDir, isMock } = getParsedCliParams();
 export const internalTestState = { expectIntentionalErrReport: false }; // updated when a particular test that causes an error is run
 const DEBUG_BROWSER_LOG = false; // set to true to print / export information from browser
 const DEBUG_MOCK_LOG = false; // set to true to print mock server logs
@@ -38,7 +38,7 @@ const consts = {
   TIMEOUT_EACH_RETRY: minutes(4),
   TIMEOUT_ALL_RETRIES: minutes(55),
   ATTEMPTS: testGroup === 'STANDARD-GROUP' ? oneIfNotPooled(3) : process.argv.includes('--retry=false') ? 1 : 3,
-  POOL_SIZE: oneIfNotPooled(isMock ? 20 : 3),
+  POOL_SIZE: poolSize ?? (isMock ? 20 : 3),
   IS_LOCAL_DEBUG: process.argv.includes('--debug') ? true : false, // run locally by developer, not in ci
 };
 
@@ -236,10 +236,6 @@ if (testGroup === 'UNIT-TESTS') {
   defineUnitBrowserTests(testVariant, testWithBrowser);
 } else if (testGroup === 'FLAKY-GROUP') {
   defineFlakyTests(testVariant, testWithBrowser);
-} else if (testGroup === 'FLAKY-GROUP-1') {
-  defineFlakyTests(testVariant, testWithBrowser, 0, 2);
-} else if (testGroup === 'FLAKY-GROUP-2') {
-  defineFlakyTests(testVariant, testWithBrowser, 1, 2);
 } else if (testGroup === 'CONTENT-SCRIPT-TESTS') {
   defineContentScriptTests(testWithBrowser);
 } else {

test/source/tests/flaky.ts

@@ -29,12 +29,9 @@ import { minutes } from './tooling';
 // these tests are run serially, one after another, because they are somewhat more sensitive to parallel testing
 // eg if they are very cpu-sensitive (create key tests)
 
-export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser, shardIndex = 0, totalShards = 1) => {
-  let testCounter = 0;
+export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => {
   const test = (title: string, impl: Implementation<unknown[]>) => {
-    if (testCounter++ % totalShards === shardIndex) {
-      avaTest(title, impl);
-    }
+    avaTest(title, impl);
   };
   test.skip = avaTest.skip;
 

test/source/util/index.ts

@@ -14,7 +14,7 @@ const ROOT_DIR = process.cwd();
 
 export const getParsedCliParams = () => {
   let testVariant: TestVariant;
-  let testGroup: 'FLAKY-GROUP' | 'FLAKY-GROUP-1' | 'FLAKY-GROUP-2' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined;
+  let testGroup: 'FLAKY-GROUP' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined;
   if (process.argv.includes('CONTENT-SCRIPT-TESTS')) {
     testVariant = 'CONSUMER-CONTENT-SCRIPT-TESTS-MOCK';
     testGroup = 'CONTENT-SCRIPT-TESTS';
@@ -28,13 +28,15 @@ export const getParsedCliParams = () => {
     throw new Error('Unknown test type: specify CONSUMER-MOCK or ENTERPRISE-MOCK CONSUMER-LIVE-GMAIL');
   }
   if (!testGroup) {
-    testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : process.argv.includes('FLAKY-GROUP-1') ? 'FLAKY-GROUP-1' : process.argv.includes('FLAKY-GROUP-2') ? 'FLAKY-GROUP-2' : 'STANDARD-GROUP';
+    testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : 'STANDARD-GROUP';
   }
   const buildDir = join(ROOT_DIR, `build/chrome-${(testVariant === 'CONSUMER-LIVE-GMAIL' ? 'CONSUMER' : testVariant).toLowerCase()}`);
-  const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'FLAKY-GROUP-1', 'FLAKY-GROUP-2', 'CONTENT-SCRIPT-TESTS'].includes(testGroup);
-  const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? Math.min(1, suggestedPoolSize) : suggestedPoolSize);
-  console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSizeOne: ${poolSizeOne})`);
-  return { testVariant, testGroup, oneIfNotPooled, buildDir, isMock: testVariant.includes('-MOCK') };
+  const poolSizeArg = process.argv.find(a => a.startsWith('--pool-size='));
+  const poolSize = poolSizeArg ? parseInt(poolSizeArg.split('=')[1], 10) : undefined;
+  const poolSizeOne = poolSize === 1;
+  const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? 1 : suggestedPoolSize);
+  console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSize: ${poolSize ?? 'default'})`);
+  return { testVariant, testGroup, oneIfNotPooled, poolSize, buildDir, isMock: testVariant.includes('-MOCK') };
 };
 
 export type TestMessage = {