Tests: reuse browser workers in BrowserStack tests (#5428)

- reuse BrowserStack workers.
- add support for "latest" and "latest-1" in browser version filters
- add support for specifying non-final browser versions, such as beta versions
- more accurate eslint for files in test/runner
- switched `--no-isolate` command flag to `--isolate`. Now that browser instances are shared, it made more sense to me to default to no isolation unless specified. This turned out to be cleaner because the only place we isolate is in browserstack.yml.
- fixed an issue with retries where it wasn't always waiting for the retried test run
- enable strict mode in test yargs command

.github/workflows/browserstack.yml

@@ -16,24 +16,25 @@ jobs:
     name: ${{ matrix.BROWSER }}
     concurrency:
       group: ${{ github.workflow }} ${{ matrix.BROWSER }}
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         BROWSER:
           - 'IE_11'
-          - 'Safari_17'
-          - 'Safari_16'
-          - 'Chrome_120'
-          - 'Chrome_119'
-          - 'Edge_120'
-          - 'Edge_119'
-          - 'Firefox_121'
-          - 'Firefox_120'
+          - 'Safari_latest'
+          - 'Safari_latest-1'
+          - 'Chrome_latest'
+          - 'Chrome_latest-1'
+          - 'Opera_latest'
+          - 'Edge_latest'
+          - 'Edge_latest-1'
+          - 'Firefox_latest'
+          - 'Firefox_latest-1'
           - 'Firefox_115'
           - '__iOS_17'
           - '__iOS_16'
           - '__iOS_15'
-          - 'Opera_106'
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -61,4 +62,4 @@ jobs:
         run: npm run pretest
 
       - name: Run tests
-        run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --retries 3
+        run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --run-id ${{ github.run_id }} --isolate --retries 3

eslint.config.js

@@ -21,8 +21,7 @@ export default [
 			"test/node_smoke_tests/commonjs/**",
 			"test/node_smoke_tests/module/**",
 			"test/promises_aplus_adapters/**",
-			"test/middleware-mockserver.cjs",
-			"test/runner/**/*.js"
+			"test/middleware-mockserver.cjs"
 		],
 		languageOptions: {
 			globals: {
@@ -35,13 +34,6 @@ export default [
 		}
 	},
 
-	{
-		files: [ "test/runner/listeners.js" ],
-		languageOptions: {
-			sourceType: "script"
-		}
-	},
-
 	// Source
 	{
 		files: [ "src/**" ],
@@ -222,6 +214,29 @@ export default [
 		}
 	},
 
+	{
+		files: [
+			"test/runner/**/*.js"
+		],
+		languageOptions: {
+			globals: {
+				...globals.node
+			},
+			sourceType: "module"
+		},
+		rules: {
+			...jqueryConfig.rules
+		}
+	},
+
+	{
+		files: [ "test/runner/listeners.js" ],
+		languageOptions: {
+			ecmaVersion: 5,
+			sourceType: "script"
+		}
+	},
+
 	{
 		files: [
 			"test/data/testrunner.js",

package.json

@@ -54,20 +54,20 @@
     "pretest": "npm run qunit-fixture && npm run babel:tests && npm run npmcopy",
     "qunit-fixture": "node build/tasks/qunit-fixture.js",
     "start": "node -e \"require('./build/tasks/build.js').buildDefaultFiles({ watch: true })\"",
-    "test:browser": "npm run pretest && npm run build:main && npm run test:unit -- -b chrome -b firefox --no-isolate -h",
+    "test:browser": "npm run pretest && npm run build:main && npm run test:unit -- -b chrome -b firefox -h",
     "test:browserless": "npm run pretest && npm run build:all && node build/tasks/node_smoke_tests.js && node build/tasks/promises_aplus_tests.js && npm run test:unit -- -b jsdom -m basic",
     "test:jsdom": "npm run pretest && npm run build:main && npm run test:unit -- -b jsdom -m basic",
     "test:node_smoke_tests": "npm run pretest && npm run build:all && node build/tasks/node_smoke_tests.js",
     "test:promises_aplus": "npm run build:main && node build/tasks/promises_aplus_tests.js",
-    "test:firefox": "npm run pretest && npm run build:main && npm run test:unit -- -v -b firefox --no-isolate -h",
-    "test:safari": "npm run pretest && npm run build:main && npm run test:unit -- -b safari --no-isolate",
+    "test:firefox": "npm run pretest && npm run build:main && npm run test:unit -- -v -b firefox -h",
+    "test:safari": "npm run pretest && npm run build:main && npm run test:unit -- -b safari",
     "test:server": "node test/runner/server.js",
-    "test:esm": "npm run pretest && npm run build:main && npm run test:unit -- --esm --no-isolate -h",
-    "test:no-deprecated": "npm run pretest && npm run build -- -e deprecated && npm run test:unit -- --no-isolate -h",
-    "test:selector-native": "npm run pretest && npm run build -- -e selector && npm run test:unit -- --no-isolate -h",
-    "test:slim": "npm run pretest && npm run build -- --slim && npm run test:unit -- --no-isolate -h",
+    "test:esm": "npm run pretest && npm run build:main && npm run test:unit -- --esm -h",
+    "test:no-deprecated": "npm run pretest && npm run build -- -e deprecated && npm run test:unit -- -h",
+    "test:selector-native": "npm run pretest && npm run build -- -e selector && npm run test:unit -- -h",
+    "test:slim": "npm run pretest && npm run build -- --slim && npm run test:unit -- -h",
     "test:unit": "node test/runner/command.js",
-    "test": "npm run build:all && npm run lint && npm run test:browserless && npm run test:browser && npm run test:esmodules && npm run test:slim && npm run test:no-deprecated && npm run test:selector-native"
+    "test": "npm run build:all && npm run lint && npm run test:browserless && npm run test:browser && npm run test:esm && npm run test:slim && npm run test:no-deprecated && npm run test:selector-native"
   },
   "homepage": "https://jquery.com",
   "author": {

test/data/testinit.js

@@ -433,7 +433,6 @@ this.loadTests = function() {
 				}
 
 			} else {
-				QUnit.load();
 
 				/**
 				 * Run in noConflict mode

test/index.html

@@ -35,7 +35,7 @@
 			// We need to read both.
 			var esmodules = QUnit.config.esmodules || QUnit.urlParams.esmodules;
 
-			// `loadTests()` will call `QUnit.load()` because tests
+			// `loadTests()` will call `QUnit.start()` because tests
 			// such as unit/ready.js should run after document ready.
 			if ( !esmodules ) {
 				loadTests();

test/runner/browserstack/api.js

@@ -14,6 +14,7 @@ const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
 // iOS has null for version numbers,
 // and we do not need a similar check for OS versions.
 const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+const rlatest = /^latest-(\d+)$/;
 
 const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
 
@@ -84,6 +85,15 @@ function compareVersionNumbers( a, b ) {
 			return 1;
 		}
 	}
+
+	if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) {
+		return -1;
+	}
+	if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) {
+		return 1;
+	}
+
+	return 0;
 }
 
 function sortBrowsers( a, b ) {
@@ -148,17 +158,62 @@ export async function filterBrowsers( filter ) {
 	const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
 	const filterDevice = ( filter.device ?? "" ).toLowerCase();
 
-	return browsers.filter( ( browser ) => {
+	const filteredWithoutVersion = browsers.filter( ( browser ) => {
 		return (
 			( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
-			( !filterVersion ||
-				matchVersion( browser.browser_version, filterVersion ) ) &&
 			( !filterOs || filterOs === browser.os.toLowerCase() ) &&
-			( !filterOsVersion ||
-				filterOsVersion === browser.os_version.toLowerCase() ) &&
+			( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) &&
 			( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
 		);
 	} );
+
+	if ( !filterVersion ) {
+		return filteredWithoutVersion;
+	}
+
+	if ( filterVersion.startsWith( "latest" ) ) {
+		const groupedByName = filteredWithoutVersion
+			.filter( ( b ) => rfinalVersion.test( b.browser_version ) )
+			.reduce( ( acc, browser ) => {
+				acc[ browser.browser ] = acc[ browser.browser ] ?? [];
+				acc[ browser.browser ].push( browser );
+				return acc;
+			}, Object.create( null ) );
+
+		const filtered = [];
+		for ( const group of Object.values( groupedByName ) ) {
+			const latest = group[ group.length - 1 ];
+
+			// Mobile devices do not have browser version.
+			// Skip the version check for these,
+			// but include the latest in the list if it made it
+			// through filtering.
+			if ( !latest.browser_version ) {
+
+				// Do not include in the list for latest-n.
+				if ( filterVersion === "latest" ) {
+					filtered.push( latest );
+				}
+				continue;
+			}
+
+			// Get the latest version and subtract the number from the filter,
+			// ignoring any patch versions, which may differ between major versions.
+			const num = rlatest.exec( filterVersion );
+			const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 );
+			const match = group.findLast( ( browser ) => {
+				return matchVersion( browser.browser_version, version.toString() );
+			} );
+			if ( match ) {
+				filtered.push( match );
+			}
+		}
+		return filtered;
+	}
+
+	return filteredWithoutVersion.filter( ( browser ) => {
+		return matchVersion( browser.browser_version, filterVersion );
+	} );
 }
 
 export async function listBrowsers( filter ) {
@@ -177,13 +232,11 @@ export async function listBrowsers( filter ) {
 }
 
 export async function getLatestBrowser( filter ) {
+	if ( !filter.browser_version ) {
+		filter.browser_version = "latest";
+	}
 	const browsers = await filterBrowsers( filter );
-
-	// The list is sorted in ascending order,
-	// so the last item is the latest.
-	return browsers.findLast( ( browser ) =>
-		rfinalVersion.test( browser.browser_version )
-	);
+	return browsers[ browsers.length - 1 ];
 }
 
 /**
@@ -229,31 +282,14 @@ export function getWorker( id ) {
 	return fetchAPI( `/worker/${ id }` );
 }
 
-export async function deleteWorker( id, verbose ) {
-	await fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
-	if ( verbose ) {
-		console.log( `\nWorker ${ id } stopped.` );
-	}
+export async function deleteWorker( id ) {
+	return fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
 }
 
 export function getWorkers() {
 	return fetchAPI( "/workers" );
 }
 
-/**
- * Change the URL of a worker,
- * or refresh if it's the same URL.
- */
-export function changeUrl( id, url ) {
-	return fetchAPI( `/worker/${ id }/url.json`, {
-		method: "PUT",
-		body: JSON.stringify( {
-			timeout: 20,
-			url: encodeURI( url )
-		} )
-	} );
-}
-
 /**
  * Stop all workers
  */
@@ -262,15 +298,17 @@ export async function stopWorkers() {
 
 	// Run each request on its own
 	// to avoid connect timeout errors.
+	console.log( `${ workers.length } workers running...` );
 	for ( const worker of workers ) {
 		try {
-			await deleteWorker( worker.id, true );
+			await deleteWorker( worker.id );
 		} catch ( error ) {
 
 			// Log the error, but continue trying to remove workers.
 			console.error( error );
 		}
 	}
+	console.log( "All workers stopped." );
 }
 
 /**
@@ -284,6 +322,11 @@ export function getPlan() {
 }
 
 export async function getAvailableSessions() {
+	try {
 		const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
 		return plan.parallel_sessions_max_allowed - workers.length;
+	} catch ( error ) {
+		console.error( error );
+		return 0;
+	}
 }

test/runner/browserstack/browsers.js

@@ -0,0 +1,199 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { createWorker, deleteWorker, getAvailableSessions } from "./api.js";
+
+const workers = Object.create( null );
+
+/**
+ * Keys are browser strings
+ * Structure of a worker:
+ * {
+ * 	 debug: boolean, // Stops the worker from being cleaned up when finished
+ *   id: string,
+ *   lastTouch: number, // The last time a request was received
+ *   url: string,
+ *   browser: object, // The browser object
+ *   options: object // The options to create the worker
+ * }
+ */
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_INTERVAL = 1000;
+const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
+
+const MAX_WORKER_RESTARTS = 5;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+
+const WORKER_WAIT_TIME = 30000;
+
+export function touchBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+async function waitForAck( worker, { fullBrowser, verbose } ) {
+	delete worker.lastTouch;
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\n${ fullBrowser } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_INTERVAL );
+
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			reject(
+				new Error(
+					`${ fullBrowser } not acknowledged after ${
+						ACKNOWLEDGE_TIMEOUT / 1000 / 60
+					}min.`
+				)
+			);
+		}, ACKNOWLEDGE_TIMEOUT );
+	} );
+}
+
+async function ensureAcknowledged( worker, restarts ) {
+	const fullBrowser = getBrowserString( worker.browser );
+	const verbose = worker.options.verbose;
+	try {
+		await waitForAck( worker, { fullBrowser, verbose } );
+		return worker;
+	} catch ( error ) {
+		console.error( error.message );
+		await cleanupWorker( worker, { verbose } );
+		await createBrowserWorker(
+			worker.url,
+			worker.browser,
+			worker.options,
+			restarts + 1
+		);
+	}
+}
+
+export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
+	if ( restarts > MAX_WORKER_RESTARTS ) {
+		throw new Error(
+			`Reached the maximum number of restarts for ${ chalk.yellow(
+				getBrowserString( browser )
+			) }`
+		);
+	}
+	const verbose = options.verbose;
+	while ( ( await getAvailableSessions() ) <= 0 ) {
+		if ( verbose ) {
+			console.log( "\nWaiting for available sessions..." );
+		}
+		await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+	}
+
+	const { debug, runId, tunnelId } = options;
+	const fullBrowser = getBrowserString( browser );
+
+	const worker = await createWorker( {
+		...browser,
+		url: encodeURI( url ),
+		project: "jquery",
+		build: `Run ${ runId }`,
+
+		// This is the maximum timeout allowed
+		// by BrowserStack. We do this because
+		// we control the timeout in the runner.
+		// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
+		timeout: 1800,
+
+		// Not documented in the API docs,
+		// but required to make local testing work.
+		// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+		"browserstack.local": true,
+		"browserstack.localIdentifier": tunnelId
+	} );
+
+	browser.debug = !!debug;
+	worker.url = url;
+	worker.browser = browser;
+	worker.restarts = restarts;
+	worker.options = options;
+	touchBrowser( browser );
+	workers[ fullBrowser ] = worker;
+
+	// Wait for the worker to show up in the list
+	// before returning it.
+	return ensureAcknowledged( worker, restarts );
+}
+
+export async function setBrowserWorkerUrl( browser, url ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.url = url;
+	}
+}
+
+/**
+ * Checks that all browsers have received
+ * a response in the given amount of time.
+ * If not, the worker is restarted.
+ */
+export async function checkLastTouches() {
+	for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
+		if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
+			const options = worker.options;
+			if ( options.verbose ) {
+				console.log(
+					`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
+						RUN_WORKER_TIMEOUT / 1000 / 60
+					}min.`
+				);
+			}
+			await cleanupWorker( worker, options );
+			await createBrowserWorker(
+				worker.url,
+				worker.browser,
+				options,
+				worker.restarts
+			);
+		}
+	}
+}
+
+export async function cleanupWorker( worker, { verbose } ) {
+	for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
+		if ( w === worker ) {
+			delete workers[ fullBrowser ];
+			await deleteWorker( worker.id );
+			if ( verbose ) {
+				console.log( `\nStopped ${ fullBrowser }.` );
+			}
+			return;
+		}
+	}
+}
+
+export async function cleanupAllBrowsers( { verbose } ) {
+	const workersRemaining = Object.values( workers );
+	const numRemaining = workersRemaining.length;
+	if ( numRemaining ) {
+		await Promise.all(
+			workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
+		);
+		if ( verbose ) {
+			console.log(
+				`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+			);
+		}
+	}
+}

test/runner/browserstack/buildBrowserFromString.js

@@ -3,8 +3,18 @@ export function buildBrowserFromString( str ) {
 
 	// If the version starts with a colon, it's a device
 	if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
-		return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion };
+		return {
+			browser,
+			device: versionOrDevice.slice( 1 ),
+			os,
+			os_version: osVersion
+		};
 	}
 
-	return { browser, browser_version: versionOrDevice, os, os_version: osVersion };
+	return {
+		browser,
+		browser_version: versionOrDevice,
+		os,
+		os_version: osVersion
+	};
 }

test/runner/browserstack/queue.js

@@ -0,0 +1,90 @@
+import chalk from "chalk";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { checkLastTouches, createBrowserWorker, setBrowserWorkerUrl } from "./browsers.js";
+
+const TEST_POLL_TIMEOUT = 1000;
+
+const queue = [];
+
+export function getNextBrowserTest( reportId ) {
+	const index = queue.findIndex( ( test ) => test.id === reportId );
+	if ( index === -1 ) {
+		return;
+	}
+
+	// Remove the completed test from the queue
+	const previousTest = queue[ index ];
+	queue.splice( index, 1 );
+
+	// Find the next test for the same browser
+	for ( const test of queue.slice( index ) ) {
+		if ( test.fullBrowser === previousTest.fullBrowser ) {
+
+			// Set the URL for our tracking
+			setBrowserWorkerUrl( test.browser, test.url );
+			test.running = true;
+
+			// Return the URL for the next test.
+			// listeners.js will use this to set the browser URL.
+			return { url: test.url };
+		}
+	}
+}
+
+export function retryTest( reportId, maxRetries ) {
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.retries++;
+		if ( test.retries <= maxRetries ) {
+			console.log(
+				`Retrying test ${ reportId } for ${ chalk.yellow(
+					test.options.modules.join( ", " )
+				) }...`
+			);
+			return test;
+		}
+	}
+}
+
+export function addBrowserStackRun( url, browser, options ) {
+	queue.push( {
+		browser,
+		fullBrowser: getBrowserString( browser ),
+		id: options.reportId,
+		url,
+		options,
+		retries: 0,
+		running: false
+	} );
+}
+
+export async function runAllBrowserStack() {
+	return new Promise( async( resolve, reject )=> {
+		while ( queue.length ) {
+			try {
+				await checkLastTouches();
+			} catch ( error ) {
+				reject( error );
+			}
+
+			// Run one test URL per browser at a time
+			const browsersTaken = [];
+			for ( const test of queue ) {
+				if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) {
+					continue;
+				}
+				browsersTaken.push( test.fullBrowser );
+				if ( !test.running ) {
+					test.running = true;
+					try {
+						await createBrowserWorker( test.url, test.browser, test.options );
+					} catch ( error ) {
+						reject( error );
+					}
+				}
+			}
+			await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) );
+		}
+		resolve();
+	} );
+}

test/runner/browserstack/workers.js

@@ -1,276 +0,0 @@
-import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
-import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js";
-
-const workers = Object.create( null );
-
-// Acknowledge the worker within the time limit.
-// BrowserStack can take much longer spinning up
-// some browsers, such as iOS 15 Safari.
-const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8;
-const ACKNOWLEDGE_WORKER_INTERVAL = 1000;
-
-// No report after the time limit
-// should refresh the worker
-const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
-const MAX_WORKER_RESTARTS = 5;
-const MAX_WORKER_REFRESHES = 1;
-const POLL_WORKER_TIMEOUT = 1000;
-
-export async function cleanupWorker( reportId, verbose ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		try {
-			delete workers[ reportId ];
-			await deleteWorker( worker.id, verbose );
-		} catch ( error ) {
-			console.error( error );
-		}
-	}
-}
-
-export function debugWorker( reportId ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.debug = true;
-	}
-}
-
-/**
- * Set the last time a request was
- * received related to the worker.
- */
-export function touchWorker( reportId ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.lastTouch = Date.now();
-	}
-}
-
-export function retryTest( reportId, retries ) {
-	const worker = workers[ reportId ];
-	if ( worker ) {
-		worker.retries ||= 0;
-		worker.retries++;
-		if ( worker.retries <= retries ) {
-			worker.retry = true;
-			console.log( `\nRetrying test ${ reportId }...${ worker.retries }` );
-			return true;
-		}
-	}
-	return false;
-}
-
-export async function cleanupAllWorkers( verbose ) {
-	const workersRemaining = Object.keys( workers ).length;
-	if ( workersRemaining ) {
-		if ( verbose ) {
-			console.log(
-				`Stopping ${ workersRemaining } stray worker${
-					workersRemaining > 1 ? "s" : ""
-				}...`
-			);
-		}
-		await Promise.all(
-			Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) )
-		);
-	}
-}
-
-async function waitForAck( id, verbose ) {
-	return new Promise( ( resolve, reject ) => {
-		const interval = setInterval( () => {
-			const worker = workers[ id ];
-			if ( !worker ) {
-				clearTimeout( timeout );
-				clearInterval( interval );
-				return reject( new Error( `Worker ${ id } not found.` ) );
-			}
-			if ( worker.lastTouch ) {
-				if ( verbose ) {
-					console.log( `\nWorker ${ id } acknowledged.` );
-				}
-				clearTimeout( timeout );
-				clearInterval( interval );
-				resolve();
-			}
-		}, ACKNOWLEDGE_WORKER_INTERVAL );
-		const timeout = setTimeout( () => {
-			clearInterval( interval );
-			const worker = workers[ id ];
-			reject(
-				new Error(
-					`Worker ${
-						worker ? worker.id : ""
-					} for test ${ id } not acknowledged after ${
-						ACKNOWLEDGE_WORKER_TIMEOUT / 1000
-					}s.`
-				)
-			);
-		}, ACKNOWLEDGE_WORKER_TIMEOUT );
-	} );
-}
-
-export async function runWorker(
-	url,
-	browser,
-	options,
-	restarts = 0
-) {
-	const { modules, reportId, runId, verbose } = options;
-	const worker = await createWorker( {
-		...browser,
-		url: encodeURI( url ),
-		project: "jquery",
-		build: `Run ${ runId }`,
-		name: `${ modules.join( "," ) } (${ reportId })`,
-
-		// Set the max here, so that we can
-		// control the timeout
-		timeout: 1800,
-
-		// Not documented in the API docs,
-		// but required to make local testing work.
-		// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
-		"browserstack.local": true,
-		"browserstack.localIdentifier": runId
-	} );
-
-	workers[ reportId ] = worker;
-
-	const timeMessage = `\nWorker ${
-		worker.id
-	} created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`;
-
-	if ( verbose ) {
-		console.time( timeMessage );
-	}
-
-	async function retryWorker() {
-		await cleanupWorker( reportId, verbose );
-		if ( verbose ) {
-			console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` );
-		}
-		return runWorker( url, browser, options, restarts + 1 );
-	}
-
-	// Wait for the worker to be acknowledged
-	try {
-		await waitForAck( reportId );
-	} catch ( error ) {
-		if ( !workers[ reportId ] ) {
-
-			// The worker has already been cleaned up
-			return;
-		}
-
-		if ( restarts < MAX_WORKER_RESTARTS ) {
-			return retryWorker();
-		}
-
-		throw error;
-	}
-
-	if ( verbose ) {
-		console.timeEnd( timeMessage );
-	}
-
-	let refreshes = 0;
-	let loggedStarted = false;
-	return new Promise( ( resolve, reject ) => {
-		async function refreshWorker() {
-			try {
-				await changeUrl( worker.id, url );
-				touchWorker( reportId );
-				return tick();
-			} catch ( error ) {
-				if ( !workers[ reportId ] ) {
-
-					// The worker has already been cleaned up
-					return resolve();
-				}
-				console.error( error );
-				return retryWorker().then( resolve, reject );
-			}
-		}
-
-		async function checkWorker() {
-			const worker = workers[ reportId ];
-
-			if ( !worker || worker.debug ) {
-				return resolve();
-			}
-
-			let fetchedWorker;
-			try {
-				fetchedWorker = await getWorker( worker.id );
-			} catch ( error ) {
-				return reject( error );
-			}
-			if (
-				!fetchedWorker ||
-				( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" )
-			) {
-				return resolve();
-			}
-
-			if ( verbose && !loggedStarted ) {
-				loggedStarted = true;
-				console.log(
-					`\nTest ${ chalk.bold( reportId ) } is ${
-						worker.status === "running" ? "running" : "in the queue"
-					}.`
-				);
-				console.log( `  View at ${ fetchedWorker.browser_url }.` );
-			}
-
-			// Refresh the worker if a retry is triggered
-			if ( worker.retry ) {
-				worker.retry = false;
-
-				// Reset recovery refreshes
-				refreshes = 0;
-				return refreshWorker();
-			}
-
-			if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) {
-				return tick();
-			}
-
-			refreshes++;
-
-			if ( refreshes >= MAX_WORKER_REFRESHES ) {
-				if ( restarts < MAX_WORKER_RESTARTS ) {
-					if ( verbose ) {
-						console.log(
-							`Worker ${ worker.id } not acknowledged after ${
-								ACKNOWLEDGE_WORKER_TIMEOUT / 1000
-							}s.`
-						);
-					}
-					return retryWorker().then( resolve, reject );
-				}
-				await cleanupWorker( reportId, verbose );
-				return reject(
-					new Error(
-						`Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.`
-					)
-				);
-			}
-
-			if ( verbose ) {
-				console.log(
-					`\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }`
-				);
-			}
-
-			return refreshWorker();
-		}
-
-		function tick() {
-			setTimeout( checkWorker, POLL_WORKER_TIMEOUT );
-		}
-
-		checkWorker();
-	} );
-}

test/runner/command.js

@@ -7,6 +7,7 @@ import { run } from "./run.js";
 
 const argv = yargs( process.argv.slice( 2 ) )
 	.version( false )
+	.strict()
 	.command( {
 		command: "[options]",
 		describe: "Run jQuery tests in a browser"
@@ -48,7 +49,8 @@ const argv = yargs( process.argv.slice( 2 ) )
 		type: "number",
 		description:
 			"Run tests in parallel in multiple browsers. " +
-			"Defaults to 8 in normal mode. In browserstack mode, defaults to the maximum available under your BrowserStack plan."
+			"Defaults to 8 in normal mode. In browserstack mode, " +
+			"defaults to the maximum available under your BrowserStack plan."
 	} )
 	.option( "debug", {
 		alias: "d",
@@ -65,21 +67,28 @@ const argv = yargs( process.argv.slice( 2 ) )
 	.option( "retries", {
 		alias: "r",
 		type: "number",
-		description: "Number of times to retry failed tests.",
-		default: 0
+		description: "Number of times to retry failed tests in BrowserStack.",
+		implies: [ "browserstack" ]
 	} )
-	.option( "no-isolate", {
+	.option( "run-id", {
+		type: "string",
+		description: "A unique identifier for this run."
+	} )
+	.option( "isolate", {
 		type: "boolean",
-		description: "Run all modules in the same browser instance."
+		description: "Run each module by itself in the test page. This can extend testing time."
 	} )
 	.option( "browserstack", {
 		type: "array",
 		description:
-			"Run tests in BrowserStack.\nRequires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
+			"Run tests in BrowserStack.\n" +
+			"Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
 			"The value can be empty for the default configuration, or a string in the format of\n" +
 			"\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" +
-			"Pass multiple browsers by repeating the option. The --browser option is ignored when --browserstack has a value.\n" +
-			"Otherwise, the --browser option will be used, with the latest version/device for that browser, on a matching OS."
+			"Pass multiple browsers by repeating the option.\n" +
+			"The --browser option is ignored when --browserstack has a value.\n" +
+			"Otherwise, the --browser option will be used, " +
+			"with the latest version/device for that browser, on a matching OS."
 	} )
 	.option( "list-browsers", {
 		type: "string",
@@ -88,8 +97,11 @@ const argv = yargs( process.argv.slice( 2 ) )
 			"Leave blank to view all browsers or pass " +
 			"\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " +
 			"separated by an underscore to filter the list (any can be omitted).\n" +
+			"\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" +
+			"\"latest-n\" can be used to find the nth latest browser version.\n" +
 			"Use a colon to indicate a device.\n" +
-			"Examples: \"chrome__windows_10\", \"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
+			"Examples: \"chrome__windows_10\", \"safari_latest\", " +
+			"\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
 			"Use quotes if spaces are necessary."
 	} )
 	.option( "stop-workers", {

test/runner/createTestServer.js

@@ -35,7 +35,11 @@ export async function createTestServer( report ) {
 	// Bind the reporter
 	app.post( "/api/report", bodyParser.json( { limit: "50mb" } ), ( req, res ) => {
 		if ( report ) {
-			report( req.body );
+			const response = report( req.body );
+			if ( response ) {
+				res.json( response );
+				return;
+			}
 		}
 		res.sendStatus( 204 );
 	} );

test/runner/jsdom.js

@@ -24,7 +24,7 @@ export async function runJSDOM( url, { reportId, verbose } ) {
 	} );
 }
 
-export function cleanupJSDOM( reportId, verbose ) {
+export function cleanupJSDOM( reportId, { verbose } ) {
 	const window = windows[ reportId ];
 	if ( window ) {
 		if ( window.finish ) {
@@ -38,7 +38,7 @@ export function cleanupJSDOM( reportId, verbose ) {
 	}
 }
 
-export function cleanupAllJSDOM( verbose ) {
+export function cleanupAllJSDOM( { verbose } ) {
 	const windowsRemaining = Object.keys( windows ).length;
 	if ( windowsRemaining ) {
 		if ( verbose ) {
@@ -49,7 +49,7 @@ export function cleanupAllJSDOM( verbose ) {
 			);
 		}
 		for ( const id in windows ) {
-			cleanupJSDOM( id, verbose );
+			cleanupJSDOM( id, { verbose } );
 		}
 	}
 }

test/runner/listeners.js

@@ -68,6 +68,7 @@
 		request.open( "POST", "/api/report", true );
 		request.setRequestHeader( "Content-Type", "application/json" );
 		request.send( json );
+		return request;
 	}
 
 	// Send acknowledgement to the server.
@@ -83,6 +84,16 @@
 		// childSuites is large and unused.
 		data.childSuites = undefined;
 
-		send( "runEnd", data );
+		var request = send( "runEnd", data );
+		request.onload = function() {
+			if ( request.status === 200 && request.responseText ) {
+				try {
+					var json = JSON.parse( request.responseText );
+					window.location = json.url;
+				} catch ( e ) {
+					console.error( e );
+				}
+			}
+		};
 	} );
 } )();

test/runner/run.js

@@ -4,20 +4,20 @@ import { getLatestBrowser } from "./browserstack/api.js";
 import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
 import { localTunnel } from "./browserstack/local.js";
 import { reportEnd, reportTest } from "./reporter.js";
-import {
-	cleanupAllWorkers,
-	cleanupWorker,
-	debugWorker,
-	retryTest,
-	touchWorker
-} from "./browserstack/workers.js";
 import { createTestServer } from "./createTestServer.js";
 import { buildTestUrl } from "./lib/buildTestUrl.js";
 import { generateHash, printModuleHashes } from "./lib/generateHash.js";
 import { getBrowserString } from "./lib/getBrowserString.js";
-import { addRun, runFullQueue } from "./queue.js";
 import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js";
 import { modules as allModules } from "./modules.js";
+import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js";
+import {
+	addBrowserStackRun,
+	getNextBrowserTest,
+	retryTest,
+	runAllBrowserStack
+} from "./browserstack/queue.js";
+import { addSeleniumRun, runAllSelenium } from "./selenium/queue.js";
 
 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
 
@@ -31,11 +31,12 @@ export async function run( {
 	debug,
 	esm,
 	headless,
-	isolate = true,
+	isolate,
 	modules = [],
-	retries = 3,
+	retries = 0,
+	runId,
 	verbose
-} = {} ) {
+} ) {
 	if ( !browserNames || !browserNames.length ) {
 		browserNames = [ "chrome" ];
 	}
@@ -57,24 +58,28 @@ export async function run( {
 
 	// Convert browser names to browser objects
 	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
-
-	// A unique identifier for this run
-	const runId = generateHash(
+	const tunnelId = generateHash(
 		`${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] )
 			.concat( browserNames )
 			.join( ":" ) }`
 	);
 
+	// A unique identifier for this run
+	if ( !runId ) {
+		runId = tunnelId;
+	}
+
 	// Create the test app and
 	// hook it up to the reporter
 	const reports = Object.create( null );
-	const app = await createTestServer( async( message ) => {
+	const app = await createTestServer( ( message ) => {
 		switch ( message.type ) {
 			case "testEnd": {
 				const reportId = message.id;
-				touchWorker( reportId );
-				const errors = reportTest( message.data, reportId, reports[ reportId ] );
-				pendingErrors[ reportId ] ||= {};
+				const report = reports[ reportId ];
+				touchBrowser( report.browser );
+				const errors = reportTest( message.data, reportId, report );
+				pendingErrors[ reportId ] ??= Object.create( null );
 				if ( errors ) {
 					pendingErrors[ reportId ][ message.data.name ] = errors;
 				} else {
@@ -85,6 +90,7 @@ export async function run( {
 			case "runEnd": {
 				const reportId = message.id;
 				const report = reports[ reportId ];
+				touchBrowser( report.browser );
 				const { failed, total } = reportEnd(
 					message.data,
 					message.id,
@@ -92,31 +98,34 @@ export async function run( {
 				);
 				report.total = total;
 
+				cleanupJSDOM( reportId, { verbose } );
+
+				// Handle failure
 				if ( failed ) {
-					if ( !retryTest( reportId, retries ) ) {
-						if ( debug ) {
-							debugWorker( reportId );
+					const retry = retryTest( reportId, retries );
+					if ( retry ) {
+						return retry;
 					}
 					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
+					return getNextBrowserTest( reportId );
 				}
-				} else {
-					if ( Object.keys( pendingErrors[ reportId ] ).length ) {
+
+				// Handle success
+				if (
+					pendingErrors[ reportId ] &&
+					Object.keys( pendingErrors[ reportId ] ).length
+				) {
 					console.warn( "Detected flaky tests:" );
 					for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) {
 						console.warn( chalk.italic( chalk.gray( error ) ) );
 					}
 					delete pendingErrors[ reportId ];
 				}
-				}
-				await cleanupWorker( reportId, verbose );
-				cleanupJSDOM( reportId, verbose );
-				break;
+				return getNextBrowserTest( reportId );
 			}
 			case "ack": {
-				touchWorker( message.id );
-				if ( verbose ) {
-					console.log( `\nWorker for test ${ message.id } acknowledged.` );
-				}
+				const report = reports[ message.id ];
+				touchBrowser( report.browser );
 				break;
 			}
 			default:
@@ -165,8 +174,8 @@ export async function run( {
 			}
 		}
 
-		await cleanupAllWorkers( verbose );
-		cleanupAllJSDOM( verbose );
+		await cleanupAllBrowsers( { verbose } );
+		cleanupAllJSDOM( { verbose } );
 	}
 
 	asyncExitHook(
@@ -210,13 +219,16 @@ export async function run( {
 
 				const latestMatch = await getLatestBrowser( browser );
 				if ( !latestMatch ) {
-					throw new Error( `Browser not found: ${ getBrowserString( browser ) }.` );
+					console.error(
+						chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
+					);
+					gracefulExit( 1 );
 				}
 				return latestMatch;
 			} )
 		);
 
-		tunnel = await localTunnel( runId );
+		tunnel = await localTunnel( tunnelId );
 		if ( verbose ) {
 			console.log( "Started BrowserStackLocal." );
 
@@ -237,15 +249,21 @@ export async function run( {
 			reportId
 		} );
 
-		addRun( url, browser, {
+		const options = {
 			debug,
 			headless,
 			modules,
 			reportId,
-			retries,
 			runId,
+			tunnelId,
 			verbose
-		} );
+		};
+
+		if ( browserstack ) {
+			addBrowserStackRun( url, browser, options );
+		} else {
+			addSeleniumRun( url, browser, options );
+		}
 	}
 
 	for ( const browser of browsers ) {
@@ -260,7 +278,11 @@ export async function run( {
 
 	try {
 		console.log( `Starting Run ${ runId }...` );
-		await runFullQueue( { browserstack, concurrency, verbose } );
+		if ( browserstack ) {
+			await runAllBrowserStack( { verbose } );
+		} else {
+			await runAllSelenium( { concurrency, verbose } );
+		}
 	} catch ( error ) {
 		console.error( error );
 		if ( !debug ) {

test/runner/selenium/createDriver.js

@@ -55,7 +55,8 @@ export default async function createDriver( { browserName, headless, verbose } )
 		edgeOptions.addArguments( "--headless=new" );
 		if ( !browserSupportsHeadless( browserName ) ) {
 			console.log(
-				`Headless mode is not supported for ${ browserName }. Running in normal mode instead.`
+				`Headless mode is not supported for ${ browserName }.` +
+					"Running in normal mode instead."
 			);
 		}
 	}

test/runner/selenium/queue.js

@@ -3,33 +3,25 @@
 // and refills the queue when one promise resolves.
 
 import chalk from "chalk";
-import { getAvailableSessions } from "./browserstack/api.js";
-import { runWorker } from "./browserstack/workers.js";
-import { getBrowserString } from "./lib/getBrowserString.js";
-import { runSelenium } from "./selenium/runSelenium.js";
-import { runJSDOM } from "./jsdom.js";
+import { getBrowserString } from "../lib/getBrowserString.js";
+import { runSelenium } from "./runSelenium.js";
+import { runJSDOM } from "../jsdom.js";
 
-const queue = [];
 const promises = [];
+const queue = [];
 
 const SELENIUM_WAIT_TIME = 100;
-const BROWSERSTACK_WAIT_TIME = 5000;
-const WORKER_WAIT_TIME = 30000;
 
 // Limit concurrency to 8 by default in selenium
 // BrowserStack defaults to the max allowed by the plan
 // More than this will log MaxListenersExceededWarning
 const MAX_CONCURRENCY = 8;
 
-export function addRun( url, browser, options ) {
+export function addSeleniumRun( url, browser, options ) {
 	queue.push( { url, browser, options } );
 }
 
-export async function runFullQueue( {
-	browserstack,
-	concurrency: defaultConcurrency,
-	verbose
-} ) {
+export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) {
 	while ( queue.length ) {
 		const next = queue.shift();
 		const { url, browser, options } = next;
@@ -43,39 +35,15 @@ export async function runFullQueue( {
 		// Wait enough time between requests
 		// to give concurrency a chance to update.
 		// In selenium, this helps avoid undici connect timeout errors.
-		await new Promise( ( resolve ) =>
-			setTimeout(
-				resolve,
-				browserstack ? BROWSERSTACK_WAIT_TIME : SELENIUM_WAIT_TIME
-			)
-		);
-
-		const concurrency =
-			browserstack && !defaultConcurrency ?
-				await getAvailableSessions() :
-				defaultConcurrency || MAX_CONCURRENCY;
+		await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) );
 
 		if ( verbose ) {
-			console.log(
-				`\nConcurrency: ${ concurrency }. Tests remaining: ${ queue.length + 1 }.`
-			);
-		}
-
-		// If concurrency is 0, wait a bit and try again
-		if ( concurrency <= 0 ) {
-			if ( verbose ) {
-				console.log( "\nWaiting for available sessions..." );
-			}
-			queue.unshift( next );
-			await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
-			continue;
+			console.log( `\nTests remaining: ${ queue.length + 1 }.` );
 		}
 
 		let promise;
 		if ( browser.browser === "jsdom" ) {
 			promise = runJSDOM( url, options );
-		} else if ( browserstack ) {
-			promise = runWorker( url, browser, options );
 		} else {
 			promise = runSelenium( url, browser, options );
 		}

test/runner/selenium/runSelenium.js

@@ -1,4 +1,3 @@
-import chalk from "chalk";
 import createDriver from "./createDriver.js";
 
 export async function runSelenium(