Tests: align test runner with other repos

Close gh-2234

.gitignore

@@ -3,3 +3,4 @@ bower_components
 node_modules
 .sizecache.json
 package-lock.json
+local.log

package.json

@@ -56,6 +56,7 @@
 	},
 	"devDependencies": {
 		"body-parser": "1.20.2",
+		"browserstack-local": "1.5.5",
 		"commitplease": "3.2.0",
 		"diff": "5.2.0",
 		"eslint-config-jquery": "3.0.2",

tests/runner/.eslintrc.json

@@ -7,13 +7,9 @@
 		{
 			"files": ["**/*"],
 			"env": {
+				"es6": true,
 				"node": true
 			},
-			"globals": {
-				"fetch": false,
-				"Promise": false,
-				"require": false
-			},
 			"parserOptions": {
 				"ecmaVersion": 2022,
 				"sourceType": "module"
@@ -27,7 +23,8 @@
 			},
 			"globals": {
 				"QUnit": false,
-				"Symbol": false
+				"Symbol": false,
+				"require": false
 			},
 			"parserOptions": {
 				"ecmaVersion": 5,

tests/runner/browsers.js

@@ -1,4 +1,242 @@
-// This list is static, so no requests are required
-// in the command help menu.
+import chalk from "chalk";
+import { getBrowserString } from "./lib/getBrowserString.js";
+import {
+	createWorker,
+	deleteWorker,
+	getAvailableSessions
+} from "./browserstack/api.js";
+import createDriver from "./selenium/createDriver.js";
 
-export const browsers = [ "chrome", "ie", "firefox", "edge", "safari", "opera" ];
+const workers = Object.create( null );
+
+/**
+ * Keys are browser strings
+ * Structure of a worker:
+ * {
+ *   browser: object // The browser object
+ * 	 debug: boolean // Stops the worker from being cleaned up when finished
+ *   lastTouch: number // The last time a request was received
+ *   restarts: number // The number of times the worker has been restarted
+ *   options: object // The options to create the worker
+ *   url: string // The URL the worker is on
+ *   quit: function // A function to stop the worker
+ * }
+ */
+
+// Acknowledge the worker within the time limit.
+// BrowserStack can take much longer spinning up
+// some browsers, such as iOS 15 Safari.
+const ACKNOWLEDGE_INTERVAL = 1000;
+const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
+
+const MAX_WORKER_RESTARTS = 5;
+
+// No report after the time limit
+// should refresh the worker
+const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
+
+const WORKER_WAIT_TIME = 30000;
+
+// Limit concurrency to 8 by default in selenium
+const MAX_SELENIUM_CONCURRENCY = 8;
+
+export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
+	if ( restarts > MAX_WORKER_RESTARTS ) {
+		throw new Error(
+			`Reached the maximum number of restarts for ${ chalk.yellow(
+				getBrowserString( browser )
+			) }`
+		);
+	}
+	const { browserstack, debug, headless, runId, tunnelId, verbose } = options;
+	while ( await maxWorkersReached( options ) ) {
+		if ( verbose ) {
+			console.log( "\nWaiting for available sessions..." );
+		}
+		await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
+	}
+
+	const fullBrowser = getBrowserString( browser );
+
+	let worker;
+
+	if ( browserstack ) {
+		worker = await createWorker( {
+			...browser,
+			url: encodeURI( url ),
+			project: "jquery",
+			build: `Run ${ runId }`,
+
+			// This is the maximum timeout allowed
+			// by BrowserStack. We do this because
+			// we control the timeout in the runner.
+			// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
+			timeout: 1800,
+
+			// Not documented in the API docs,
+			// but required to make local testing work.
+			// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
+			"browserstack.local": true,
+			"browserstack.localIdentifier": tunnelId
+		} );
+		worker.quit = () => deleteWorker( worker.id );
+	} else {
+		const driver = await createDriver( {
+			browserName: browser.browser,
+			headless,
+			url,
+			verbose
+		} );
+		worker = {
+			quit: () => driver.quit()
+		};
+	}
+
+	worker.debug = !!debug;
+	worker.url = url;
+	worker.browser = browser;
+	worker.restarts = restarts;
+	worker.options = options;
+	touchBrowser( browser );
+	workers[ fullBrowser ] = worker;
+
+	// Wait for the worker to show up in the list
+	// before returning it.
+	return ensureAcknowledged( worker );
+}
+
+export function touchBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.lastTouch = Date.now();
+	}
+}
+
+export async function setBrowserWorkerUrl( browser, url ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		worker.url = url;
+	}
+}
+
+export async function restartBrowser( browser ) {
+	const fullBrowser = getBrowserString( browser );
+	const worker = workers[ fullBrowser ];
+	if ( worker ) {
+		await restartWorker( worker );
+	}
+}
+
+/**
+ * Checks that all browsers have received
+ * a response in the given amount of time.
+ * If not, the worker is restarted.
+ */
+export async function checkLastTouches() {
+	for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
+		if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
+			const options = worker.options;
+			if ( options.verbose ) {
+				console.log(
+					`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
+						RUN_WORKER_TIMEOUT / 1000 / 60
+					}min.`
+				);
+			}
+			await restartWorker( worker );
+		}
+	}
+}
+
+export async function cleanupAllBrowsers( { verbose } ) {
+	const workersRemaining = Object.values( workers );
+	const numRemaining = workersRemaining.length;
+	if ( numRemaining ) {
+		try {
+			await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
+			if ( verbose ) {
+				console.log(
+					`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+				);
+			}
+		} catch ( error ) {
+
+			// Log the error, but do not consider the test run failed
+			console.error( error );
+		}
+	}
+}
+
+async function maxWorkersReached( {
+	browserstack,
+	concurrency = MAX_SELENIUM_CONCURRENCY
+} ) {
+	if ( browserstack ) {
+		return ( await getAvailableSessions() ) <= 0;
+	}
+	return workers.length >= concurrency;
+}
+
+async function waitForAck( worker, { fullBrowser, verbose } ) {
+	delete worker.lastTouch;
+	return new Promise( ( resolve, reject ) => {
+		const interval = setInterval( () => {
+			if ( worker.lastTouch ) {
+				if ( verbose ) {
+					console.log( `\n${ fullBrowser } acknowledged.` );
+				}
+				clearTimeout( timeout );
+				clearInterval( interval );
+				resolve();
+			}
+		}, ACKNOWLEDGE_INTERVAL );
+
+		const timeout = setTimeout( () => {
+			clearInterval( interval );
+			reject(
+				new Error(
+					`${ fullBrowser } not acknowledged after ${
+						ACKNOWLEDGE_TIMEOUT / 1000 / 60
+					}min.`
+				)
+			);
+		}, ACKNOWLEDGE_TIMEOUT );
+	} );
+}
+
+async function ensureAcknowledged( worker ) {
+	const fullBrowser = getBrowserString( worker.browser );
+	const verbose = worker.options.verbose;
+	try {
+		await waitForAck( worker, { fullBrowser, verbose } );
+		return worker;
+	} catch ( error ) {
+		console.error( error.message );
+		await restartWorker( worker );
+	}
+}
+
+async function cleanupWorker( worker, { verbose } ) {
+	for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
+		if ( w === worker ) {
+			delete workers[ fullBrowser ];
+			await worker.quit();
+			if ( verbose ) {
+				console.log( `\nStopped ${ fullBrowser }.` );
+			}
+			return;
+		}
+	}
+}
+
+async function restartWorker( worker ) {
+	await cleanupWorker( worker, worker.options );
+	await createBrowserWorker(
+		worker.url,
+		worker.browser,
+		worker.options,
+		worker.restarts + 1
+	);
+}

tests/runner/browserstack/api.js

@@ -0,0 +1,332 @@
+/**
+ * Browserstack API is documented at
+ * https://github.com/browserstack/api
+ */
+
+import { createAuthHeader } from "./createAuthHeader.js";
+
+const browserstackApi = "https://api.browserstack.com";
+const apiVersion = 5;
+
+const username = process.env.BROWSERSTACK_USERNAME;
+const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
+
+// iOS has null for version numbers,
+// and we do not need a similar check for OS versions.
+const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
+const rlatest = /^latest-(\d+)$/;
+
+const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
+
+async function fetchAPI( path, options = {}, versioned = true ) {
+	if ( !username || !accessKey ) {
+		throw new Error(
+			"BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
+		);
+	}
+	const init = {
+		method: "GET",
+		...options,
+		headers: {
+			authorization: createAuthHeader( username, accessKey ),
+			accept: "application/json",
+			"content-type": "application/json",
+			...options.headers
+		}
+	};
+	const response = await fetch(
+		`${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
+		init
+	);
+	if ( !response.ok ) {
+		console.log(
+			`\n${ init.method } ${ path }`,
+			response.status,
+			response.statusText
+		);
+		throw new Error( `Error fetching ${ path }` );
+	}
+	return response.json();
+}
+
+/**
+ * =============================
+ * Browsers API
+ * =============================
+ */
+
+function compareVersionNumbers( a, b ) {
+	if ( a != null && b == null ) {
+		return -1;
+	}
+	if ( a == null && b != null ) {
+		return 1;
+	}
+	if ( a == null && b == null ) {
+		return 0;
+	}
+	const aParts = a.replace( rnonDigits, "" ).split( "." );
+	const bParts = b.replace( rnonDigits, "" ).split( "." );
+
+	if ( aParts.length > bParts.length ) {
+		return -1;
+	}
+	if ( aParts.length < bParts.length ) {
+		return 1;
+	}
+
+	for ( let i = 0; i < aParts.length; i++ ) {
+		const aPart = Number( aParts[ i ] );
+		const bPart = Number( bParts[ i ] );
+		if ( aPart < bPart ) {
+			return -1;
+		}
+		if ( aPart > bPart ) {
+			return 1;
+		}
+	}
+
+	if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) {
+		return -1;
+	}
+	if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) {
+		return 1;
+	}
+
+	return 0;
+}
+
+function sortBrowsers( a, b ) {
+	if ( a.browser < b.browser ) {
+		return -1;
+	}
+	if ( a.browser > b.browser ) {
+		return 1;
+	}
+	const browserComparison = compareVersionNumbers(
+		a.browser_version,
+		b.browser_version
+	);
+	if ( browserComparison ) {
+		return browserComparison;
+	}
+	if ( a.os < b.os ) {
+		return -1;
+	}
+	if ( a.os > b.os ) {
+		return 1;
+	}
+	const osComparison = compareVersionNumbers( a.os_version, b.os_version );
+	if ( osComparison ) {
+		return osComparison;
+	}
+	const deviceComparison = compareVersionNumbers( a.device, b.device );
+	if ( deviceComparison ) {
+		return deviceComparison;
+	}
+	return 0;
+}
+
+export async function getBrowsers( { flat = false } = {} ) {
+	const query = new URLSearchParams();
+	if ( flat ) {
+		query.append( "flat", true );
+	}
+	const browsers = await fetchAPI( `/browsers?${ query }` );
+	return browsers.sort( sortBrowsers );
+}
+
+function matchVersion( browserVersion, version ) {
+	if ( !version ) {
+		return false;
+	}
+	const regex = new RegExp(
+		`^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
+		"i"
+	);
+	return regex.test( browserVersion );
+}
+
+export async function filterBrowsers( filter ) {
+	const browsers = await getBrowsers( { flat: true } );
+	if ( !filter ) {
+		return browsers;
+	}
+	const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
+	const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
+	const filterOs = ( filter.os ?? "" ).toLowerCase();
+	const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
+	const filterDevice = ( filter.device ?? "" ).toLowerCase();
+
+	const filteredWithoutVersion = browsers.filter( ( browser ) => {
+		return (
+			( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
+			( !filterOs || filterOs === browser.os.toLowerCase() ) &&
+			( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) &&
+			( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
+		);
+	} );
+
+	if ( !filterVersion ) {
+		return filteredWithoutVersion;
+	}
+
+	if ( filterVersion.startsWith( "latest" ) ) {
+		const groupedByName = filteredWithoutVersion
+			.filter( ( b ) => rfinalVersion.test( b.browser_version ) )
+			.reduce( ( acc, browser ) => {
+				acc[ browser.browser ] = acc[ browser.browser ] ?? [];
+				acc[ browser.browser ].push( browser );
+				return acc;
+			}, Object.create( null ) );
+
+		const filtered = [];
+		for ( const group of Object.values( groupedByName ) ) {
+			const latest = group[ group.length - 1 ];
+
+			// Mobile devices do not have browser version.
+			// Skip the version check for these,
+			// but include the latest in the list if it made it
+			// through filtering.
+			if ( !latest.browser_version ) {
+
+				// Do not include in the list for latest-n.
+				if ( filterVersion === "latest" ) {
+					filtered.push( latest );
+				}
+				continue;
+			}
+
+			// Get the latest version and subtract the number from the filter,
+			// ignoring any patch versions, which may differ between major versions.
+			const num = rlatest.exec( filterVersion );
+			const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 );
+			const match = group.findLast( ( browser ) => {
+				return matchVersion( browser.browser_version, version.toString() );
+			} );
+			if ( match ) {
+				filtered.push( match );
+			}
+		}
+		return filtered;
+	}
+
+	return filteredWithoutVersion.filter( ( browser ) => {
+		return matchVersion( browser.browser_version, filterVersion );
+	} );
+}
+
+export async function listBrowsers( filter ) {
+	const browsers = await filterBrowsers( filter );
+	console.log( "Available browsers:" );
+	for ( const browser of browsers ) {
+		let message = `    ${ browser.browser }_`;
+		if ( browser.device ) {
+			message += `:${ browser.device }_`;
+		} else {
+			message += `${ browser.browser_version }_`;
+		}
+		message += `${ browser.os }_${ browser.os_version }`;
+		console.log( message );
+	}
+}
+
+export async function getLatestBrowser( filter ) {
+	if ( !filter.browser_version ) {
+		filter.browser_version = "latest";
+	}
+	const browsers = await filterBrowsers( filter );
+	return browsers[ browsers.length - 1 ];
+}
+
+/**
+ * =============================
+ * Workers API
+ * =============================
+ */
+
+/**
+ * A browser object may only have one of `browser` or `device` set;
+ * which property is set will depend on `os`.
+ *
+ * `options`: is an object with the following properties:
+ *   `os`: The operating system.
+ *   `os_version`: The operating system version.
+ *   `browser`: The browser name.
+ *   `browser_version`: The browser version.
+ *   `device`: The device name.
+ *   `url` (optional): Which URL to navigate to upon creation.
+ *   `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
+ *   `name` (optional): Provide a name for the worker.
+ *   `build` (optional): Group workers into a build.
+ *   `project` (optional): Provide the project the worker belongs to.
+ *   `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
+ *   `browserstack.local` (optional): Set to `true` to mark as local testing.
+ *   `browserstack.video` (optional): Set to `false` to disable video recording.
+ *   `browserstack.localIdentifier` (optional): ID of the local tunnel.
+ */
+export function createWorker( options ) {
+	return fetchAPI( "/worker", {
+		method: "POST",
+		body: JSON.stringify( options )
+	} );
+}
+
+/**
+ * Returns a worker object, if one exists, with the following properties:
+ *   `id`: The worker id.
+ *   `status`: A string representing the current status of the worker.
+ *     Possible statuses: `"running"`, `"queue"`.
+ */
+export function getWorker( id ) {
+	return fetchAPI( `/worker/${ id }` );
+}
+
+export async function deleteWorker( id ) {
+	return fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
+}
+
+export function getWorkers() {
+	return fetchAPI( "/workers" );
+}
+
+/**
+ * Stop all workers
+ */
+export async function stopWorkers() {
+	const workers = await getWorkers();
+
+	// Run each request on its own
+	// to avoid connect timeout errors.
+	console.log( `${ workers.length } workers running...` );
+	for ( const worker of workers ) {
+		try {
+			await deleteWorker( worker.id );
+		} catch ( error ) {
+
+			// Log the error, but continue trying to remove workers.
+			console.error( error );
+		}
+	}
+	console.log( "All workers stopped." );
+}
+
+/**
+ * =============================
+ * Plan API
+ * =============================
+ */
+
+export function getPlan() {
+	return fetchAPI( "/automate/plan.json", {}, false );
+}
+
+export async function getAvailableSessions() {
+	try {
+		const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
+		return plan.parallel_sessions_max_allowed - workers.length;
+	} catch ( error ) {
+		console.error( error );
+		return 0;
+	}
+}

tests/runner/browserstack/buildBrowserFromString.js

@@ -0,0 +1,20 @@
+export function buildBrowserFromString( str ) {
+	const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
+
+	// If the version starts with a colon, it's a device
+	if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
+		return {
+			browser,
+			device: versionOrDevice.slice( 1 ),
+			os,
+			os_version: osVersion
+		};
+	}
+
+	return {
+		browser,
+		browser_version: versionOrDevice,
+		os,
+		os_version: osVersion
+	};
+}

tests/runner/browserstack/createAuthHeader.js

@@ -0,0 +1,7 @@
+const textEncoder = new TextEncoder();
+
+export function createAuthHeader( username, accessKey ) {
+	const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
+	const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
+	return `Basic ${ base64 }`;
+}

tests/runner/browserstack/local.js

@@ -0,0 +1,34 @@
+import browserstackLocal from "browserstack-local";
+
+export async function localTunnel( localIdentifier, opts = {} ) {
+	const tunnel = new browserstackLocal.Local();
+
+	return new Promise( ( resolve, reject ) => {
+
+		// https://www.browserstack.com/docs/local-testing/binary-params
+		tunnel.start(
+			{
+				"enable-logging-for-api": "",
+				localIdentifier,
+				...opts
+			},
+			async( error ) => {
+				if ( error || !tunnel.isRunning() ) {
+					return reject( error );
+				}
+				resolve( {
+					stop: function stopTunnel() {
+						return new Promise( ( resolve, reject ) => {
+							tunnel.stop( ( error ) => {
+								if ( error ) {
+									return reject( error );
+								}
+								resolve();
+							} );
+						} );
+					}
+				} );
+			}
+		);
+	} );
+}

tests/runner/command.js

@@ -1,8 +1,10 @@
 import yargs from "yargs/yargs";
-import { browsers } from "./browsers.js";
-import { suites } from "./suites.js";
+import { browsers } from "./flags/browsers.js";
+import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { jquery } from "./flags/jquery.js";
+import { suites } from "./flags/suites.js";
 import { run } from "./run.js";
-import { jquery } from "./jquery.js";
 
 const argv = yargs( process.argv.slice( 2 ) )
 	.version( false )
@@ -40,16 +42,25 @@ const argv = yargs( process.argv.slice( 2 ) )
 		type: "array",
 		choices: browsers,
 		description:
-			"Run tests in a specific browser.\n" +
-			"Pass multiple browsers by repeating the option.",
+			"Run tests in a specific browser." +
+			"Pass multiple browsers by repeating the option." +
+			"If using BrowserStack, specify browsers using --browserstack.",
 		default: [ "chrome" ]
 	} )
 	.option( "headless", {
 		alias: "h",
 		type: "boolean",
 		description:
-			"Run tests in headless mode. Cannot be used with --debug.",
-		conflicts: [ "debug" ]
+			"Run tests in headless mode. Cannot be used with --debug or --browserstack.",
+		conflicts: [ "debug", "browserstack" ]
+	} )
+	.option( "concurrency", {
+		alias: "c",
+		type: "number",
+		description:
+			"Run tests in parallel in multiple browsers. " +
+			"Defaults to 8 in normal mode. In browserstack mode, " +
+			"defaults to the maximum available under your BrowserStack plan."
 	} )
 	.option( "debug", {
 		alias: "d",
@@ -61,18 +72,69 @@ const argv = yargs( process.argv.slice( 2 ) )
 	.option( "retries", {
 		alias: "r",
 		type: "number",
-		description: "Number of times to retry failed tests."
+		description: "Number of times to retry failed tests by refreshing the URL."
 	} )
-	.option( "concurrency", {
-		alias: "c",
+	.option( "hard-retries", {
 		type: "number",
-		description: "Run tests in parallel in multiple browsers. Defaults to 8."
+		description:
+			"Number of times to retry failed tests by restarting the worker. " +
+			"This is in addition to the normal retries " +
+			"and are only used when the normal retries are exhausted."
 	} )
 	.option( "verbose", {
 		alias: "v",
 		type: "boolean",
 		description: "Log additional information."
 	} )
+	.option( "browserstack", {
+		type: "array",
+		description:
+			"Run tests in BrowserStack.\n" +
+			"Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
+			"The value can be empty for the default configuration, or a string in the format of\n" +
+			"\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" +
+			"Pass multiple browsers by repeating the option.\n" +
+			"The --browser option is ignored when --browserstack has a value.\n" +
+			"Otherwise, the --browser option will be used, " +
+			"with the latest version/device for that browser, on a matching OS."
+	} )
+	.option( "run-id", {
+		type: "string",
+		description: "A unique identifier for the run in BrowserStack."
+	} )
+	.option( "list-browsers", {
+		type: "string",
+		description:
+			"List available BrowserStack browsers and exit.\n" +
+			"Leave blank to view all browsers or pass " +
+			"\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " +
+			"separated by an underscore to filter the list (any can be omitted).\n" +
+			"\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" +
+			"\"latest-n\" can be used to find the nth latest browser version.\n" +
+			"Use a colon to indicate a device.\n" +
+			"Examples: \"chrome__windows_10\", \"safari_latest\", " +
+			"\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
+			"Use quotes if spaces are necessary."
+	} )
+	.option( "stop-workers", {
+		type: "boolean",
+		description:
+			"WARNING: This will stop all BrowserStack workers that may exist and exit," +
+			"including any workers running from other projects.\n" +
+			"This can be used as a failsafe when there are too many stray workers."
+	} )
+	.option( "browserstack-plan", {
+		type: "boolean",
+		description: "Show BrowserStack plan information and exit."
+	} )
 	.help().argv;
 
+if ( typeof argv.listBrowsers === "string" ) {
+	listBrowsers( buildBrowserFromString( argv.listBrowsers ) );
+} else if ( argv.stopWorkers ) {
+	stopWorkers();
+} else if ( argv.browserstackPlan ) {
+	console.log( await getPlan() );
+} else {
 	run( argv );
+}

tests/runner/createTestServer.js

@@ -1,7 +1,7 @@
+import { readFile } from "node:fs/promises";
 import bodyParser from "body-parser";
 import express from "express";
 import bodyParserErrorHandler from "express-body-parser-error-handler";
-import { readFile } from "node:fs/promises";
 
 export async function createTestServer( report ) {
 	const app = express();

tests/runner/flags/browsers.js

@@ -0,0 +1,24 @@
+// This list is static, so no requests are required
+// in the command help menu.
+
+import { getBrowsers } from "../browserstack/api.js";
+
+export const browsers = [
+	"chrome",
+	"ie",
+	"firefox",
+	"edge",
+	"safari",
+	"opera",
+	"yandex",
+	"IE Mobile",
+	"Android Browser",
+	"Mobile Safari"
+];
+
+// A function that can be used to update the above list.
+export async function getAvailableBrowsers() {
+	const browsers = await getBrowsers( { flat: true } );
+	const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ];
+	return available;
+}

tests/runner/lib/buildTestUrl.js

@@ -1,4 +1,4 @@
-export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) {
+export function buildTestUrl( suite, { browserstack, jquery, migrate, port, reportId } ) {
 	if ( !port ) {
 		throw new Error( "No port specified." );
 	}
@@ -17,5 +17,8 @@ export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) {
 		query.append( "reportId", reportId );
 	}
 
-	return `http://localhost:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
+	// BrowserStack supplies a custom domain for local testing,
+	// which is especially necessary for iOS testing.
+	const host = browserstack ? "bs-local.com" : "localhost";
+	return `http://${ host }:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
 }

tests/runner/lib/getBrowserString.js

@@ -3,7 +3,6 @@ const browserMap = {
 	edge: "Edge",
 	firefox: "Firefox",
 	ie: "IE",
-	jsdom: "JSDOM",
 	opera: "Opera",
 	safari: "Safari"
 };

tests/runner/queue.js

@@ -1,8 +1,9 @@
 import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
+import { getBrowserString } from "./lib/getBrowserString.js";
 import {
 	checkLastTouches,
 	createBrowserWorker,
+	restartBrowser,
 	setBrowserWorkerUrl
 } from "./browsers.js";
 
@@ -44,23 +45,44 @@ export function retryTest( reportId, maxRetries ) {
 		test.retries++;
 		if ( test.retries <= maxRetries ) {
 			console.log(
-				`\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${
-					test.retries
-				}`
+				`\nRetrying test ${ reportId } for ${ chalk.yellow(
+					test.options.suite
+				) }...${ test.retries }`
 			);
 			return test;
 		}
 	}
 }
 
+export async function hardRetryTest( reportId, maxHardRetries ) {
+	if ( !maxHardRetries ) {
+		return false;
+	}
+	const test = queue.find( ( test ) => test.id === reportId );
+	if ( test ) {
+		test.hardRetries++;
+		if ( test.hardRetries <= maxHardRetries ) {
+			console.log(
+				`\nHard retrying test ${ reportId } for ${ chalk.yellow(
+					test.options.suite
+				) }...${ test.hardRetries }`
+			);
+			await restartBrowser( test.browser );
+			return true;
+		}
+	}
+	return false;
+}
+
 export function addRun( url, browser, options ) {
 	queue.push( {
 		browser,
 		fullBrowser: getBrowserString( browser ),
+		hardRetries: 0,
 		id: options.reportId,
-		retries: 0,
 		url,
 		options,
+		retries: 0,
 		running: false
 	} );
 }

tests/runner/reporter.js

@@ -1,7 +1,7 @@
 import chalk from "chalk";
+import * as Diff from "diff";
 import { getBrowserString } from "./lib/getBrowserString.js";
 import { prettyMs } from "./lib/prettyMs.js";
-import * as Diff from "diff";
 
 function serializeForDiff( value ) {
 

tests/runner/run.js

@@ -1,13 +1,22 @@
 import chalk from "chalk";
 import { asyncExitHook, gracefulExit } from "exit-hook";
+import { getLatestBrowser } from "./browserstack/api.js";
+import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
+import { localTunnel } from "./browserstack/local.js";
 import { reportEnd, reportTest } from "./reporter.js";
 import { createTestServer } from "./createTestServer.js";
 import { buildTestUrl } from "./lib/buildTestUrl.js";
 import { generateHash } from "./lib/generateHash.js";
 import { getBrowserString } from "./lib/getBrowserString.js";
-import { suites as allSuites } from "./suites.js";
-import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
-import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";
+import { suites as allSuites } from "./flags/suites.js";
+import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
+import {
+	addRun,
+	getNextBrowserTest,
+	hardRetryTest,
+	retryTest,
+	runAll
+} from "./queue.js";
 
 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
 
@@ -16,12 +25,15 @@ const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
  */
 export async function run( {
 	browser: browserNames = [],
+	browserstack,
 	concurrency,
 	debug,
+	hardRetries,
 	headless,
 	jquery: jquerys = [],
 	migrate,
 	retries = 0,
+	runId,
 	suite: suites = [],
 	verbose
 } ) {
@@ -40,11 +52,25 @@ export async function run( {
 		);
 	}
 
+	if ( verbose ) {
+		console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
+	}
+
 	const errorMessages = [];
 	const pendingErrors = {};
 
 	// Convert browser names to browser objects
 	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
+	const tunnelId = generateHash(
+		`${ Date.now() }-${ suites.join( ":" ) }-${ ( browserstack || [] )
+			.concat( browserNames )
+			.join( ":" ) }`
+	);
+
+	// A unique identifier for this run
+	if ( !runId ) {
+		runId = tunnelId;
+	}
 
 	// Create the test app and
 	// hook it up to the reporter
@@ -96,6 +122,10 @@ export async function run( {
 						return retry;
 					}
 
+					// Return early if hardRetryTest returns true
+					if ( await hardRetryTest( reportId, hardRetries ) ) {
+						return;
+					}
 					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
 				}
 
@@ -143,22 +173,84 @@ export async function run( {
 		} );
 	}
 
+	async function cleanup() {
+		console.log( "Cleaning up..." );
+
+		await cleanupAllBrowsers( { verbose } );
+
+		if ( tunnel ) {
+			await tunnel.stop();
+			if ( verbose ) {
+				console.log( "Stopped BrowserStackLocal." );
+			}
+		}
+	}
+
 	asyncExitHook(
 		async() => {
-			await cleanupAllBrowsers( { verbose } );
+			await cleanup();
 			await stopServer();
 		},
 		{ wait: EXIT_HOOK_WAIT_TIMEOUT }
 	);
 
+	// Start up BrowserStackLocal
+	let tunnel;
+	if ( browserstack ) {
+		if ( headless ) {
+			console.warn(
+				chalk.italic(
+					"BrowserStack does not support headless mode. Running in normal mode."
+				)
+			);
+			headless = false;
+		}
+
+		// Convert browserstack to browser objects.
+		// If browserstack is an empty array, fall back
+		// to the browsers array.
+		if ( browserstack.length ) {
+			browsers = browserstack.map( ( b ) => {
+				if ( !b ) {
+					return browsers[ 0 ];
+				}
+				return buildBrowserFromString( b );
+			} );
+		}
+
+		// Fill out browser defaults
+		browsers = await Promise.all(
+			browsers.map( async( browser ) => {
+
+				// Avoid undici connect timeout errors
+				await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
+
+				const latestMatch = await getLatestBrowser( browser );
+				if ( !latestMatch ) {
+					console.error(
+						chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
+					);
+					gracefulExit( 1 );
+				}
+				return latestMatch;
+			} )
+		);
+
+		tunnel = await localTunnel( tunnelId );
+		if ( verbose ) {
+			console.log( "Started BrowserStackLocal." );
+		}
+	}
+
 	function queueRuns( suite, browser ) {
 		const fullBrowser = getBrowserString( browser, headless );
 
 		for ( const jquery of jquerys ) {
-			const reportId = generateHash( `${ suite } ${ fullBrowser }` );
+			const reportId = generateHash( `${ suite } ${ jquery } ${ fullBrowser }` );
 			reports[ reportId ] = { browser, headless, jquery, migrate, suite };
 
 			const url = buildTestUrl( suite, {
+				browserstack,
 				jquery,
 				migrate,
 				port,
@@ -166,12 +258,16 @@ export async function run( {
 			} );
 
 			const options = {
+				browserstack,
+				concurrency,
 				debug,
 				headless,
 				jquery,
 				migrate,
 				reportId,
+				runId,
 				suite,
+				tunnelId,
 				verbose
 			};
 
@@ -181,12 +277,13 @@ export async function run( {
 
 	for ( const browser of browsers ) {
 		for ( const suite of suites ) {
-			queueRuns( suite, browser );
+			queueRuns( [ suite ], browser );
 		}
 	}
 
 	try {
-		await runAll( { concurrency, verbose } );
+		console.log( `Starting Run ${ runId }...` );
+		await runAll();
 	} catch ( error ) {
 		console.error( error );
 		if ( !debug ) {
@@ -213,7 +310,7 @@ export async function run( {
 			}
 			console.log( chalk.green( "All tests passed!" ) );
 
-			if ( !debug ) {
+			if ( !debug || browserstack ) {
 				gracefulExit( 0 );
 			}
 		} else {
@@ -224,7 +321,14 @@ export async function run( {
 
 			if ( debug ) {
 				console.log();
+				if ( browserstack ) {
+					console.log( "Leaving browsers with failures open for debugging." );
+					console.log(
+						"View running sessions at https://automate.browserstack.com/dashboard/v2/"
+					);
+				} else {
 					console.log( "Leaving browsers open for debugging." );
+				}
 				console.log( "Press Ctrl+C to exit." );
 			} else {
 				gracefulExit( 1 );

tests/runner/selenium/browsers.js

@@ -1,200 +0,0 @@
-import chalk from "chalk";
-import { getBrowserString } from "../lib/getBrowserString.js";
-import createDriver from "./createDriver.js";
-
-const workers = Object.create( null );
-
-/**
- * Keys are browser strings
- * Structure of a worker:
- * {
- * 	 debug: boolean, // Stops the worker from being cleaned up when finished
- *   id: string,
- *   lastTouch: number, // The last time a request was received
- *   url: string,
- *   browser: object, // The browser object
- *   options: object // The options to create the worker
- * }
- */
-
-// Acknowledge the worker within the time limit.
-const ACKNOWLEDGE_INTERVAL = 1000;
-const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 1;
-
-const MAX_WORKER_RESTARTS = 5;
-
-// No report after the time limit
-// should refresh the worker
-const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
-
-const WORKER_WAIT_TIME = 30000;
-
-// Limit concurrency to 8 by default in selenium
-const MAX_CONCURRENCY = 8;
-
-export function touchBrowser( browser ) {
-	const fullBrowser = getBrowserString( browser );
-	const worker = workers[ fullBrowser ];
-	if ( worker ) {
-		worker.lastTouch = Date.now();
-	}
-}
-
-async function waitForAck( worker, { fullBrowser, verbose } ) {
-	delete worker.lastTouch;
-	return new Promise( ( resolve, reject ) => {
-		const interval = setInterval( () => {
-			if ( worker.lastTouch ) {
-				if ( verbose ) {
-					console.log( `\n${ fullBrowser } acknowledged.` );
-				}
-				clearTimeout( timeout );
-				clearInterval( interval );
-				resolve();
-			}
-		}, ACKNOWLEDGE_INTERVAL );
-
-		const timeout = setTimeout( () => {
-			clearInterval( interval );
-			reject(
-				new Error(
-					`${ fullBrowser } not acknowledged after ${
-						ACKNOWLEDGE_TIMEOUT / 1000 / 60
-					}min.`
-				)
-			);
-		}, ACKNOWLEDGE_TIMEOUT );
-	} );
-}
-
-async function restartWorker( worker ) {
-	await cleanupWorker( worker, worker.options );
-	await createBrowserWorker(
-		worker.url,
-		worker.browser,
-		worker.options,
-		worker.restarts + 1
-	);
-}
-
-async function ensureAcknowledged( worker ) {
-	const fullBrowser = getBrowserString( worker.browser );
-	const verbose = worker.options.verbose;
-	try {
-		await waitForAck( worker, { fullBrowser, verbose } );
-		return worker;
-	} catch ( error ) {
-		console.error( error.message );
-		await restartWorker( worker );
-	}
-}
-
-export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
-	if ( restarts > MAX_WORKER_RESTARTS ) {
-		throw new Error(
-			`Reached the maximum number of restarts for ${ chalk.yellow(
-				getBrowserString( browser )
-			) }`
-		);
-	}
-	const { concurrency = MAX_CONCURRENCY, debug, headless, verbose } = options;
-	while ( workers.length >= concurrency ) {
-		if ( verbose ) {
-			console.log( "\nWaiting for available sessions..." );
-		}
-		await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
-	}
-
-	const fullBrowser = getBrowserString( browser );
-
-	const driver = await createDriver( {
-		browserName: browser.browser,
-		headless,
-		url,
-		verbose
-	} );
-
-	const worker = {
-		debug: !!debug,
-		driver,
-		url,
-		browser,
-		restarts,
-		options
-	};
-
-	worker.debug = !!debug;
-	worker.url = url;
-	worker.browser = browser;
-	worker.restarts = restarts;
-	worker.options = options;
-	touchBrowser( browser );
-	workers[ fullBrowser ] = worker;
-
-	// Wait for the worker to show up in the list
-	// before returning it.
-	return ensureAcknowledged( worker );
-}
-
-export async function setBrowserWorkerUrl( browser, url ) {
-	const fullBrowser = getBrowserString( browser );
-	const worker = workers[ fullBrowser ];
-	if ( worker ) {
-		worker.url = url;
-	}
-}
-
-/**
- * Checks that all browsers have received
- * a response in the given amount of time.
- * If not, the worker is restarted.
- */
-export async function checkLastTouches() {
-	for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
-		if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
-			const options = worker.options;
-			if ( options.verbose ) {
-				console.log(
-					`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
-						RUN_WORKER_TIMEOUT / 1000 / 60
-					}min.`
-				);
-			}
-			await restartWorker( worker );
-		}
-	}
-}
-
-export async function cleanupWorker( worker, { verbose } ) {
-	for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
-		if ( w === worker ) {
-			delete workers[ fullBrowser ];
-			await w.driver.quit();
-			if ( verbose ) {
-				console.log( `\nStopped ${ fullBrowser }.` );
-			}
-			return;
-		}
-	}
-}
-
-export async function cleanupAllBrowsers( { verbose } ) {
-	const workersRemaining = Object.values( workers );
-	const numRemaining = workersRemaining.length;
-	if ( numRemaining ) {
-		try {
-			await Promise.all(
-				workersRemaining.map( ( worker ) => worker.driver.quit() )
-			);
-			if ( verbose ) {
-				console.log(
-					`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
-				);
-			}
-		} catch ( error ) {
-
-			// Log the error, but do not consider the test run failed
-			console.error( error );
-		}
-	}
-}