jquery-ui/tests/runner/run.js

import chalk from "chalk";
import { asyncExitHook, gracefulExit } from "exit-hook";
import { reportEnd, reportTest } from "./reporter.js";
import { createTestServer } from "./createTestServer.js";
import { buildTestUrl } from "./lib/buildTestUrl.js";
import { generateHash } from "./lib/generateHash.js";
import { getBrowserString } from "./lib/getBrowserString.js";
import { suites as allSuites } from "./suites.js";
import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js";
import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js";

const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;

/**
 * Run test suites in parallel in different browser instances.
 */
export async function run( {
	browser: browserNames = [],
	concurrency,
	debug,
	headless,
	jquery: jquerys = [],
	migrate,
	retries = 0,
	suite: suites = [],
	verbose
} ) {
	if ( !browserNames.length ) {
		browserNames = [ "chrome" ];
	}
	if ( !suites.length ) {
		suites = allSuites;
	}
	if ( !jquerys.length ) {
		jquerys = [ "3.7.1" ];
	}
	if ( headless && debug ) {
		throw new Error(
			"Cannot run in headless mode and debug mode at the same time."
		);
	}

	const errorMessages = [];
	const pendingErrors = {};

	// Convert browser names to browser objects
	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );

	// Create the test app and
	// hook it up to the reporter
	const reports = Object.create( null );
	const app = await createTestServer( async( message ) => {
		switch ( message.type ) {
			case "testEnd": {
				const reportId = message.id;
				const report = reports[ reportId ];
				touchBrowser( report.browser );
				const errors = reportTest( message.data, reportId, report );
				pendingErrors[ reportId ] ??= Object.create( null );
				if ( errors ) {
					pendingErrors[ reportId ][ message.data.name ] = errors;
				} else {
					const existing = pendingErrors[ reportId ][ message.data.name ];

					// Show a message for flakey tests
					if ( existing ) {
						console.log();
						console.warn(
							chalk.italic(
								chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
							)
						);
						console.log();
						delete pendingErrors[ reportId ][ message.data.name ];
					}
				}
				break;
			}
			case "runEnd": {
				const reportId = message.id;
				const report = reports[ reportId ];
				touchBrowser( report.browser );
				const { failed, total } = reportEnd(
					message.data,
					message.id,
					reports[ reportId ]
				);
				report.total = total;

				// Handle failure
				if ( failed ) {
					const retry = retryTest( reportId, retries );

					// Retry if retryTest returns a test
					if ( retry ) {
						return retry;
					}

					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
				}

				// Run the next test
				return getNextBrowserTest( reportId );
			}
			case "ack": {
				const report = reports[ message.id ];
				touchBrowser( report.browser );
				break;
			}
			default:
				console.warn( "Received unknown message type:", message.type );
		}
	} );

	// Start up local test server
	let server;
	let port;
	await new Promise( ( resolve ) => {

		// Pass 0 to choose a random, unused port
		server = app.listen( 0, () => {
			port = server.address().port;
			resolve();
		} );
	} );

	if ( !server || !port ) {
		throw new Error( "Server not started." );
	}

	if ( verbose ) {
		console.log( `Server started on port ${ port }.` );
	}

	function stopServer() {
		return new Promise( ( resolve ) => {
			server.close( () => {
				if ( verbose ) {
					console.log( "Server stopped." );
				}
				resolve();
			} );
		} );
	}

	asyncExitHook(
		async() => {
			await cleanupAllBrowsers( { verbose } );
			await stopServer();
		},
		{ wait: EXIT_HOOK_WAIT_TIMEOUT }
	);

	function queueRuns( suite, browser ) {
		const fullBrowser = getBrowserString( browser, headless );

		for ( const jquery of jquerys ) {
			const reportId = generateHash( `${ suite } ${ fullBrowser }` );
			reports[ reportId ] = { browser, headless, jquery, migrate, suite };

			const url = buildTestUrl( suite, {
				jquery,
				migrate,
				port,
				reportId
			} );

			const options = {
				debug,
				headless,
				jquery,
				migrate,
				reportId,
				suite,
				verbose
			};

			addRun( url, browser, options );
		}
	}

	for ( const browser of browsers ) {
		for ( const suite of suites ) {
			queueRuns( suite, browser );
		}
	}

	try {
		await runAll( { concurrency, verbose } );
	} catch ( error ) {
		console.error( error );
		if ( !debug ) {
			gracefulExit( 1 );
		}
	} finally {
		console.log();
		if ( errorMessages.length === 0 ) {
			let stop = false;
			for ( const report of Object.values( reports ) ) {
				if ( !report.total ) {
					stop = true;
					console.error(
						chalk.red(
							`No tests were run for ${ report.suite } in ${ getBrowserString(
								report.browser
							) }`
						)
					);
				}
			}
			if ( stop ) {
				return gracefulExit( 1 );
			}
			console.log( chalk.green( "All tests passed!" ) );

			if ( !debug ) {
				gracefulExit( 0 );
			}
		} else {
			console.error( chalk.red( `${ errorMessages.length } tests failed.` ) );
			console.log(
				errorMessages.map( ( error, i ) => `\n${ i + 1 }. ${ error }` ).join( "\n" )
			);

			if ( debug ) {
				console.log();
				console.log( "Leaving browsers open for debugging." );
				console.log( "Press Ctrl+C to exit." );
			} else {
				gracefulExit( 1 );
			}
		}
	}
}