Tests: fix flakey message logs; ignore delete worker failures

Close gh-5432
This commit is contained in:
Timmy Willison 2024-03-05 15:50:51 -05:00 committed by GitHub
parent 95a4c94b81
commit 02d2347828
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 29 additions and 20 deletions

View File

@ -187,6 +187,7 @@ export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers ); const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length; const numRemaining = workersRemaining.length;
if ( numRemaining ) { if ( numRemaining ) {
try {
await Promise.all( await Promise.all(
workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
); );
@ -195,5 +196,10 @@ export async function cleanupAllBrowsers( { verbose } ) {
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
); );
} }
} catch ( error ) {
// Log the error, but do not consider the test run failed
console.error( error );
}
} }
} }

View File

@ -39,7 +39,7 @@ export function retryTest( reportId, maxRetries ) {
console.log( console.log(
`Retrying test ${ reportId } for ${ chalk.yellow( `Retrying test ${ reportId } for ${ chalk.yellow(
test.options.modules.join( ", " ) test.options.modules.join( ", " )
) }...` ) }...${ test.retries }`
); );
return test; return test;
} }

View File

@ -83,8 +83,20 @@ export async function run( {
if ( errors ) { if ( errors ) {
pendingErrors[ reportId ][ message.data.name ] = errors; pendingErrors[ reportId ][ message.data.name ] = errors;
} else { } else {
const existing = pendingErrors[ reportId ][ message.data.name ];
// Show a message for flakey tests
if ( existing ) {
console.log();
console.warn(
chalk.italic(
chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
)
);
console.log();
delete pendingErrors[ reportId ][ message.data.name ]; delete pendingErrors[ reportId ][ message.data.name ];
} }
}
break; break;
} }
case "runEnd": { case "runEnd": {
@ -103,24 +115,15 @@ export async function run( {
// Handle failure // Handle failure
if ( failed ) { if ( failed ) {
const retry = retryTest( reportId, retries ); const retry = retryTest( reportId, retries );
// Retry if retryTest returns a test
if ( retry ) { if ( retry ) {
return retry; return retry;
} }
errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
return getNextBrowserTest( reportId );
} }
// Handle success // Run the next test
if (
pendingErrors[ reportId ] &&
Object.keys( pendingErrors[ reportId ] ).length
) {
console.warn( "Detected flaky tests:" );
for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) {
console.warn( chalk.italic( chalk.gray( error ) ) );
}
delete pendingErrors[ reportId ];
}
return getNextBrowserTest( reportId ); return getNextBrowserTest( reportId );
} }
case "ack": { case "ack": {