library-mirrors/jquery
1
0
Fork 0
mirror of https://github.com/jquery/jquery.git synced 2024-11-23 02:54:22 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Tests: share queue/browser handling for all worker types

Browse Source 
- one queue to rule them all: browserstack, selenium, and jsdom
- retries and hard retries are now supported in selenium
- selenium tests now re-use browsers in the same way as browserstack

Close gh-5460
This commit is contained in:
Timmy Willison 2024-04-01 10:23:36 -04:00 committed by GitHub
parent 691c0aeede
commit 284b082eb8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 333 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

265
test/runner/browsers.js
View File

@ -1,25 +1,248 @@
// This list is static, so no requests are required import chalk from "chalk";
// in the command help menu. import { getBrowserString } from "./lib/getBrowserString.js";
import {
createWorker,
deleteWorker,
getAvailableSessions
} from "./browserstack/api.js";
import createDriver from "./selenium/createDriver.js";
import createWindow from "./jsdom/createWindow.js";
import { getBrowsers } from "./browserstack/api.js"; const workers = Object.create( null );
export const browsers = [ /**
"chrome", * Keys are browser strings
"ie", * Structure of a worker:
"firefox", * {
"edge", * browser: object // The browser object
"safari", * debug: boolean // Stops the worker from being cleaned up when finished
"opera", * lastTouch: number // The last time a request was received
"yandex", * restarts: number // The number of times the worker has been restarted
"IE Mobile", * options: object // The options to create the worker
"Android Browser", * url: string // The URL the worker is on
"Mobile Safari", * quit: function // A function to stop the worker
"jsdom" * }
]; */
// A function that can be used to update the above list. // Acknowledge the worker within the time limit.
export async function getAvailableBrowsers() { // BrowserStack can take much longer spinning up
const browsers = await getBrowsers( { flat: true } ); // some browsers, such as iOS 15 Safari.
const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ]; const ACKNOWLEDGE_INTERVAL = 1000;
return available.concat( "jsdom" ); const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
const MAX_WORKER_RESTARTS = 5;
// No report after the time limit
// should refresh the worker
const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
const WORKER_WAIT_TIME = 30000;
// Limit concurrency to 8 by default in selenium
const MAX_SELENIUM_CONCURRENCY = 8;
export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
if ( restarts > MAX_WORKER_RESTARTS ) {
throw new Error(
`Reached the maximum number of restarts for ${ chalk.yellow(
getBrowserString( browser )
) }`
);
}
const { browserstack, debug, headless, reportId, runId, tunnelId, verbose } = options;
while ( await maxWorkersReached( options ) ) {
if ( verbose ) {
console.log( "\nWaiting for available sessions..." );
}
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
}
const fullBrowser = getBrowserString( browser );
let worker;
if ( browserstack ) {
worker = await createWorker( {
...browser,
url: encodeURI( url ),
project: "jquery",
build: `Run ${ runId }`,
// This is the maximum timeout allowed
// by BrowserStack. We do this because
// we control the timeout in the runner.
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
timeout: 1800,
// Not documented in the API docs,
// but required to make local testing work.
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
"browserstack.local": true,
"browserstack.localIdentifier": tunnelId
} );
worker.quit = () => deleteWorker( worker.id );
} else if ( browser.browser === "jsdom" ) {
const window = await createWindow( { reportId, url, verbose } );
worker = {
quit: () => window.close()
};
} else {
const driver = await createDriver( {
browserName: browser.browser,
headless,
url,
verbose
} );
worker = {
quit: () => driver.quit()
};
}
worker.debug = !!debug;
worker.url = url;
worker.browser = browser;
worker.restarts = restarts;
worker.options = options;
touchBrowser( browser );
workers[ fullBrowser ] = worker;
// Wait for the worker to show up in the list
// before returning it.
return ensureAcknowledged( worker );
}
export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}
export async function setBrowserWorkerUrl( browser, url ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.url = url;
}
}
export async function restartBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
await restartWorker( worker );
}
}
/**
* Checks that all browsers have received
* a response in the given amount of time.
* If not, the worker is restarted.
*/
export async function checkLastTouches() {
for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
const options = worker.options;
if ( options.verbose ) {
console.log(
`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
RUN_WORKER_TIMEOUT / 1000 / 60
}min.`
);
}
await restartWorker( worker );
}
}
}
export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length;
if ( numRemaining ) {
try {
await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
if ( verbose ) {
console.log(
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
);
}
} catch ( error ) {
// Log the error, but do not consider the test run failed
console.error( error );
}
}
}
async function maxWorkersReached( {
browserstack,
concurrency = MAX_SELENIUM_CONCURRENCY
} ) {
if ( browserstack ) {
return ( await getAvailableSessions() ) <= 0;
}
return workers.length >= concurrency;
}
async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );
const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}
async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}
async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await worker.quit();
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}
async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
} }

211
test/runner/browserstack/browsers.js
View File

@ -1,211 +0,0 @@
import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js";
import { createWorker, deleteWorker, getAvailableSessions } from "./api.js";
const workers = Object.create( null );
/**
* Keys are browser strings
* Structure of a worker:
* {
* debug: boolean, // Stops the worker from being cleaned up when finished
* id: string,
* lastTouch: number, // The last time a request was received
* url: string,
* browser: object, // The browser object
* options: object // The options to create the worker
* }
*/
// Acknowledge the worker within the time limit.
// BrowserStack can take much longer spinning up
// some browsers, such as iOS 15 Safari.
const ACKNOWLEDGE_INTERVAL = 1000;
const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
const MAX_WORKER_RESTARTS = 5;
// No report after the time limit
// should refresh the worker
const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
const WORKER_WAIT_TIME = 30000;
export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}
async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );
const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}
async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
}
export async function restartBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
await restartWorker( worker );
}
}
async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}
export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
if ( restarts > MAX_WORKER_RESTARTS ) {
throw new Error(
`Reached the maximum number of restarts for ${ chalk.yellow(
getBrowserString( browser )
) }`
);
}
const verbose = options.verbose;
while ( ( await getAvailableSessions() ) <= 0 ) {
if ( verbose ) {
console.log( "\nWaiting for available sessions..." );
}
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
}
const { debug, runId, tunnelId } = options;
const fullBrowser = getBrowserString( browser );
const worker = await createWorker( {
...browser,
url: encodeURI( url ),
project: "jquery",
build: `Run ${ runId }`,
// This is the maximum timeout allowed
// by BrowserStack. We do this because
// we control the timeout in the runner.
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
timeout: 1800,
// Not documented in the API docs,
// but required to make local testing work.
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
"browserstack.local": true,
"browserstack.localIdentifier": tunnelId
} );
browser.debug = !!debug;
worker.url = url;
worker.browser = browser;
worker.restarts = restarts;
worker.options = options;
touchBrowser( browser );
workers[ fullBrowser ] = worker;
// Wait for the worker to show up in the list
// before returning it.
return ensureAcknowledged( worker );
}
export async function setBrowserWorkerUrl( browser, url ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.url = url;
}
}
/**
* Checks that all browsers have received
* a response in the given amount of time.
* If not, the worker is restarted.
*/
export async function checkLastTouches() {
for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
const options = worker.options;
if ( options.verbose ) {
console.log(
`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
RUN_WORKER_TIMEOUT / 1000 / 60
}min.`
);
}
await restartWorker( worker );
}
}
}
export async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await deleteWorker( worker.id );
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}
export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length;
if ( numRemaining ) {
try {
await Promise.all(
workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
);
if ( verbose ) {
console.log(
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
);
}
} catch ( error ) {
// Log the error, but do not consider the test run failed
console.error( error );
}
}
}

42
test/runner/command.js
View File

@ -1,8 +1,8 @@
import yargs from "yargs/yargs"; import yargs from "yargs/yargs";
import { browsers } from "./browsers.js"; import { browsers } from "./flags/browsers.js";
import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js"; import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
import { modules } from "./modules.js"; import { modules } from "./flags/modules.js";
import { run } from "./run.js"; import { run } from "./run.js";
const argv = yargs( process.argv.slice( 2 ) ) const argv = yargs( process.argv.slice( 2 ) )
@ -59,15 +59,23 @@ const argv = yargs( process.argv.slice( 2 ) )
"Leave the browser open for debugging. Cannot be used with --headless.", "Leave the browser open for debugging. Cannot be used with --headless.",
conflicts: [ "headless" ] conflicts: [ "headless" ]
} ) } )
.option( "retries", {
alias: "r",
type: "number",
description: "Number of times to retry failed tests by refreshing the URL."
} )
.option( "hard-retries", {
type: "number",
description:
"Number of times to retry failed tests by restarting the worker. " +
"This is in addition to the normal retries " +
"and are only used when the normal retries are exhausted."
} )
.option( "verbose", { .option( "verbose", {
alias: "v", alias: "v",
type: "boolean", type: "boolean",
description: "Log additional information." description: "Log additional information."
} ) } )
.option( "run-id", {
type: "string",
description: "A unique identifier for this run."
} )
.option( "isolate", { .option( "isolate", {
type: "boolean", type: "boolean",
description: "Run each module by itself in the test page. This can extend testing time." description: "Run each module by itself in the test page. This can extend testing time."
@ -84,19 +92,9 @@ const argv = yargs( process.argv.slice( 2 ) )
"Otherwise, the --browser option will be used, " + "Otherwise, the --browser option will be used, " +
"with the latest version/device for that browser, on a matching OS." "with the latest version/device for that browser, on a matching OS."
} ) } )
.option( "retries", { .option( "run-id", {
alias: "r", type: "string",
type: "number", description: "A unique identifier for the run in BrowserStack."
description: "Number of times to retry failed tests in BrowserStack.",
implies: [ "browserstack" ]
} )
.option( "hard-retries", {
type: "number",
description:
"Number of times to retry failed tests in BrowserStack " +
"by restarting the worker. This is in addition to the normal retries " +
"and are only used when the normal retries are exhausted.",
implies: [ "browserstack" ]
} ) } )
.option( "list-browsers", { .option( "list-browsers", {
type: "string", type: "string",
@ -132,9 +130,5 @@ if ( typeof argv.listBrowsers === "string" ) {
} else if ( argv.browserstackPlan ) { } else if ( argv.browserstackPlan ) {
console.log( await getPlan() ); console.log( await getPlan() );
} else { } else {
run( { run( argv );
...argv,
browsers: argv.browser,
modules: argv.module
} );
} }

25
test/runner/flags/browsers.js Normal file
View File

@ -0,0 +1,25 @@
// This list is static, so no requests are required
// in the command help menu.
import { getBrowsers } from "../browserstack/api.js";
export const browsers = [
"chrome",
"ie",
"firefox",
"edge",
"safari",
"opera",
"yandex",
"IE Mobile",
"Android Browser",
"Mobile Safari",
"jsdom"
];
// A function that can be used to update the above list.
export async function getAvailableBrowsers() {
const browsers = await getBrowsers( { flat: true } );
const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ];
return available.concat( "jsdom" );
}

0
test/runner/modules.js → test/runner/flags/modules.js
View File

55
test/runner/jsdom.js
View File

@ -1,55 +0,0 @@
import jsdom from "jsdom";
const { JSDOM } = jsdom;
const windows = Object.create( null );
export async function runJSDOM( url, { reportId, verbose } ) {
const virtualConsole = new jsdom.VirtualConsole();
virtualConsole.sendTo( console );
virtualConsole.removeAllListeners( "clear" );
const { window } = await JSDOM.fromURL( url, {
resources: "usable",
runScripts: "dangerously",
virtualConsole
} );
if ( verbose ) {
console.log( "JSDOM window opened.", reportId );
}
windows[ reportId ] = window;
return new Promise( ( resolve ) => {
window.finish = resolve;
} );
}
export function cleanupJSDOM( reportId, { verbose } ) {
const window = windows[ reportId ];
if ( window ) {
if ( window.finish ) {
window.finish();
}
window.close();
delete windows[ reportId ];
if ( verbose ) {
console.log( "Closed JSDOM window.", reportId );
}
}
}
export function cleanupAllJSDOM( { verbose } ) {
const windowsRemaining = Object.keys( windows ).length;
if ( windowsRemaining ) {
if ( verbose ) {
console.log(
`Cleaning up ${ windowsRemaining } JSDOM window${
windowsRemaining > 1 ? "s" : ""
}...`
);
}
for ( const id in windows ) {
cleanupJSDOM( id, { verbose } );
}
}
}

21
test/runner/jsdom/createWindow.js Normal file
View File

@ -0,0 +1,21 @@
import jsdom from "jsdom";
const { JSDOM } = jsdom;
export default async function createWindow( { reportId, url, verbose } ) {
const virtualConsole = new jsdom.VirtualConsole();
virtualConsole.sendTo( console );
virtualConsole.removeAllListeners( "clear" );
const { window } = await JSDOM.fromURL( url, {
resources: "usable",
runScripts: "dangerously",
virtualConsole
} );
if ( verbose ) {
console.log( `JSDOM window created (${ reportId })` );
}
return window;
}

10
test/runner/browserstack/queue.js → test/runner/queue.js
View File

@ -1,5 +1,5 @@
import chalk from "chalk"; import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js"; import { getBrowserString } from "./lib/getBrowserString.js";
import { import {
checkLastTouches, checkLastTouches,
createBrowserWorker, createBrowserWorker,
@ -45,7 +45,7 @@ export function retryTest( reportId, maxRetries ) {
test.retries++; test.retries++;
if ( test.retries <= maxRetries ) { if ( test.retries <= maxRetries ) {
console.log( console.log(
`Retrying test ${ reportId } for ${ chalk.yellow( `\nRetrying test ${ reportId } for ${ chalk.yellow(
test.options.modules.join( ", " ) test.options.modules.join( ", " )
) }...${ test.retries }` ) }...${ test.retries }`
); );
@ -63,7 +63,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) {
test.hardRetries++; test.hardRetries++;
if ( test.hardRetries <= maxHardRetries ) { if ( test.hardRetries <= maxHardRetries ) {
console.log( console.log(
`Hard retrying test ${ reportId } for ${ chalk.yellow( `\nHard retrying test ${ reportId } for ${ chalk.yellow(
test.options.modules.join( ", " ) test.options.modules.join( ", " )
) }...${ test.hardRetries }` ) }...${ test.hardRetries }`
); );
@ -74,7 +74,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) {
return false; return false;
} }
export function addBrowserStackRun( url, browser, options ) { export function addRun( url, browser, options ) {
queue.push( { queue.push( {
browser, browser,
fullBrowser: getBrowserString( browser ), fullBrowser: getBrowserString( browser ),
@ -87,7 +87,7 @@ export function addBrowserStackRun( url, browser, options ) {
} ); } );
} }
export async function runAllBrowserStack() { export async function runAll() {
return new Promise( async( resolve, reject ) => { return new Promise( async( resolve, reject ) => {
while ( queue.length ) { while ( queue.length ) {
try { try {

7
test/runner/reporter.js
View File

@ -115,10 +115,11 @@ export function reportTest( test, reportId, { browser, headless } ) {
} }
export function reportEnd( result, reportId, { browser, headless, modules } ) { export function reportEnd( result, reportId, { browser, headless, modules } ) {
const fullBrowser = getBrowserString( browser, headless );
console.log( console.log(
`\n\nTests for ${ chalk.yellow( modules.join( ", " ) ) } on ${ chalk.yellow( `\n\nTests finished in ${ prettyMs( result.runtime ) } ` +
getBrowserString( browser, headless ) `for ${ chalk.yellow( modules.join( "," ) ) } ` +
) } finished in ${ prettyMs( result.runtime ) } (${ chalk.bold( reportId ) }).` `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( reportId ) })...`
); );
console.log( console.log(
( result.status !== "passed" ? ( result.status !== "passed" ?

35
test/runner/run.js
View File

@ -8,17 +8,15 @@ import { createTestServer } from "./createTestServer.js";
import { buildTestUrl } from "./lib/buildTestUrl.js"; import { buildTestUrl } from "./lib/buildTestUrl.js";
import { generateHash, printModuleHashes } from "./lib/generateHash.js"; import { generateHash, printModuleHashes } from "./lib/generateHash.js";
import { getBrowserString } from "./lib/getBrowserString.js"; import { getBrowserString } from "./lib/getBrowserString.js";
import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js"; import { modules as allModules } from "./flags/modules.js";
import { modules as allModules } from "./modules.js"; import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js";
import { import {
addBrowserStackRun, addRun,
getNextBrowserTest, getNextBrowserTest,
hardRetryTest, hardRetryTest,
retryTest, retryTest,
runAllBrowserStack runAll
} from "./browserstack/queue.js"; } from "./queue.js";
import { addSeleniumRun, runAllSelenium } from "./selenium/queue.js";
const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
@ -26,7 +24,7 @@ const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
* Run modules in parallel in different browser instances. * Run modules in parallel in different browser instances.
*/ */
export async function run( { export async function run( {
browsers: browserNames, browser: browserNames = [],
browserstack, browserstack,
concurrency, concurrency,
debug, debug,
@ -34,12 +32,12 @@ export async function run( {
hardRetries, hardRetries,
headless, headless,
isolate, isolate,
modules = [], module: modules = [],
retries = 0, retries = 0,
runId, runId,
verbose verbose
} ) { } ) {
if ( !browserNames || !browserNames.length ) { if ( !browserNames.length ) {
browserNames = [ "chrome" ]; browserNames = [ "chrome" ];
} }
if ( !modules.length ) { if ( !modules.length ) {
@ -112,8 +110,6 @@ export async function run( {
); );
report.total = total; report.total = total;
cleanupJSDOM( reportId, { verbose } );
// Handle failure // Handle failure
if ( failed ) { if ( failed ) {
const retry = retryTest( reportId, retries ); const retry = retryTest( reportId, retries );
@ -178,7 +174,6 @@ export async function run( {
console.log( "Cleaning up..." ); console.log( "Cleaning up..." );
await cleanupAllBrowsers( { verbose } ); await cleanupAllBrowsers( { verbose } );
cleanupAllJSDOM( { verbose } );
if ( tunnel ) { if ( tunnel ) {
await tunnel.stop(); await tunnel.stop();
@ -260,6 +255,8 @@ export async function run( {
} ); } );
const options = { const options = {
browserstack,
concurrency,
debug, debug,
headless, headless,
modules, modules,
@ -269,11 +266,7 @@ export async function run( {
verbose verbose
}; };
if ( browserstack ) { addRun( url, browser, options );
addBrowserStackRun( url, browser, options );
} else {
addSeleniumRun( url, browser, options );
}
} }
for ( const browser of browsers ) { for ( const browser of browsers ) {
@ -288,11 +281,7 @@ export async function run( {
try { try {
console.log( `Starting Run ${ runId }...` ); console.log( `Starting Run ${ runId }...` );
if ( browserstack ) { await runAll();
await runAllBrowserStack( { verbose } );
} else {
await runAllSelenium( { concurrency, verbose } );
}
} catch ( error ) { } catch ( error ) {
console.error( error ); console.error( error );
if ( !debug ) { if ( !debug ) {

5
test/runner/selenium/createDriver.js
View File

@ -7,7 +7,7 @@ import { browserSupportsHeadless } from "../lib/getBrowserString.js";
// Set script timeout to 10min // Set script timeout to 10min
const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10; const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10;
export default async function createDriver( { browserName, headless, verbose } ) { export default async function createDriver( { browserName, headless, url, verbose } ) {
const capabilities = Capabilities[ browserName ](); const capabilities = Capabilities[ browserName ]();
const prefs = new logging.Preferences(); const prefs = new logging.Preferences();
prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL ); prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL );
@ -77,5 +77,8 @@ export default async function createDriver( { browserName, headless, verbose } )
// Increase script timeout to 10min // Increase script timeout to 10min
await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } ); await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } );
// Set the first URL for the browser
await driver.get( url );
return driver; return driver;
} }

70
test/runner/selenium/queue.js
View File

@ -1,70 +0,0 @@
// Build a queue that runs both browsers and modules
// in parallel when the length reaches the concurrency limit
// and refills the queue when one promise resolves.
import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js";
import { runSelenium } from "./runSelenium.js";
import { runJSDOM } from "../jsdom.js";
const promises = [];
const queue = [];
const SELENIUM_WAIT_TIME = 100;
// Limit concurrency to 8 by default in selenium
// BrowserStack defaults to the max allowed by the plan
// More than this will log MaxListenersExceededWarning
const MAX_CONCURRENCY = 8;
export function addSeleniumRun( url, browser, options ) {
queue.push( { url, browser, options } );
}
export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) {
while ( queue.length ) {
const next = queue.shift();
const { url, browser, options } = next;
const fullBrowser = getBrowserString( browser, options.headless );
console.log(
`\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` +
`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...`
);
// Wait enough time between requests
// to give concurrency a chance to update.
// In selenium, this helps avoid undici connect timeout errors.
await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) );
if ( verbose ) {
console.log( `\nTests remaining: ${ queue.length + 1 }.` );
}
let promise;
if ( browser.browser === "jsdom" ) {
promise = runJSDOM( url, options );
} else {
promise = runSelenium( url, browser, options );
}
// Remove the promise from the list when it resolves
promise.then( () => {
const index = promises.indexOf( promise );
if ( index !== -1 ) {
promises.splice( index, 1 );
}
} );
// Add the promise to the list
promises.push( promise );
// Wait until at least one promise resolves
// if we've reached the concurrency limit
if ( promises.length >= concurrency ) {
await Promise.any( promises );
}
}
await Promise.all( promises );
}

30
test/runner/selenium/runSelenium.js
View File

@ -1,30 +0,0 @@
import createDriver from "./createDriver.js";
export async function runSelenium(
url,
{ browser },
{ debug, headless, verbose } = {}
) {
if ( debug && headless ) {
throw new Error( "Cannot debug in headless mode." );
}
const driver = await createDriver( {
browserName: browser,
headless,
verbose
} );
try {
await driver.get( url );
await driver.executeScript(
`return new Promise( ( resolve ) => {
QUnit.on( "runEnd", resolve );
} )`
);
} finally {
if ( !debug || headless ) {
await driver.quit();
}
}
}