From cc44f768344b318109b2f74d6fed28258841470e Mon Sep 17 00:00:00 2001 From: Timmy Willison Date: Mon, 1 Apr 2024 12:34:13 -0400 Subject: [PATCH] Tests: share queue/browser handling for all worker types - one queue to rule them all: browserstack, selenium, and jsdom - retries and hard retries are now supported in selenium - selenium tests now re-use browsers in the same way as browserstack Close gh-5465 --- test/runner/browsers.js | 265 ++++++++++++++++++++++-- test/runner/browserstack/browsers.js | 211 ------------------- test/runner/command.js | 42 ++-- test/runner/flags/browsers.js | 25 +++ test/runner/{ => flags}/modules.js | 0 test/runner/jsdom.js | 55 ----- test/runner/jsdom/createWindow.js | 21 ++ test/runner/{browserstack => }/queue.js | 10 +- test/runner/reporter.js | 7 +- test/runner/run.js | 35 ++-- test/runner/selenium/createDriver.js | 5 +- test/runner/selenium/queue.js | 70 ------- test/runner/selenium/runSelenium.js | 30 --- 13 files changed, 333 insertions(+), 443 deletions(-) delete mode 100644 test/runner/browserstack/browsers.js create mode 100644 test/runner/flags/browsers.js rename test/runner/{ => flags}/modules.js (100%) delete mode 100644 test/runner/jsdom.js create mode 100644 test/runner/jsdom/createWindow.js rename test/runner/{browserstack => }/queue.js (89%) delete mode 100644 test/runner/selenium/queue.js delete mode 100644 test/runner/selenium/runSelenium.js diff --git a/test/runner/browsers.js b/test/runner/browsers.js index a3a8df0d8..5962ea160 100644 --- a/test/runner/browsers.js +++ b/test/runner/browsers.js @@ -1,25 +1,248 @@ -// This list is static, so no requests are required -// in the command help menu. +import chalk from "chalk"; +import { getBrowserString } from "./lib/getBrowserString.js"; +import { + createWorker, + deleteWorker, + getAvailableSessions +} from "./browserstack/api.js"; +import createDriver from "./selenium/createDriver.js"; +import createWindow from "./jsdom/createWindow.js"; -import { getBrowsers } from "./browserstack/api.js"; +const workers = Object.create( null ); -export const browsers = [ - "chrome", - "ie", - "firefox", - "edge", - "safari", - "opera", - "yandex", - "IE Mobile", - "Android Browser", - "Mobile Safari", - "jsdom" -]; +/** + * Keys are browser strings + * Structure of a worker: + * { + * browser: object // The browser object + * debug: boolean // Stops the worker from being cleaned up when finished + * lastTouch: number // The last time a request was received + * restarts: number // The number of times the worker has been restarted + * options: object // The options to create the worker + * url: string // The URL the worker is on + * quit: function // A function to stop the worker + * } + */ -// A function that can be used to update the above list. -export async function getAvailableBrowsers() { - const browsers = await getBrowsers( { flat: true } ); - const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ]; - return available.concat( "jsdom" ); +// Acknowledge the worker within the time limit. +// BrowserStack can take much longer spinning up +// some browsers, such as iOS 15 Safari. +const ACKNOWLEDGE_INTERVAL = 1000; +const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5; + +const MAX_WORKER_RESTARTS = 5; + +// No report after the time limit +// should refresh the worker +const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; + +const WORKER_WAIT_TIME = 30000; + +// Limit concurrency to 8 by default in selenium +const MAX_SELENIUM_CONCURRENCY = 8; + +export async function createBrowserWorker( url, browser, options, restarts = 0 ) { + if ( restarts > MAX_WORKER_RESTARTS ) { + throw new Error( + `Reached the maximum number of restarts for ${ chalk.yellow( + getBrowserString( browser ) + ) }` + ); + } + const { browserstack, debug, headless, reportId, runId, tunnelId, verbose } = options; + while ( await maxWorkersReached( options ) ) { + if ( verbose ) { + console.log( "\nWaiting for available sessions..." ); + } + await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); + } + + const fullBrowser = getBrowserString( browser ); + + let worker; + + if ( browserstack ) { + worker = await createWorker( { + ...browser, + url: encodeURI( url ), + project: "jquery", + build: `Run ${ runId }`, + + // This is the maximum timeout allowed + // by BrowserStack. We do this because + // we control the timeout in the runner. + // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 + timeout: 1800, + + // Not documented in the API docs, + // but required to make local testing work. + // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs + "browserstack.local": true, + "browserstack.localIdentifier": tunnelId + } ); + worker.quit = () => deleteWorker( worker.id ); + } else if ( browser.browser === "jsdom" ) { + const window = await createWindow( { reportId, url, verbose } ); + worker = { + quit: () => window.close() + }; + } else { + const driver = await createDriver( { + browserName: browser.browser, + headless, + url, + verbose + } ); + worker = { + quit: () => driver.quit() + }; + } + + worker.debug = !!debug; + worker.url = url; + worker.browser = browser; + worker.restarts = restarts; + worker.options = options; + touchBrowser( browser ); + workers[ fullBrowser ] = worker; + + // Wait for the worker to show up in the list + // before returning it. + return ensureAcknowledged( worker ); +} + +export function touchBrowser( browser ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + worker.lastTouch = Date.now(); + } +} + +export async function setBrowserWorkerUrl( browser, url ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + worker.url = url; + } +} + +export async function restartBrowser( browser ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + await restartWorker( worker ); + } +} + +/** + * Checks that all browsers have received + * a response in the given amount of time. + * If not, the worker is restarted. + */ +export async function checkLastTouches() { + for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) { + if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) { + const options = worker.options; + if ( options.verbose ) { + console.log( + `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${ + RUN_WORKER_TIMEOUT / 1000 / 60 + }min.` + ); + } + await restartWorker( worker ); + } + } +} + +export async function cleanupAllBrowsers( { verbose } ) { + const workersRemaining = Object.values( workers ); + const numRemaining = workersRemaining.length; + if ( numRemaining ) { + try { + await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) ); + if ( verbose ) { + console.log( + `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` + ); + } + } catch ( error ) { + + // Log the error, but do not consider the test run failed + console.error( error ); + } + } +} + +async function maxWorkersReached( { + browserstack, + concurrency = MAX_SELENIUM_CONCURRENCY +} ) { + if ( browserstack ) { + return ( await getAvailableSessions() ) <= 0; + } + return workers.length >= concurrency; +} + +async function waitForAck( worker, { fullBrowser, verbose } ) { + delete worker.lastTouch; + return new Promise( ( resolve, reject ) => { + const interval = setInterval( () => { + if ( worker.lastTouch ) { + if ( verbose ) { + console.log( `\n${ fullBrowser } acknowledged.` ); + } + clearTimeout( timeout ); + clearInterval( interval ); + resolve(); + } + }, ACKNOWLEDGE_INTERVAL ); + + const timeout = setTimeout( () => { + clearInterval( interval ); + reject( + new Error( + `${ fullBrowser } not acknowledged after ${ + ACKNOWLEDGE_TIMEOUT / 1000 / 60 + }min.` + ) + ); + }, ACKNOWLEDGE_TIMEOUT ); + } ); +} + +async function ensureAcknowledged( worker ) { + const fullBrowser = getBrowserString( worker.browser ); + const verbose = worker.options.verbose; + try { + await waitForAck( worker, { fullBrowser, verbose } ); + return worker; + } catch ( error ) { + console.error( error.message ); + await restartWorker( worker ); + } +} + +async function cleanupWorker( worker, { verbose } ) { + for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { + if ( w === worker ) { + delete workers[ fullBrowser ]; + await worker.quit(); + if ( verbose ) { + console.log( `\nStopped ${ fullBrowser }.` ); + } + return; + } + } +} + +async function restartWorker( worker ) { + await cleanupWorker( worker, worker.options ); + await createBrowserWorker( + worker.url, + worker.browser, + worker.options, + worker.restarts + 1 + ); } diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js deleted file mode 100644 index 6489cf965..000000000 --- a/test/runner/browserstack/browsers.js +++ /dev/null @@ -1,211 +0,0 @@ -import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; -import { createWorker, deleteWorker, getAvailableSessions } from "./api.js"; - -const workers = Object.create( null ); - -/** - * Keys are browser strings - * Structure of a worker: - * { - * debug: boolean, // Stops the worker from being cleaned up when finished - * id: string, - * lastTouch: number, // The last time a request was received - * url: string, - * browser: object, // The browser object - * options: object // The options to create the worker - * } - */ - -// Acknowledge the worker within the time limit. -// BrowserStack can take much longer spinning up -// some browsers, such as iOS 15 Safari. -const ACKNOWLEDGE_INTERVAL = 1000; -const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5; - -const MAX_WORKER_RESTARTS = 5; - -// No report after the time limit -// should refresh the worker -const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; - -const WORKER_WAIT_TIME = 30000; - -export function touchBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - worker.lastTouch = Date.now(); - } -} - -async function waitForAck( worker, { fullBrowser, verbose } ) { - delete worker.lastTouch; - return new Promise( ( resolve, reject ) => { - const interval = setInterval( () => { - if ( worker.lastTouch ) { - if ( verbose ) { - console.log( `\n${ fullBrowser } acknowledged.` ); - } - clearTimeout( timeout ); - clearInterval( interval ); - resolve(); - } - }, ACKNOWLEDGE_INTERVAL ); - - const timeout = setTimeout( () => { - clearInterval( interval ); - reject( - new Error( - `${ fullBrowser } not acknowledged after ${ - ACKNOWLEDGE_TIMEOUT / 1000 / 60 - }min.` - ) - ); - }, ACKNOWLEDGE_TIMEOUT ); - } ); -} - -async function restartWorker( worker ) { - await cleanupWorker( worker, worker.options ); - await createBrowserWorker( - worker.url, - worker.browser, - worker.options, - worker.restarts + 1 - ); -} - -export async function restartBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - await restartWorker( worker ); - } -} - -async function ensureAcknowledged( worker ) { - const fullBrowser = getBrowserString( worker.browser ); - const verbose = worker.options.verbose; - try { - await waitForAck( worker, { fullBrowser, verbose } ); - return worker; - } catch ( error ) { - console.error( error.message ); - await restartWorker( worker ); - } -} - -export async function createBrowserWorker( url, browser, options, restarts = 0 ) { - if ( restarts > MAX_WORKER_RESTARTS ) { - throw new Error( - `Reached the maximum number of restarts for ${ chalk.yellow( - getBrowserString( browser ) - ) }` - ); - } - const verbose = options.verbose; - while ( ( await getAvailableSessions() ) <= 0 ) { - if ( verbose ) { - console.log( "\nWaiting for available sessions..." ); - } - await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); - } - - const { debug, runId, tunnelId } = options; - const fullBrowser = getBrowserString( browser ); - - const worker = await createWorker( { - ...browser, - url: encodeURI( url ), - project: "jquery", - build: `Run ${ runId }`, - - // This is the maximum timeout allowed - // by BrowserStack. We do this because - // we control the timeout in the runner. - // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 - timeout: 1800, - - // Not documented in the API docs, - // but required to make local testing work. - // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs - "browserstack.local": true, - "browserstack.localIdentifier": tunnelId - } ); - - browser.debug = !!debug; - worker.url = url; - worker.browser = browser; - worker.restarts = restarts; - worker.options = options; - touchBrowser( browser ); - workers[ fullBrowser ] = worker; - - // Wait for the worker to show up in the list - // before returning it. - return ensureAcknowledged( worker ); -} - -export async function setBrowserWorkerUrl( browser, url ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - worker.url = url; - } -} - -/** - * Checks that all browsers have received - * a response in the given amount of time. - * If not, the worker is restarted. - */ -export async function checkLastTouches() { - for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) { - if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) { - const options = worker.options; - if ( options.verbose ) { - console.log( - `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${ - RUN_WORKER_TIMEOUT / 1000 / 60 - }min.` - ); - } - await restartWorker( worker ); - } - } -} - -export async function cleanupWorker( worker, { verbose } ) { - for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { - if ( w === worker ) { - delete workers[ fullBrowser ]; - await deleteWorker( worker.id ); - if ( verbose ) { - console.log( `\nStopped ${ fullBrowser }.` ); - } - return; - } - } -} - -export async function cleanupAllBrowsers( { verbose } ) { - const workersRemaining = Object.values( workers ); - const numRemaining = workersRemaining.length; - if ( numRemaining ) { - try { - await Promise.all( - workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) - ); - if ( verbose ) { - console.log( - `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` - ); - } - } catch ( error ) { - - // Log the error, but do not consider the test run failed - console.error( error ); - } - } -} diff --git a/test/runner/command.js b/test/runner/command.js index fb1d16883..7b6629f4a 100644 --- a/test/runner/command.js +++ b/test/runner/command.js @@ -1,8 +1,8 @@ import yargs from "yargs/yargs"; -import { browsers } from "./browsers.js"; +import { browsers } from "./flags/browsers.js"; import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js"; import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; -import { modules } from "./modules.js"; +import { modules } from "./flags/modules.js"; import { run } from "./run.js"; const argv = yargs( process.argv.slice( 2 ) ) @@ -58,15 +58,23 @@ const argv = yargs( process.argv.slice( 2 ) ) "Leave the browser open for debugging. Cannot be used with --headless.", conflicts: [ "headless" ] } ) + .option( "retries", { + alias: "r", + type: "number", + description: "Number of times to retry failed tests by refreshing the URL." + } ) + .option( "hard-retries", { + type: "number", + description: + "Number of times to retry failed tests by restarting the worker. " + + "This is in addition to the normal retries " + + "and are only used when the normal retries are exhausted." + } ) .option( "verbose", { alias: "v", type: "boolean", description: "Log additional information." } ) - .option( "run-id", { - type: "string", - description: "A unique identifier for this run." - } ) .option( "isolate", { type: "boolean", description: "Run each module by itself in the test page. This can extend testing time." @@ -83,19 +91,9 @@ const argv = yargs( process.argv.slice( 2 ) ) "Otherwise, the --browser option will be used, " + "with the latest version/device for that browser, on a matching OS." } ) - .option( "retries", { - alias: "r", - type: "number", - description: "Number of times to retry failed tests in BrowserStack.", - implies: [ "browserstack" ] - } ) - .option( "hard-retries", { - type: "number", - description: - "Number of times to retry failed tests in BrowserStack " + - "by restarting the worker. This is in addition to the normal retries " + - "and are only used when the normal retries are exhausted.", - implies: [ "browserstack" ] + .option( "run-id", { + type: "string", + description: "A unique identifier for the run in BrowserStack." } ) .option( "list-browsers", { type: "string", @@ -131,9 +129,5 @@ if ( typeof argv.listBrowsers === "string" ) { } else if ( argv.browserstackPlan ) { console.log( await getPlan() ); } else { - run( { - ...argv, - browsers: argv.browser, - modules: argv.module - } ); + run( argv ); } diff --git a/test/runner/flags/browsers.js b/test/runner/flags/browsers.js new file mode 100644 index 000000000..c15d7085e --- /dev/null +++ b/test/runner/flags/browsers.js @@ -0,0 +1,25 @@ +// This list is static, so no requests are required +// in the command help menu. + +import { getBrowsers } from "../browserstack/api.js"; + +export const browsers = [ + "chrome", + "ie", + "firefox", + "edge", + "safari", + "opera", + "yandex", + "IE Mobile", + "Android Browser", + "Mobile Safari", + "jsdom" +]; + +// A function that can be used to update the above list. +export async function getAvailableBrowsers() { + const browsers = await getBrowsers( { flat: true } ); + const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ]; + return available.concat( "jsdom" ); +} diff --git a/test/runner/modules.js b/test/runner/flags/modules.js similarity index 100% rename from test/runner/modules.js rename to test/runner/flags/modules.js diff --git a/test/runner/jsdom.js b/test/runner/jsdom.js deleted file mode 100644 index d9ff9dda7..000000000 --- a/test/runner/jsdom.js +++ /dev/null @@ -1,55 +0,0 @@ -import jsdom from "jsdom"; - -const { JSDOM } = jsdom; - -const windows = Object.create( null ); - -export async function runJSDOM( url, { reportId, verbose } ) { - const virtualConsole = new jsdom.VirtualConsole(); - virtualConsole.sendTo( console ); - virtualConsole.removeAllListeners( "clear" ); - - const { window } = await JSDOM.fromURL( url, { - resources: "usable", - runScripts: "dangerously", - virtualConsole - } ); - if ( verbose ) { - console.log( "JSDOM window opened.", reportId ); - } - windows[ reportId ] = window; - - return new Promise( ( resolve ) => { - window.finish = resolve; - } ); -} - -export function cleanupJSDOM( reportId, { verbose } ) { - const window = windows[ reportId ]; - if ( window ) { - if ( window.finish ) { - window.finish(); - } - window.close(); - delete windows[ reportId ]; - if ( verbose ) { - console.log( "Closed JSDOM window.", reportId ); - } - } -} - -export function cleanupAllJSDOM( { verbose } ) { - const windowsRemaining = Object.keys( windows ).length; - if ( windowsRemaining ) { - if ( verbose ) { - console.log( - `Cleaning up ${ windowsRemaining } JSDOM window${ - windowsRemaining > 1 ? "s" : "" - }...` - ); - } - for ( const id in windows ) { - cleanupJSDOM( id, { verbose } ); - } - } -} diff --git a/test/runner/jsdom/createWindow.js b/test/runner/jsdom/createWindow.js new file mode 100644 index 000000000..de6c63ffa --- /dev/null +++ b/test/runner/jsdom/createWindow.js @@ -0,0 +1,21 @@ +import jsdom from "jsdom"; + +const { JSDOM } = jsdom; + +export default async function createWindow( { reportId, url, verbose } ) { + const virtualConsole = new jsdom.VirtualConsole(); + virtualConsole.sendTo( console ); + virtualConsole.removeAllListeners( "clear" ); + + const { window } = await JSDOM.fromURL( url, { + resources: "usable", + runScripts: "dangerously", + virtualConsole + } ); + + if ( verbose ) { + console.log( `JSDOM window created (${ reportId })` ); + } + + return window; +} diff --git a/test/runner/browserstack/queue.js b/test/runner/queue.js similarity index 89% rename from test/runner/browserstack/queue.js rename to test/runner/queue.js index 6d1c8d51f..843d5672f 100644 --- a/test/runner/browserstack/queue.js +++ b/test/runner/queue.js @@ -1,5 +1,5 @@ import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; +import { getBrowserString } from "./lib/getBrowserString.js"; import { checkLastTouches, createBrowserWorker, @@ -45,7 +45,7 @@ export function retryTest( reportId, maxRetries ) { test.retries++; if ( test.retries <= maxRetries ) { console.log( - `Retrying test ${ reportId } for ${ chalk.yellow( + `\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.modules.join( ", " ) ) }...${ test.retries }` ); @@ -63,7 +63,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) { test.hardRetries++; if ( test.hardRetries <= maxHardRetries ) { console.log( - `Hard retrying test ${ reportId } for ${ chalk.yellow( + `\nHard retrying test ${ reportId } for ${ chalk.yellow( test.options.modules.join( ", " ) ) }...${ test.hardRetries }` ); @@ -74,7 +74,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) { return false; } -export function addBrowserStackRun( url, browser, options ) { +export function addRun( url, browser, options ) { queue.push( { browser, fullBrowser: getBrowserString( browser ), @@ -87,7 +87,7 @@ export function addBrowserStackRun( url, browser, options ) { } ); } -export async function runAllBrowserStack() { +export async function runAll() { return new Promise( async( resolve, reject ) => { while ( queue.length ) { try { diff --git a/test/runner/reporter.js b/test/runner/reporter.js index 1c7467d6c..e79059648 100644 --- a/test/runner/reporter.js +++ b/test/runner/reporter.js @@ -115,10 +115,11 @@ export function reportTest( test, reportId, { browser, headless } ) { } export function reportEnd( result, reportId, { browser, headless, modules } ) { + const fullBrowser = getBrowserString( browser, headless ); console.log( - `\n\nTests for ${ chalk.yellow( modules.join( ", " ) ) } on ${ chalk.yellow( - getBrowserString( browser, headless ) - ) } finished in ${ prettyMs( result.runtime ) } (${ chalk.bold( reportId ) }).` + `\n\nTests finished in ${ prettyMs( result.runtime ) } ` + + `for ${ chalk.yellow( modules.join( "," ) ) } ` + + `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( reportId ) })...` ); console.log( ( result.status !== "passed" ? diff --git a/test/runner/run.js b/test/runner/run.js index 0b3ffec30..50037c6a2 100644 --- a/test/runner/run.js +++ b/test/runner/run.js @@ -8,17 +8,15 @@ import { createTestServer } from "./createTestServer.js"; import { buildTestUrl } from "./lib/buildTestUrl.js"; import { generateHash, printModuleHashes } from "./lib/generateHash.js"; import { getBrowserString } from "./lib/getBrowserString.js"; -import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js"; -import { modules as allModules } from "./modules.js"; -import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js"; +import { modules as allModules } from "./flags/modules.js"; +import { cleanupAllBrowsers, touchBrowser } from "./browsers.js"; import { - addBrowserStackRun, + addRun, getNextBrowserTest, hardRetryTest, retryTest, - runAllBrowserStack -} from "./browserstack/queue.js"; -import { addSeleniumRun, runAllSelenium } from "./selenium/queue.js"; + runAll +} from "./queue.js"; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; @@ -27,19 +25,19 @@ const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; */ export async function run( { amd, - browsers: browserNames, + browser: browserNames = [], browserstack, concurrency, debug, hardRetries, headless, isolate, - modules = [], + module: modules = [], retries = 0, runId, verbose } ) { - if ( !browserNames || !browserNames.length ) { + if ( !browserNames.length ) { browserNames = [ "chrome" ]; } if ( !modules.length ) { @@ -112,8 +110,6 @@ export async function run( { ); report.total = total; - cleanupJSDOM( reportId, { verbose } ); - // Handle failure if ( failed ) { const retry = retryTest( reportId, retries ); @@ -178,7 +174,6 @@ export async function run( { console.log( "Cleaning up..." ); await cleanupAllBrowsers( { verbose } ); - cleanupAllJSDOM( { verbose } ); if ( tunnel ) { await tunnel.stop(); @@ -260,6 +255,8 @@ export async function run( { } ); const options = { + browserstack, + concurrency, debug, headless, modules, @@ -269,11 +266,7 @@ export async function run( { verbose }; - if ( browserstack ) { - addBrowserStackRun( url, browser, options ); - } else { - addSeleniumRun( url, browser, options ); - } + addRun( url, browser, options ); } for ( const browser of browsers ) { @@ -288,11 +281,7 @@ export async function run( { try { console.log( `Starting Run ${ runId }...` ); - if ( browserstack ) { - await runAllBrowserStack( { verbose } ); - } else { - await runAllSelenium( { concurrency, verbose } ); - } + await runAll(); } catch ( error ) { console.error( error ); if ( !debug ) { diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js index d1680b22d..095c12214 100644 --- a/test/runner/selenium/createDriver.js +++ b/test/runner/selenium/createDriver.js @@ -7,7 +7,7 @@ import { browserSupportsHeadless } from "../lib/getBrowserString.js"; // Set script timeout to 10min const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10; -export default async function createDriver( { browserName, headless, verbose } ) { +export default async function createDriver( { browserName, headless, url, verbose } ) { const capabilities = Capabilities[ browserName ](); const prefs = new logging.Preferences(); prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL ); @@ -77,5 +77,8 @@ export default async function createDriver( { browserName, headless, verbose } ) // Increase script timeout to 10min await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } ); + // Set the first URL for the browser + await driver.get( url ); + return driver; } diff --git a/test/runner/selenium/queue.js b/test/runner/selenium/queue.js deleted file mode 100644 index 863db4d9b..000000000 --- a/test/runner/selenium/queue.js +++ /dev/null @@ -1,70 +0,0 @@ -// Build a queue that runs both browsers and modules -// in parallel when the length reaches the concurrency limit -// and refills the queue when one promise resolves. - -import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; -import { runSelenium } from "./runSelenium.js"; -import { runJSDOM } from "../jsdom.js"; - -const promises = []; -const queue = []; - -const SELENIUM_WAIT_TIME = 100; - -// Limit concurrency to 8 by default in selenium -// BrowserStack defaults to the max allowed by the plan -// More than this will log MaxListenersExceededWarning -const MAX_CONCURRENCY = 8; - -export function addSeleniumRun( url, browser, options ) { - queue.push( { url, browser, options } ); -} - -export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) { - while ( queue.length ) { - const next = queue.shift(); - const { url, browser, options } = next; - - const fullBrowser = getBrowserString( browser, options.headless ); - console.log( - `\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` + - `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...` - ); - - // Wait enough time between requests - // to give concurrency a chance to update. - // In selenium, this helps avoid undici connect timeout errors. - await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) ); - - if ( verbose ) { - console.log( `\nTests remaining: ${ queue.length + 1 }.` ); - } - - let promise; - if ( browser.browser === "jsdom" ) { - promise = runJSDOM( url, options ); - } else { - promise = runSelenium( url, browser, options ); - } - - // Remove the promise from the list when it resolves - promise.then( () => { - const index = promises.indexOf( promise ); - if ( index !== -1 ) { - promises.splice( index, 1 ); - } - } ); - - // Add the promise to the list - promises.push( promise ); - - // Wait until at least one promise resolves - // if we've reached the concurrency limit - if ( promises.length >= concurrency ) { - await Promise.any( promises ); - } - } - - await Promise.all( promises ); -} diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js deleted file mode 100644 index 848db36c7..000000000 --- a/test/runner/selenium/runSelenium.js +++ /dev/null @@ -1,30 +0,0 @@ -import createDriver from "./createDriver.js"; - -export async function runSelenium( - url, - { browser }, - { debug, headless, verbose } = {} -) { - if ( debug && headless ) { - throw new Error( "Cannot debug in headless mode." ); - } - - const driver = await createDriver( { - browserName: browser, - headless, - verbose - } ); - - try { - await driver.get( url ); - await driver.executeScript( -`return new Promise( ( resolve ) => { - QUnit.on( "runEnd", resolve ); -} )` - ); - } finally { - if ( !debug || headless ) { - await driver.quit(); - } - } -}