Tests: align test runner with other repos

Close gh-2234
This commit is contained in:
Timmy Willison 2024-04-09 13:31:27 -04:00 committed by GitHub
parent 213fdbaa28
commit 4af5caed7a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 884 additions and 240 deletions

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ bower_components
node_modules node_modules
.sizecache.json .sizecache.json
package-lock.json package-lock.json
local.log

View File

@ -56,6 +56,7 @@
}, },
"devDependencies": { "devDependencies": {
"body-parser": "1.20.2", "body-parser": "1.20.2",
"browserstack-local": "1.5.5",
"commitplease": "3.2.0", "commitplease": "3.2.0",
"diff": "5.2.0", "diff": "5.2.0",
"eslint-config-jquery": "3.0.2", "eslint-config-jquery": "3.0.2",

View File

@ -7,13 +7,9 @@
{ {
"files": ["**/*"], "files": ["**/*"],
"env": { "env": {
"es6": true,
"node": true "node": true
}, },
"globals": {
"fetch": false,
"Promise": false,
"require": false
},
"parserOptions": { "parserOptions": {
"ecmaVersion": 2022, "ecmaVersion": 2022,
"sourceType": "module" "sourceType": "module"
@ -27,7 +23,8 @@
}, },
"globals": { "globals": {
"QUnit": false, "QUnit": false,
"Symbol": false "Symbol": false,
"require": false
}, },
"parserOptions": { "parserOptions": {
"ecmaVersion": 5, "ecmaVersion": 5,

View File

@ -1,4 +1,242 @@
// This list is static, so no requests are required import chalk from "chalk";
// in the command help menu. import { getBrowserString } from "./lib/getBrowserString.js";
import {
createWorker,
deleteWorker,
getAvailableSessions
} from "./browserstack/api.js";
import createDriver from "./selenium/createDriver.js";
export const browsers = [ "chrome", "ie", "firefox", "edge", "safari", "opera" ]; const workers = Object.create( null );
/**
* Keys are browser strings
* Structure of a worker:
* {
* browser: object // The browser object
* debug: boolean // Stops the worker from being cleaned up when finished
* lastTouch: number // The last time a request was received
* restarts: number // The number of times the worker has been restarted
* options: object // The options to create the worker
* url: string // The URL the worker is on
* quit: function // A function to stop the worker
* }
*/
// Acknowledge the worker within the time limit.
// BrowserStack can take much longer spinning up
// some browsers, such as iOS 15 Safari.
const ACKNOWLEDGE_INTERVAL = 1000;
const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5;
const MAX_WORKER_RESTARTS = 5;
// No report after the time limit
// should refresh the worker
const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
const WORKER_WAIT_TIME = 30000;
// Limit concurrency to 8 by default in selenium
const MAX_SELENIUM_CONCURRENCY = 8;
export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
if ( restarts > MAX_WORKER_RESTARTS ) {
throw new Error(
`Reached the maximum number of restarts for ${ chalk.yellow(
getBrowserString( browser )
) }`
);
}
const { browserstack, debug, headless, runId, tunnelId, verbose } = options;
while ( await maxWorkersReached( options ) ) {
if ( verbose ) {
console.log( "\nWaiting for available sessions..." );
}
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
}
const fullBrowser = getBrowserString( browser );
let worker;
if ( browserstack ) {
worker = await createWorker( {
...browser,
url: encodeURI( url ),
project: "jquery",
build: `Run ${ runId }`,
// This is the maximum timeout allowed
// by BrowserStack. We do this because
// we control the timeout in the runner.
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
timeout: 1800,
// Not documented in the API docs,
// but required to make local testing work.
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
"browserstack.local": true,
"browserstack.localIdentifier": tunnelId
} );
worker.quit = () => deleteWorker( worker.id );
} else {
const driver = await createDriver( {
browserName: browser.browser,
headless,
url,
verbose
} );
worker = {
quit: () => driver.quit()
};
}
worker.debug = !!debug;
worker.url = url;
worker.browser = browser;
worker.restarts = restarts;
worker.options = options;
touchBrowser( browser );
workers[ fullBrowser ] = worker;
// Wait for the worker to show up in the list
// before returning it.
return ensureAcknowledged( worker );
}
export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}
export async function setBrowserWorkerUrl( browser, url ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.url = url;
}
}
export async function restartBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
await restartWorker( worker );
}
}
/**
* Checks that all browsers have received
* a response in the given amount of time.
* If not, the worker is restarted.
*/
export async function checkLastTouches() {
for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
const options = worker.options;
if ( options.verbose ) {
console.log(
`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
RUN_WORKER_TIMEOUT / 1000 / 60
}min.`
);
}
await restartWorker( worker );
}
}
}
export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length;
if ( numRemaining ) {
try {
await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
if ( verbose ) {
console.log(
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
);
}
} catch ( error ) {
// Log the error, but do not consider the test run failed
console.error( error );
}
}
}
async function maxWorkersReached( {
browserstack,
concurrency = MAX_SELENIUM_CONCURRENCY
} ) {
if ( browserstack ) {
return ( await getAvailableSessions() ) <= 0;
}
return workers.length >= concurrency;
}
async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );
const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}
async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}
async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await worker.quit();
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}
async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
}

View File

@ -0,0 +1,332 @@
/**
* Browserstack API is documented at
* https://github.com/browserstack/api
*/
import { createAuthHeader } from "./createAuthHeader.js";
const browserstackApi = "https://api.browserstack.com";
const apiVersion = 5;
const username = process.env.BROWSERSTACK_USERNAME;
const accessKey = process.env.BROWSERSTACK_ACCESS_KEY;
// iOS has null for version numbers,
// and we do not need a similar check for OS versions.
const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/;
const rlatest = /^latest-(\d+)$/;
const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g;
async function fetchAPI( path, options = {}, versioned = true ) {
if ( !username || !accessKey ) {
throw new Error(
"BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables must be set."
);
}
const init = {
method: "GET",
...options,
headers: {
authorization: createAuthHeader( username, accessKey ),
accept: "application/json",
"content-type": "application/json",
...options.headers
}
};
const response = await fetch(
`${ browserstackApi }/${ versioned ? `${ apiVersion }/` : "" }${ path }`,
init
);
if ( !response.ok ) {
console.log(
`\n${ init.method } ${ path }`,
response.status,
response.statusText
);
throw new Error( `Error fetching ${ path }` );
}
return response.json();
}
/**
* =============================
* Browsers API
* =============================
*/
function compareVersionNumbers( a, b ) {
if ( a != null && b == null ) {
return -1;
}
if ( a == null && b != null ) {
return 1;
}
if ( a == null && b == null ) {
return 0;
}
const aParts = a.replace( rnonDigits, "" ).split( "." );
const bParts = b.replace( rnonDigits, "" ).split( "." );
if ( aParts.length > bParts.length ) {
return -1;
}
if ( aParts.length < bParts.length ) {
return 1;
}
for ( let i = 0; i < aParts.length; i++ ) {
const aPart = Number( aParts[ i ] );
const bPart = Number( bParts[ i ] );
if ( aPart < bPart ) {
return -1;
}
if ( aPart > bPart ) {
return 1;
}
}
if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) {
return -1;
}
if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) {
return 1;
}
return 0;
}
function sortBrowsers( a, b ) {
if ( a.browser < b.browser ) {
return -1;
}
if ( a.browser > b.browser ) {
return 1;
}
const browserComparison = compareVersionNumbers(
a.browser_version,
b.browser_version
);
if ( browserComparison ) {
return browserComparison;
}
if ( a.os < b.os ) {
return -1;
}
if ( a.os > b.os ) {
return 1;
}
const osComparison = compareVersionNumbers( a.os_version, b.os_version );
if ( osComparison ) {
return osComparison;
}
const deviceComparison = compareVersionNumbers( a.device, b.device );
if ( deviceComparison ) {
return deviceComparison;
}
return 0;
}
export async function getBrowsers( { flat = false } = {} ) {
const query = new URLSearchParams();
if ( flat ) {
query.append( "flat", true );
}
const browsers = await fetchAPI( `/browsers?${ query }` );
return browsers.sort( sortBrowsers );
}
function matchVersion( browserVersion, version ) {
if ( !version ) {
return false;
}
const regex = new RegExp(
`^${ version.replace( /\\/g, "\\\\" ).replace( /\./g, "\\." ) }\\b`,
"i"
);
return regex.test( browserVersion );
}
export async function filterBrowsers( filter ) {
const browsers = await getBrowsers( { flat: true } );
if ( !filter ) {
return browsers;
}
const filterBrowser = ( filter.browser ?? "" ).toLowerCase();
const filterVersion = ( filter.browser_version ?? "" ).toLowerCase();
const filterOs = ( filter.os ?? "" ).toLowerCase();
const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase();
const filterDevice = ( filter.device ?? "" ).toLowerCase();
const filteredWithoutVersion = browsers.filter( ( browser ) => {
return (
( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) &&
( !filterOs || filterOs === browser.os.toLowerCase() ) &&
( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) &&
( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() )
);
} );
if ( !filterVersion ) {
return filteredWithoutVersion;
}
if ( filterVersion.startsWith( "latest" ) ) {
const groupedByName = filteredWithoutVersion
.filter( ( b ) => rfinalVersion.test( b.browser_version ) )
.reduce( ( acc, browser ) => {
acc[ browser.browser ] = acc[ browser.browser ] ?? [];
acc[ browser.browser ].push( browser );
return acc;
}, Object.create( null ) );
const filtered = [];
for ( const group of Object.values( groupedByName ) ) {
const latest = group[ group.length - 1 ];
// Mobile devices do not have browser version.
// Skip the version check for these,
// but include the latest in the list if it made it
// through filtering.
if ( !latest.browser_version ) {
// Do not include in the list for latest-n.
if ( filterVersion === "latest" ) {
filtered.push( latest );
}
continue;
}
// Get the latest version and subtract the number from the filter,
// ignoring any patch versions, which may differ between major versions.
const num = rlatest.exec( filterVersion );
const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 );
const match = group.findLast( ( browser ) => {
return matchVersion( browser.browser_version, version.toString() );
} );
if ( match ) {
filtered.push( match );
}
}
return filtered;
}
return filteredWithoutVersion.filter( ( browser ) => {
return matchVersion( browser.browser_version, filterVersion );
} );
}
export async function listBrowsers( filter ) {
const browsers = await filterBrowsers( filter );
console.log( "Available browsers:" );
for ( const browser of browsers ) {
let message = ` ${ browser.browser }_`;
if ( browser.device ) {
message += `:${ browser.device }_`;
} else {
message += `${ browser.browser_version }_`;
}
message += `${ browser.os }_${ browser.os_version }`;
console.log( message );
}
}
export async function getLatestBrowser( filter ) {
if ( !filter.browser_version ) {
filter.browser_version = "latest";
}
const browsers = await filterBrowsers( filter );
return browsers[ browsers.length - 1 ];
}
/**
* =============================
* Workers API
* =============================
*/
/**
* A browser object may only have one of `browser` or `device` set;
* which property is set will depend on `os`.
*
* `options`: is an object with the following properties:
* `os`: The operating system.
* `os_version`: The operating system version.
* `browser`: The browser name.
* `browser_version`: The browser version.
* `device`: The device name.
* `url` (optional): Which URL to navigate to upon creation.
* `timeout` (optional): Maximum life of the worker (in seconds). Maximum value of `1800`. Specifying `0` will use the default of `300`.
* `name` (optional): Provide a name for the worker.
* `build` (optional): Group workers into a build.
* `project` (optional): Provide the project the worker belongs to.
* `resolution` (optional): Specify the screen resolution (e.g. "1024x768").
* `browserstack.local` (optional): Set to `true` to mark as local testing.
* `browserstack.video` (optional): Set to `false` to disable video recording.
* `browserstack.localIdentifier` (optional): ID of the local tunnel.
*/
export function createWorker( options ) {
return fetchAPI( "/worker", {
method: "POST",
body: JSON.stringify( options )
} );
}
/**
* Returns a worker object, if one exists, with the following properties:
* `id`: The worker id.
* `status`: A string representing the current status of the worker.
* Possible statuses: `"running"`, `"queue"`.
*/
export function getWorker( id ) {
return fetchAPI( `/worker/${ id }` );
}
export async function deleteWorker( id ) {
return fetchAPI( `/worker/${ id }`, { method: "DELETE" } );
}
export function getWorkers() {
return fetchAPI( "/workers" );
}
/**
* Stop all workers
*/
export async function stopWorkers() {
const workers = await getWorkers();
// Run each request on its own
// to avoid connect timeout errors.
console.log( `${ workers.length } workers running...` );
for ( const worker of workers ) {
try {
await deleteWorker( worker.id );
} catch ( error ) {
// Log the error, but continue trying to remove workers.
console.error( error );
}
}
console.log( "All workers stopped." );
}
/**
* =============================
* Plan API
* =============================
*/
export function getPlan() {
return fetchAPI( "/automate/plan.json", {}, false );
}
export async function getAvailableSessions() {
try {
const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] );
return plan.parallel_sessions_max_allowed - workers.length;
} catch ( error ) {
console.error( error );
return 0;
}
}

View File

@ -0,0 +1,20 @@
export function buildBrowserFromString( str ) {
const [ browser, versionOrDevice, os, osVersion ] = str.split( "_" );
// If the version starts with a colon, it's a device
if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) {
return {
browser,
device: versionOrDevice.slice( 1 ),
os,
os_version: osVersion
};
}
return {
browser,
browser_version: versionOrDevice,
os,
os_version: osVersion
};
}

View File

@ -0,0 +1,7 @@
const textEncoder = new TextEncoder();
export function createAuthHeader( username, accessKey ) {
const encoded = textEncoder.encode( `${ username }:${ accessKey }` );
const base64 = btoa( String.fromCodePoint.apply( null, encoded ) );
return `Basic ${ base64 }`;
}

View File

@ -0,0 +1,34 @@
import browserstackLocal from "browserstack-local";
export async function localTunnel( localIdentifier, opts = {} ) {
const tunnel = new browserstackLocal.Local();
return new Promise( ( resolve, reject ) => {
// https://www.browserstack.com/docs/local-testing/binary-params
tunnel.start(
{
"enable-logging-for-api": "",
localIdentifier,
...opts
},
async( error ) => {
if ( error || !tunnel.isRunning() ) {
return reject( error );
}
resolve( {
stop: function stopTunnel() {
return new Promise( ( resolve, reject ) => {
tunnel.stop( ( error ) => {
if ( error ) {
return reject( error );
}
resolve();
} );
} );
}
} );
}
);
} );
}

View File

@ -1,8 +1,10 @@
import yargs from "yargs/yargs"; import yargs from "yargs/yargs";
import { browsers } from "./browsers.js"; import { browsers } from "./flags/browsers.js";
import { suites } from "./suites.js"; import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
import { jquery } from "./flags/jquery.js";
import { suites } from "./flags/suites.js";
import { run } from "./run.js"; import { run } from "./run.js";
import { jquery } from "./jquery.js";
const argv = yargs( process.argv.slice( 2 ) ) const argv = yargs( process.argv.slice( 2 ) )
.version( false ) .version( false )
@ -40,16 +42,25 @@ const argv = yargs( process.argv.slice( 2 ) )
type: "array", type: "array",
choices: browsers, choices: browsers,
description: description:
"Run tests in a specific browser.\n" + "Run tests in a specific browser." +
"Pass multiple browsers by repeating the option.", "Pass multiple browsers by repeating the option." +
"If using BrowserStack, specify browsers using --browserstack.",
default: [ "chrome" ] default: [ "chrome" ]
} ) } )
.option( "headless", { .option( "headless", {
alias: "h", alias: "h",
type: "boolean", type: "boolean",
description: description:
"Run tests in headless mode. Cannot be used with --debug.", "Run tests in headless mode. Cannot be used with --debug or --browserstack.",
conflicts: [ "debug" ] conflicts: [ "debug", "browserstack" ]
} )
.option( "concurrency", {
alias: "c",
type: "number",
description:
"Run tests in parallel in multiple browsers. " +
"Defaults to 8 in normal mode. In browserstack mode, " +
"defaults to the maximum available under your BrowserStack plan."
} ) } )
.option( "debug", { .option( "debug", {
alias: "d", alias: "d",
@ -61,18 +72,69 @@ const argv = yargs( process.argv.slice( 2 ) )
.option( "retries", { .option( "retries", {
alias: "r", alias: "r",
type: "number", type: "number",
description: "Number of times to retry failed tests." description: "Number of times to retry failed tests by refreshing the URL."
} ) } )
.option( "concurrency", { .option( "hard-retries", {
alias: "c",
type: "number", type: "number",
description: "Run tests in parallel in multiple browsers. Defaults to 8." description:
"Number of times to retry failed tests by restarting the worker. " +
"This is in addition to the normal retries " +
"and are only used when the normal retries are exhausted."
} ) } )
.option( "verbose", { .option( "verbose", {
alias: "v", alias: "v",
type: "boolean", type: "boolean",
description: "Log additional information." description: "Log additional information."
} ) } )
.option( "browserstack", {
type: "array",
description:
"Run tests in BrowserStack.\n" +
"Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" +
"The value can be empty for the default configuration, or a string in the format of\n" +
"\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" +
"Pass multiple browsers by repeating the option.\n" +
"The --browser option is ignored when --browserstack has a value.\n" +
"Otherwise, the --browser option will be used, " +
"with the latest version/device for that browser, on a matching OS."
} )
.option( "run-id", {
type: "string",
description: "A unique identifier for the run in BrowserStack."
} )
.option( "list-browsers", {
type: "string",
description:
"List available BrowserStack browsers and exit.\n" +
"Leave blank to view all browsers or pass " +
"\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " +
"separated by an underscore to filter the list (any can be omitted).\n" +
"\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" +
"\"latest-n\" can be used to find the nth latest browser version.\n" +
"Use a colon to indicate a device.\n" +
"Examples: \"chrome__windows_10\", \"safari_latest\", " +
"\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" +
"Use quotes if spaces are necessary."
} )
.option( "stop-workers", {
type: "boolean",
description:
"WARNING: This will stop all BrowserStack workers that may exist and exit," +
"including any workers running from other projects.\n" +
"This can be used as a failsafe when there are too many stray workers."
} )
.option( "browserstack-plan", {
type: "boolean",
description: "Show BrowserStack plan information and exit."
} )
.help().argv; .help().argv;
run( argv ); if ( typeof argv.listBrowsers === "string" ) {
listBrowsers( buildBrowserFromString( argv.listBrowsers ) );
} else if ( argv.stopWorkers ) {
stopWorkers();
} else if ( argv.browserstackPlan ) {
console.log( await getPlan() );
} else {
run( argv );
}

View File

@ -1,7 +1,7 @@
import { readFile } from "node:fs/promises";
import bodyParser from "body-parser"; import bodyParser from "body-parser";
import express from "express"; import express from "express";
import bodyParserErrorHandler from "express-body-parser-error-handler"; import bodyParserErrorHandler from "express-body-parser-error-handler";
import { readFile } from "node:fs/promises";
export async function createTestServer( report ) { export async function createTestServer( report ) {
const app = express(); const app = express();

View File

@ -0,0 +1,24 @@
// This list is static, so no requests are required
// in the command help menu.
import { getBrowsers } from "../browserstack/api.js";
export const browsers = [
"chrome",
"ie",
"firefox",
"edge",
"safari",
"opera",
"yandex",
"IE Mobile",
"Android Browser",
"Mobile Safari"
];
// A function that can be used to update the above list.
export async function getAvailableBrowsers() {
const browsers = await getBrowsers( { flat: true } );
const available = [ ...new Set( browsers.map( ( { browser } ) => browser ) ) ];
return available;
}

View File

@ -1,4 +1,4 @@
export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) { export function buildTestUrl( suite, { browserstack, jquery, migrate, port, reportId } ) {
if ( !port ) { if ( !port ) {
throw new Error( "No port specified." ); throw new Error( "No port specified." );
} }
@ -17,5 +17,8 @@ export function buildTestUrl( suite, { jquery, migrate, port, reportId } ) {
query.append( "reportId", reportId ); query.append( "reportId", reportId );
} }
return `http://localhost:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`; // BrowserStack supplies a custom domain for local testing,
// which is especially necessary for iOS testing.
const host = browserstack ? "bs-local.com" : "localhost";
return `http://${ host }:${ port }/tests/unit/${ suite }/${ suite }.html?${ query }`;
} }

View File

@ -3,7 +3,6 @@ const browserMap = {
edge: "Edge", edge: "Edge",
firefox: "Firefox", firefox: "Firefox",
ie: "IE", ie: "IE",
jsdom: "JSDOM",
opera: "Opera", opera: "Opera",
safari: "Safari" safari: "Safari"
}; };

View File

@ -1,8 +1,9 @@
import chalk from "chalk"; import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js"; import { getBrowserString } from "./lib/getBrowserString.js";
import { import {
checkLastTouches, checkLastTouches,
createBrowserWorker, createBrowserWorker,
restartBrowser,
setBrowserWorkerUrl setBrowserWorkerUrl
} from "./browsers.js"; } from "./browsers.js";
@ -44,23 +45,44 @@ export function retryTest( reportId, maxRetries ) {
test.retries++; test.retries++;
if ( test.retries <= maxRetries ) { if ( test.retries <= maxRetries ) {
console.log( console.log(
`\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.suite ) }...${ `\nRetrying test ${ reportId } for ${ chalk.yellow(
test.retries test.options.suite
}` ) }...${ test.retries }`
); );
return test; return test;
} }
} }
} }
export async function hardRetryTest( reportId, maxHardRetries ) {
if ( !maxHardRetries ) {
return false;
}
const test = queue.find( ( test ) => test.id === reportId );
if ( test ) {
test.hardRetries++;
if ( test.hardRetries <= maxHardRetries ) {
console.log(
`\nHard retrying test ${ reportId } for ${ chalk.yellow(
test.options.suite
) }...${ test.hardRetries }`
);
await restartBrowser( test.browser );
return true;
}
}
return false;
}
export function addRun( url, browser, options ) { export function addRun( url, browser, options ) {
queue.push( { queue.push( {
browser, browser,
fullBrowser: getBrowserString( browser ), fullBrowser: getBrowserString( browser ),
hardRetries: 0,
id: options.reportId, id: options.reportId,
retries: 0,
url, url,
options, options,
retries: 0,
running: false running: false
} ); } );
} }

View File

@ -1,7 +1,7 @@
import chalk from "chalk"; import chalk from "chalk";
import * as Diff from "diff";
import { getBrowserString } from "./lib/getBrowserString.js"; import { getBrowserString } from "./lib/getBrowserString.js";
import { prettyMs } from "./lib/prettyMs.js"; import { prettyMs } from "./lib/prettyMs.js";
import * as Diff from "diff";
function serializeForDiff( value ) { function serializeForDiff( value ) {

View File

@ -1,13 +1,22 @@
import chalk from "chalk"; import chalk from "chalk";
import { asyncExitHook, gracefulExit } from "exit-hook"; import { asyncExitHook, gracefulExit } from "exit-hook";
import { getLatestBrowser } from "./browserstack/api.js";
import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
import { localTunnel } from "./browserstack/local.js";
import { reportEnd, reportTest } from "./reporter.js"; import { reportEnd, reportTest } from "./reporter.js";
import { createTestServer } from "./createTestServer.js"; import { createTestServer } from "./createTestServer.js";
import { buildTestUrl } from "./lib/buildTestUrl.js"; import { buildTestUrl } from "./lib/buildTestUrl.js";
import { generateHash } from "./lib/generateHash.js"; import { generateHash } from "./lib/generateHash.js";
import { getBrowserString } from "./lib/getBrowserString.js"; import { getBrowserString } from "./lib/getBrowserString.js";
import { suites as allSuites } from "./suites.js"; import { suites as allSuites } from "./flags/suites.js";
import { cleanupAllBrowsers, touchBrowser } from "./selenium/browsers.js"; import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
import { addRun, getNextBrowserTest, retryTest, runAll } from "./selenium/queue.js"; import {
addRun,
getNextBrowserTest,
hardRetryTest,
retryTest,
runAll
} from "./queue.js";
const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
@ -16,12 +25,15 @@ const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
*/ */
export async function run( { export async function run( {
browser: browserNames = [], browser: browserNames = [],
browserstack,
concurrency, concurrency,
debug, debug,
hardRetries,
headless, headless,
jquery: jquerys = [], jquery: jquerys = [],
migrate, migrate,
retries = 0, retries = 0,
runId,
suite: suites = [], suite: suites = [],
verbose verbose
} ) { } ) {
@ -40,11 +52,25 @@ export async function run( {
); );
} }
if ( verbose ) {
console.log( browserstack ? "Running in BrowserStack." : "Running locally." );
}
const errorMessages = []; const errorMessages = [];
const pendingErrors = {}; const pendingErrors = {};
// Convert browser names to browser objects // Convert browser names to browser objects
let browsers = browserNames.map( ( b ) => ( { browser: b } ) ); let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
const tunnelId = generateHash(
`${ Date.now() }-${ suites.join( ":" ) }-${ ( browserstack || [] )
.concat( browserNames )
.join( ":" ) }`
);
// A unique identifier for this run
if ( !runId ) {
runId = tunnelId;
}
// Create the test app and // Create the test app and
// hook it up to the reporter // hook it up to the reporter
@ -96,6 +122,10 @@ export async function run( {
return retry; return retry;
} }
// Return early if hardRetryTest returns true
if ( await hardRetryTest( reportId, hardRetries ) ) {
return;
}
errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
} }
@ -143,22 +173,84 @@ export async function run( {
} ); } );
} }
async function cleanup() {
console.log( "Cleaning up..." );
await cleanupAllBrowsers( { verbose } );
if ( tunnel ) {
await tunnel.stop();
if ( verbose ) {
console.log( "Stopped BrowserStackLocal." );
}
}
}
asyncExitHook( asyncExitHook(
async() => { async() => {
await cleanupAllBrowsers( { verbose } ); await cleanup();
await stopServer(); await stopServer();
}, },
{ wait: EXIT_HOOK_WAIT_TIMEOUT } { wait: EXIT_HOOK_WAIT_TIMEOUT }
); );
// Start up BrowserStackLocal
let tunnel;
if ( browserstack ) {
if ( headless ) {
console.warn(
chalk.italic(
"BrowserStack does not support headless mode. Running in normal mode."
)
);
headless = false;
}
// Convert browserstack to browser objects.
// If browserstack is an empty array, fall back
// to the browsers array.
if ( browserstack.length ) {
browsers = browserstack.map( ( b ) => {
if ( !b ) {
return browsers[ 0 ];
}
return buildBrowserFromString( b );
} );
}
// Fill out browser defaults
browsers = await Promise.all(
browsers.map( async( browser ) => {
// Avoid undici connect timeout errors
await new Promise( ( resolve ) => setTimeout( resolve, 100 ) );
const latestMatch = await getLatestBrowser( browser );
if ( !latestMatch ) {
console.error(
chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` )
);
gracefulExit( 1 );
}
return latestMatch;
} )
);
tunnel = await localTunnel( tunnelId );
if ( verbose ) {
console.log( "Started BrowserStackLocal." );
}
}
function queueRuns( suite, browser ) { function queueRuns( suite, browser ) {
const fullBrowser = getBrowserString( browser, headless ); const fullBrowser = getBrowserString( browser, headless );
for ( const jquery of jquerys ) { for ( const jquery of jquerys ) {
const reportId = generateHash( `${ suite } ${ fullBrowser }` ); const reportId = generateHash( `${ suite } ${ jquery } ${ fullBrowser }` );
reports[ reportId ] = { browser, headless, jquery, migrate, suite }; reports[ reportId ] = { browser, headless, jquery, migrate, suite };
const url = buildTestUrl( suite, { const url = buildTestUrl( suite, {
browserstack,
jquery, jquery,
migrate, migrate,
port, port,
@ -166,12 +258,16 @@ export async function run( {
} ); } );
const options = { const options = {
browserstack,
concurrency,
debug, debug,
headless, headless,
jquery, jquery,
migrate, migrate,
reportId, reportId,
runId,
suite, suite,
tunnelId,
verbose verbose
}; };
@ -181,12 +277,13 @@ export async function run( {
for ( const browser of browsers ) { for ( const browser of browsers ) {
for ( const suite of suites ) { for ( const suite of suites ) {
queueRuns( suite, browser ); queueRuns( [ suite ], browser );
} }
} }
try { try {
await runAll( { concurrency, verbose } ); console.log( `Starting Run ${ runId }...` );
await runAll();
} catch ( error ) { } catch ( error ) {
console.error( error ); console.error( error );
if ( !debug ) { if ( !debug ) {
@ -213,7 +310,7 @@ export async function run( {
} }
console.log( chalk.green( "All tests passed!" ) ); console.log( chalk.green( "All tests passed!" ) );
if ( !debug ) { if ( !debug || browserstack ) {
gracefulExit( 0 ); gracefulExit( 0 );
} }
} else { } else {
@ -224,7 +321,14 @@ export async function run( {
if ( debug ) { if ( debug ) {
console.log(); console.log();
console.log( "Leaving browsers open for debugging." ); if ( browserstack ) {
console.log( "Leaving browsers with failures open for debugging." );
console.log(
"View running sessions at https://automate.browserstack.com/dashboard/v2/"
);
} else {
console.log( "Leaving browsers open for debugging." );
}
console.log( "Press Ctrl+C to exit." ); console.log( "Press Ctrl+C to exit." );
} else { } else {
gracefulExit( 1 ); gracefulExit( 1 );

View File

@ -1,200 +0,0 @@
import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js";
import createDriver from "./createDriver.js";
const workers = Object.create( null );
/**
* Keys are browser strings
* Structure of a worker:
* {
* debug: boolean, // Stops the worker from being cleaned up when finished
* id: string,
* lastTouch: number, // The last time a request was received
* url: string,
* browser: object, // The browser object
* options: object // The options to create the worker
* }
*/
// Acknowledge the worker within the time limit.
const ACKNOWLEDGE_INTERVAL = 1000;
const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 1;
const MAX_WORKER_RESTARTS = 5;
// No report after the time limit
// should refresh the worker
const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;
const WORKER_WAIT_TIME = 30000;
// Limit concurrency to 8 by default in selenium
const MAX_CONCURRENCY = 8;
export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}
async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );
const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}
async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
}
async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}
export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
if ( restarts > MAX_WORKER_RESTARTS ) {
throw new Error(
`Reached the maximum number of restarts for ${ chalk.yellow(
getBrowserString( browser )
) }`
);
}
const { concurrency = MAX_CONCURRENCY, debug, headless, verbose } = options;
while ( workers.length >= concurrency ) {
if ( verbose ) {
console.log( "\nWaiting for available sessions..." );
}
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
}
const fullBrowser = getBrowserString( browser );
const driver = await createDriver( {
browserName: browser.browser,
headless,
url,
verbose
} );
const worker = {
debug: !!debug,
driver,
url,
browser,
restarts,
options
};
worker.debug = !!debug;
worker.url = url;
worker.browser = browser;
worker.restarts = restarts;
worker.options = options;
touchBrowser( browser );
workers[ fullBrowser ] = worker;
// Wait for the worker to show up in the list
// before returning it.
return ensureAcknowledged( worker );
}
export async function setBrowserWorkerUrl( browser, url ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.url = url;
}
}
/**
* Checks that all browsers have received
* a response in the given amount of time.
* If not, the worker is restarted.
*/
export async function checkLastTouches() {
for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) {
if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) {
const options = worker.options;
if ( options.verbose ) {
console.log(
`\nNo response from ${ chalk.yellow( fullBrowser ) } in ${
RUN_WORKER_TIMEOUT / 1000 / 60
}min.`
);
}
await restartWorker( worker );
}
}
}
export async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await w.driver.quit();
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}
export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length;
if ( numRemaining ) {
try {
await Promise.all(
workersRemaining.map( ( worker ) => worker.driver.quit() )
);
if ( verbose ) {
console.log(
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
);
}
} catch ( error ) {
// Log the error, but do not consider the test run failed
console.error( error );
}
}
}