diff --git a/.github/workflows/browserstack.yml b/.github/workflows/browserstack.yml index 7350ca36d..b3f9a5e98 100644 --- a/.github/workflows/browserstack.yml +++ b/.github/workflows/browserstack.yml @@ -16,24 +16,25 @@ jobs: name: ${{ matrix.BROWSER }} concurrency: group: ${{ github.workflow }} ${{ matrix.BROWSER }} + timeout-minutes: 30 strategy: fail-fast: false matrix: BROWSER: - 'IE_11' - - 'Safari_17' - - 'Safari_16' - - 'Chrome_120' - - 'Chrome_119' - - 'Edge_120' - - 'Edge_119' - - 'Firefox_121' - - 'Firefox_120' + - 'Safari_latest' + - 'Safari_latest-1' + - 'Chrome_latest' + - 'Chrome_latest-1' + - 'Opera_latest' + - 'Edge_latest' + - 'Edge_latest-1' + - 'Firefox_latest' + - 'Firefox_latest-1' - 'Firefox_115' - '__iOS_17' - '__iOS_16' - '__iOS_15' - - 'Opera_106' steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -61,4 +62,4 @@ jobs: run: npm run pretest - name: Run tests - run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --retries 3 + run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --run-id ${{ github.run_id }} --isolate --retries 3 diff --git a/eslint.config.js b/eslint.config.js index dabff3157..952d39c71 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -21,8 +21,7 @@ export default [ "test/node_smoke_tests/commonjs/**", "test/node_smoke_tests/module/**", "test/promises_aplus_adapters/**", - "test/middleware-mockserver.cjs", - "test/runner/**/*.js" + "test/middleware-mockserver.cjs" ], languageOptions: { globals: { @@ -35,13 +34,6 @@ export default [ } }, - { - files: [ "test/runner/listeners.js" ], - languageOptions: { - sourceType: "script" - } - }, - // Source { files: [ "src/**" ], @@ -222,6 +214,29 @@ export default [ } }, + { + files: [ + "test/runner/**/*.js" + ], + languageOptions: { + globals: { + ...globals.node + }, + sourceType: "module" + }, + rules: { + ...jqueryConfig.rules + } + }, + + { + files: [ "test/runner/listeners.js" ], + languageOptions: { + ecmaVersion: 5, + sourceType: "script" + } + }, + { files: [ "test/data/testrunner.js", diff --git a/package.json b/package.json index c976f42ca..e27ea8a40 100644 --- a/package.json +++ b/package.json @@ -54,20 +54,20 @@ "pretest": "npm run qunit-fixture && npm run babel:tests && npm run npmcopy", "qunit-fixture": "node build/tasks/qunit-fixture.js", "start": "node -e \"require('./build/tasks/build.js').buildDefaultFiles({ watch: true })\"", - "test:browser": "npm run pretest && npm run build:main && npm run test:unit -- -b chrome -b firefox --no-isolate -h", + "test:browser": "npm run pretest && npm run build:main && npm run test:unit -- -b chrome -b firefox -h", "test:browserless": "npm run pretest && npm run build:all && node build/tasks/node_smoke_tests.js && node build/tasks/promises_aplus_tests.js && npm run test:unit -- -b jsdom -m basic", "test:jsdom": "npm run pretest && npm run build:main && npm run test:unit -- -b jsdom -m basic", "test:node_smoke_tests": "npm run pretest && npm run build:all && node build/tasks/node_smoke_tests.js", "test:promises_aplus": "npm run build:main && node build/tasks/promises_aplus_tests.js", - "test:firefox": "npm run pretest && npm run build:main && npm run test:unit -- -v -b firefox --no-isolate -h", - "test:safari": "npm run pretest && npm run build:main && npm run test:unit -- -b safari --no-isolate", + "test:firefox": "npm run pretest && npm run build:main && npm run test:unit -- -v -b firefox -h", + "test:safari": "npm run pretest && npm run build:main && npm run test:unit -- -b safari", "test:server": "node test/runner/server.js", - "test:esm": "npm run pretest && npm run build:main && npm run test:unit -- --esm --no-isolate -h", - "test:no-deprecated": "npm run pretest && npm run build -- -e deprecated && npm run test:unit -- --no-isolate -h", - "test:selector-native": "npm run pretest && npm run build -- -e selector && npm run test:unit -- --no-isolate -h", - "test:slim": "npm run pretest && npm run build -- --slim && npm run test:unit -- --no-isolate -h", + "test:esm": "npm run pretest && npm run build:main && npm run test:unit -- --esm -h", + "test:no-deprecated": "npm run pretest && npm run build -- -e deprecated && npm run test:unit -- -h", + "test:selector-native": "npm run pretest && npm run build -- -e selector && npm run test:unit -- -h", + "test:slim": "npm run pretest && npm run build -- --slim && npm run test:unit -- -h", "test:unit": "node test/runner/command.js", - "test": "npm run build:all && npm run lint && npm run test:browserless && npm run test:browser && npm run test:esmodules && npm run test:slim && npm run test:no-deprecated && npm run test:selector-native" + "test": "npm run build:all && npm run lint && npm run test:browserless && npm run test:browser && npm run test:esm && npm run test:slim && npm run test:no-deprecated && npm run test:selector-native" }, "homepage": "https://jquery.com", "author": { diff --git a/test/data/testinit.js b/test/data/testinit.js index a04bc49c7..80c1911e4 100644 --- a/test/data/testinit.js +++ b/test/data/testinit.js @@ -433,7 +433,6 @@ this.loadTests = function() { } } else { - QUnit.load(); /** * Run in noConflict mode diff --git a/test/index.html b/test/index.html index 523cc1eb2..87fe94164 100644 --- a/test/index.html +++ b/test/index.html @@ -35,7 +35,7 @@ // We need to read both. var esmodules = QUnit.config.esmodules || QUnit.urlParams.esmodules; - // `loadTests()` will call `QUnit.load()` because tests + // `loadTests()` will call `QUnit.start()` because tests // such as unit/ready.js should run after document ready. if ( !esmodules ) { loadTests(); diff --git a/test/runner/browserstack/api.js b/test/runner/browserstack/api.js index 40982c9b3..632f90c3b 100644 --- a/test/runner/browserstack/api.js +++ b/test/runner/browserstack/api.js @@ -14,6 +14,7 @@ const accessKey = process.env.BROWSERSTACK_ACCESS_KEY; // iOS has null for version numbers, // and we do not need a similar check for OS versions. const rfinalVersion = /(?:^[0-9\.]+$)|(?:^null$)/; +const rlatest = /^latest-(\d+)$/; const rnonDigits = /(?:[^\d\.]+)|(?:20\d{2})/g; @@ -84,6 +85,15 @@ function compareVersionNumbers( a, b ) { return 1; } } + + if ( rnonDigits.test( a ) && !rnonDigits.test( b ) ) { + return -1; + } + if ( !rnonDigits.test( a ) && rnonDigits.test( b ) ) { + return 1; + } + + return 0; } function sortBrowsers( a, b ) { @@ -148,17 +158,62 @@ export async function filterBrowsers( filter ) { const filterOsVersion = ( filter.os_version ?? "" ).toLowerCase(); const filterDevice = ( filter.device ?? "" ).toLowerCase(); - return browsers.filter( ( browser ) => { + const filteredWithoutVersion = browsers.filter( ( browser ) => { return ( ( !filterBrowser || filterBrowser === browser.browser.toLowerCase() ) && - ( !filterVersion || - matchVersion( browser.browser_version, filterVersion ) ) && ( !filterOs || filterOs === browser.os.toLowerCase() ) && - ( !filterOsVersion || - filterOsVersion === browser.os_version.toLowerCase() ) && + ( !filterOsVersion || matchVersion( browser.os_version, filterOsVersion ) ) && ( !filterDevice || filterDevice === ( browser.device || "" ).toLowerCase() ) ); } ); + + if ( !filterVersion ) { + return filteredWithoutVersion; + } + + if ( filterVersion.startsWith( "latest" ) ) { + const groupedByName = filteredWithoutVersion + .filter( ( b ) => rfinalVersion.test( b.browser_version ) ) + .reduce( ( acc, browser ) => { + acc[ browser.browser ] = acc[ browser.browser ] ?? []; + acc[ browser.browser ].push( browser ); + return acc; + }, Object.create( null ) ); + + const filtered = []; + for ( const group of Object.values( groupedByName ) ) { + const latest = group[ group.length - 1 ]; + + // Mobile devices do not have browser version. + // Skip the version check for these, + // but include the latest in the list if it made it + // through filtering. + if ( !latest.browser_version ) { + + // Do not include in the list for latest-n. + if ( filterVersion === "latest" ) { + filtered.push( latest ); + } + continue; + } + + // Get the latest version and subtract the number from the filter, + // ignoring any patch versions, which may differ between major versions. + const num = rlatest.exec( filterVersion ); + const version = parseInt( latest.browser_version ) - ( num ? num[ 1 ] : 0 ); + const match = group.findLast( ( browser ) => { + return matchVersion( browser.browser_version, version.toString() ); + } ); + if ( match ) { + filtered.push( match ); + } + } + return filtered; + } + + return filteredWithoutVersion.filter( ( browser ) => { + return matchVersion( browser.browser_version, filterVersion ); + } ); } export async function listBrowsers( filter ) { @@ -177,13 +232,11 @@ export async function listBrowsers( filter ) { } export async function getLatestBrowser( filter ) { + if ( !filter.browser_version ) { + filter.browser_version = "latest"; + } const browsers = await filterBrowsers( filter ); - - // The list is sorted in ascending order, - // so the last item is the latest. - return browsers.findLast( ( browser ) => - rfinalVersion.test( browser.browser_version ) - ); + return browsers[ browsers.length - 1 ]; } /** @@ -229,31 +282,14 @@ export function getWorker( id ) { return fetchAPI( `/worker/${ id }` ); } -export async function deleteWorker( id, verbose ) { - await fetchAPI( `/worker/${ id }`, { method: "DELETE" } ); - if ( verbose ) { - console.log( `\nWorker ${ id } stopped.` ); - } +export async function deleteWorker( id ) { + return fetchAPI( `/worker/${ id }`, { method: "DELETE" } ); } export function getWorkers() { return fetchAPI( "/workers" ); } -/** - * Change the URL of a worker, - * or refresh if it's the same URL. - */ -export function changeUrl( id, url ) { - return fetchAPI( `/worker/${ id }/url.json`, { - method: "PUT", - body: JSON.stringify( { - timeout: 20, - url: encodeURI( url ) - } ) - } ); -} - /** * Stop all workers */ @@ -262,15 +298,17 @@ export async function stopWorkers() { // Run each request on its own // to avoid connect timeout errors. + console.log( `${ workers.length } workers running...` ); for ( const worker of workers ) { try { - await deleteWorker( worker.id, true ); + await deleteWorker( worker.id ); } catch ( error ) { // Log the error, but continue trying to remove workers. console.error( error ); } } + console.log( "All workers stopped." ); } /** @@ -284,6 +322,11 @@ export function getPlan() { } export async function getAvailableSessions() { - const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] ); - return plan.parallel_sessions_max_allowed - workers.length; + try { + const [ plan, workers ] = await Promise.all( [ getPlan(), getWorkers() ] ); + return plan.parallel_sessions_max_allowed - workers.length; + } catch ( error ) { + console.error( error ); + return 0; + } } diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js new file mode 100644 index 000000000..957c9aac8 --- /dev/null +++ b/test/runner/browserstack/browsers.js @@ -0,0 +1,199 @@ +import chalk from "chalk"; +import { getBrowserString } from "../lib/getBrowserString.js"; +import { createWorker, deleteWorker, getAvailableSessions } from "./api.js"; + +const workers = Object.create( null ); + +/** + * Keys are browser strings + * Structure of a worker: + * { + * debug: boolean, // Stops the worker from being cleaned up when finished + * id: string, + * lastTouch: number, // The last time a request was received + * url: string, + * browser: object, // The browser object + * options: object // The options to create the worker + * } + */ + +// Acknowledge the worker within the time limit. +// BrowserStack can take much longer spinning up +// some browsers, such as iOS 15 Safari. +const ACKNOWLEDGE_INTERVAL = 1000; +const ACKNOWLEDGE_TIMEOUT = 60 * 1000 * 5; + +const MAX_WORKER_RESTARTS = 5; + +// No report after the time limit +// should refresh the worker +const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; + +const WORKER_WAIT_TIME = 30000; + +export function touchBrowser( browser ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + worker.lastTouch = Date.now(); + } +} + +async function waitForAck( worker, { fullBrowser, verbose } ) { + delete worker.lastTouch; + return new Promise( ( resolve, reject ) => { + const interval = setInterval( () => { + if ( worker.lastTouch ) { + if ( verbose ) { + console.log( `\n${ fullBrowser } acknowledged.` ); + } + clearTimeout( timeout ); + clearInterval( interval ); + resolve(); + } + }, ACKNOWLEDGE_INTERVAL ); + + const timeout = setTimeout( () => { + clearInterval( interval ); + reject( + new Error( + `${ fullBrowser } not acknowledged after ${ + ACKNOWLEDGE_TIMEOUT / 1000 / 60 + }min.` + ) + ); + }, ACKNOWLEDGE_TIMEOUT ); + } ); +} + +async function ensureAcknowledged( worker, restarts ) { + const fullBrowser = getBrowserString( worker.browser ); + const verbose = worker.options.verbose; + try { + await waitForAck( worker, { fullBrowser, verbose } ); + return worker; + } catch ( error ) { + console.error( error.message ); + await cleanupWorker( worker, { verbose } ); + await createBrowserWorker( + worker.url, + worker.browser, + worker.options, + restarts + 1 + ); + } +} + +export async function createBrowserWorker( url, browser, options, restarts = 0 ) { + if ( restarts > MAX_WORKER_RESTARTS ) { + throw new Error( + `Reached the maximum number of restarts for ${ chalk.yellow( + getBrowserString( browser ) + ) }` + ); + } + const verbose = options.verbose; + while ( ( await getAvailableSessions() ) <= 0 ) { + if ( verbose ) { + console.log( "\nWaiting for available sessions..." ); + } + await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); + } + + const { debug, runId, tunnelId } = options; + const fullBrowser = getBrowserString( browser ); + + const worker = await createWorker( { + ...browser, + url: encodeURI( url ), + project: "jquery", + build: `Run ${ runId }`, + + // This is the maximum timeout allowed + // by BrowserStack. We do this because + // we control the timeout in the runner. + // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 + timeout: 1800, + + // Not documented in the API docs, + // but required to make local testing work. + // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs + "browserstack.local": true, + "browserstack.localIdentifier": tunnelId + } ); + + browser.debug = !!debug; + worker.url = url; + worker.browser = browser; + worker.restarts = restarts; + worker.options = options; + touchBrowser( browser ); + workers[ fullBrowser ] = worker; + + // Wait for the worker to show up in the list + // before returning it. + return ensureAcknowledged( worker, restarts ); +} + +export async function setBrowserWorkerUrl( browser, url ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + worker.url = url; + } +} + +/** + * Checks that all browsers have received + * a response in the given amount of time. + * If not, the worker is restarted. + */ +export async function checkLastTouches() { + for ( const [ fullBrowser, worker ] of Object.entries( workers ) ) { + if ( Date.now() - worker.lastTouch > RUN_WORKER_TIMEOUT ) { + const options = worker.options; + if ( options.verbose ) { + console.log( + `\nNo response from ${ chalk.yellow( fullBrowser ) } in ${ + RUN_WORKER_TIMEOUT / 1000 / 60 + }min.` + ); + } + await cleanupWorker( worker, options ); + await createBrowserWorker( + worker.url, + worker.browser, + options, + worker.restarts + ); + } + } +} + +export async function cleanupWorker( worker, { verbose } ) { + for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { + if ( w === worker ) { + delete workers[ fullBrowser ]; + await deleteWorker( worker.id ); + if ( verbose ) { + console.log( `\nStopped ${ fullBrowser }.` ); + } + return; + } + } +} + +export async function cleanupAllBrowsers( { verbose } ) { + const workersRemaining = Object.values( workers ); + const numRemaining = workersRemaining.length; + if ( numRemaining ) { + await Promise.all( + workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) + ); + if ( verbose ) { + console.log( + `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` + ); + } + } +} diff --git a/test/runner/browserstack/buildBrowserFromString.js b/test/runner/browserstack/buildBrowserFromString.js index 55aa38053..e0d99a039 100644 --- a/test/runner/browserstack/buildBrowserFromString.js +++ b/test/runner/browserstack/buildBrowserFromString.js @@ -3,8 +3,18 @@ export function buildBrowserFromString( str ) { // If the version starts with a colon, it's a device if ( versionOrDevice && versionOrDevice.startsWith( ":" ) ) { - return { browser, device: versionOrDevice.slice( 1 ), os, os_version: osVersion }; + return { + browser, + device: versionOrDevice.slice( 1 ), + os, + os_version: osVersion + }; } - return { browser, browser_version: versionOrDevice, os, os_version: osVersion }; + return { + browser, + browser_version: versionOrDevice, + os, + os_version: osVersion + }; } diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js new file mode 100644 index 000000000..10ef14a2b --- /dev/null +++ b/test/runner/browserstack/queue.js @@ -0,0 +1,90 @@ +import chalk from "chalk"; +import { getBrowserString } from "../lib/getBrowserString.js"; +import { checkLastTouches, createBrowserWorker, setBrowserWorkerUrl } from "./browsers.js"; + +const TEST_POLL_TIMEOUT = 1000; + +const queue = []; + +export function getNextBrowserTest( reportId ) { + const index = queue.findIndex( ( test ) => test.id === reportId ); + if ( index === -1 ) { + return; + } + + // Remove the completed test from the queue + const previousTest = queue[ index ]; + queue.splice( index, 1 ); + + // Find the next test for the same browser + for ( const test of queue.slice( index ) ) { + if ( test.fullBrowser === previousTest.fullBrowser ) { + + // Set the URL for our tracking + setBrowserWorkerUrl( test.browser, test.url ); + test.running = true; + + // Return the URL for the next test. + // listeners.js will use this to set the browser URL. + return { url: test.url }; + } + } +} + +export function retryTest( reportId, maxRetries ) { + const test = queue.find( ( test ) => test.id === reportId ); + if ( test ) { + test.retries++; + if ( test.retries <= maxRetries ) { + console.log( + `Retrying test ${ reportId } for ${ chalk.yellow( + test.options.modules.join( ", " ) + ) }...` + ); + return test; + } + } +} + +export function addBrowserStackRun( url, browser, options ) { + queue.push( { + browser, + fullBrowser: getBrowserString( browser ), + id: options.reportId, + url, + options, + retries: 0, + running: false + } ); +} + +export async function runAllBrowserStack() { + return new Promise( async( resolve, reject )=> { + while ( queue.length ) { + try { + await checkLastTouches(); + } catch ( error ) { + reject( error ); + } + + // Run one test URL per browser at a time + const browsersTaken = []; + for ( const test of queue ) { + if ( browsersTaken.indexOf( test.fullBrowser ) > -1 ) { + continue; + } + browsersTaken.push( test.fullBrowser ); + if ( !test.running ) { + test.running = true; + try { + await createBrowserWorker( test.url, test.browser, test.options ); + } catch ( error ) { + reject( error ); + } + } + } + await new Promise( ( resolve ) => setTimeout( resolve, TEST_POLL_TIMEOUT ) ); + } + resolve(); + } ); +} diff --git a/test/runner/browserstack/workers.js b/test/runner/browserstack/workers.js deleted file mode 100644 index 8f0ab68f0..000000000 --- a/test/runner/browserstack/workers.js +++ /dev/null @@ -1,276 +0,0 @@ -import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; -import { changeUrl, createWorker, deleteWorker, getWorker } from "./api.js"; - -const workers = Object.create( null ); - -// Acknowledge the worker within the time limit. -// BrowserStack can take much longer spinning up -// some browsers, such as iOS 15 Safari. -const ACKNOWLEDGE_WORKER_TIMEOUT = 60 * 1000 * 8; -const ACKNOWLEDGE_WORKER_INTERVAL = 1000; - -// No report after the time limit -// should refresh the worker -const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; -const MAX_WORKER_RESTARTS = 5; -const MAX_WORKER_REFRESHES = 1; -const POLL_WORKER_TIMEOUT = 1000; - -export async function cleanupWorker( reportId, verbose ) { - const worker = workers[ reportId ]; - if ( worker ) { - try { - delete workers[ reportId ]; - await deleteWorker( worker.id, verbose ); - } catch ( error ) { - console.error( error ); - } - } -} - -export function debugWorker( reportId ) { - const worker = workers[ reportId ]; - if ( worker ) { - worker.debug = true; - } -} - -/** - * Set the last time a request was - * received related to the worker. - */ -export function touchWorker( reportId ) { - const worker = workers[ reportId ]; - if ( worker ) { - worker.lastTouch = Date.now(); - } -} - -export function retryTest( reportId, retries ) { - const worker = workers[ reportId ]; - if ( worker ) { - worker.retries ||= 0; - worker.retries++; - if ( worker.retries <= retries ) { - worker.retry = true; - console.log( `\nRetrying test ${ reportId }...${ worker.retries }` ); - return true; - } - } - return false; -} - -export async function cleanupAllWorkers( verbose ) { - const workersRemaining = Object.keys( workers ).length; - if ( workersRemaining ) { - if ( verbose ) { - console.log( - `Stopping ${ workersRemaining } stray worker${ - workersRemaining > 1 ? "s" : "" - }...` - ); - } - await Promise.all( - Object.values( workers ).map( ( worker ) => deleteWorker( worker.id, verbose ) ) - ); - } -} - -async function waitForAck( id, verbose ) { - return new Promise( ( resolve, reject ) => { - const interval = setInterval( () => { - const worker = workers[ id ]; - if ( !worker ) { - clearTimeout( timeout ); - clearInterval( interval ); - return reject( new Error( `Worker ${ id } not found.` ) ); - } - if ( worker.lastTouch ) { - if ( verbose ) { - console.log( `\nWorker ${ id } acknowledged.` ); - } - clearTimeout( timeout ); - clearInterval( interval ); - resolve(); - } - }, ACKNOWLEDGE_WORKER_INTERVAL ); - const timeout = setTimeout( () => { - clearInterval( interval ); - const worker = workers[ id ]; - reject( - new Error( - `Worker ${ - worker ? worker.id : "" - } for test ${ id } not acknowledged after ${ - ACKNOWLEDGE_WORKER_TIMEOUT / 1000 - }s.` - ) - ); - }, ACKNOWLEDGE_WORKER_TIMEOUT ); - } ); -} - -export async function runWorker( - url, - browser, - options, - restarts = 0 -) { - const { modules, reportId, runId, verbose } = options; - const worker = await createWorker( { - ...browser, - url: encodeURI( url ), - project: "jquery", - build: `Run ${ runId }`, - name: `${ modules.join( "," ) } (${ reportId })`, - - // Set the max here, so that we can - // control the timeout - timeout: 1800, - - // Not documented in the API docs, - // but required to make local testing work. - // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs - "browserstack.local": true, - "browserstack.localIdentifier": runId - } ); - - workers[ reportId ] = worker; - - const timeMessage = `\nWorker ${ - worker.id - } created for test ${ reportId } (${ chalk.yellow( getBrowserString( browser ) ) })`; - - if ( verbose ) { - console.time( timeMessage ); - } - - async function retryWorker() { - await cleanupWorker( reportId, verbose ); - if ( verbose ) { - console.log( `Retrying worker for test ${ reportId }...${ restarts + 1 }` ); - } - return runWorker( url, browser, options, restarts + 1 ); - } - - // Wait for the worker to be acknowledged - try { - await waitForAck( reportId ); - } catch ( error ) { - if ( !workers[ reportId ] ) { - - // The worker has already been cleaned up - return; - } - - if ( restarts < MAX_WORKER_RESTARTS ) { - return retryWorker(); - } - - throw error; - } - - if ( verbose ) { - console.timeEnd( timeMessage ); - } - - let refreshes = 0; - let loggedStarted = false; - return new Promise( ( resolve, reject ) => { - async function refreshWorker() { - try { - await changeUrl( worker.id, url ); - touchWorker( reportId ); - return tick(); - } catch ( error ) { - if ( !workers[ reportId ] ) { - - // The worker has already been cleaned up - return resolve(); - } - console.error( error ); - return retryWorker().then( resolve, reject ); - } - } - - async function checkWorker() { - const worker = workers[ reportId ]; - - if ( !worker || worker.debug ) { - return resolve(); - } - - let fetchedWorker; - try { - fetchedWorker = await getWorker( worker.id ); - } catch ( error ) { - return reject( error ); - } - if ( - !fetchedWorker || - ( fetchedWorker.status !== "running" && fetchedWorker.status !== "queue" ) - ) { - return resolve(); - } - - if ( verbose && !loggedStarted ) { - loggedStarted = true; - console.log( - `\nTest ${ chalk.bold( reportId ) } is ${ - worker.status === "running" ? "running" : "in the queue" - }.` - ); - console.log( ` View at ${ fetchedWorker.browser_url }.` ); - } - - // Refresh the worker if a retry is triggered - if ( worker.retry ) { - worker.retry = false; - - // Reset recovery refreshes - refreshes = 0; - return refreshWorker(); - } - - if ( worker.lastTouch > Date.now() - RUN_WORKER_TIMEOUT ) { - return tick(); - } - - refreshes++; - - if ( refreshes >= MAX_WORKER_REFRESHES ) { - if ( restarts < MAX_WORKER_RESTARTS ) { - if ( verbose ) { - console.log( - `Worker ${ worker.id } not acknowledged after ${ - ACKNOWLEDGE_WORKER_TIMEOUT / 1000 - }s.` - ); - } - return retryWorker().then( resolve, reject ); - } - await cleanupWorker( reportId, verbose ); - return reject( - new Error( - `Worker ${ worker.id } for test ${ reportId } timed out after ${ MAX_WORKER_RESTARTS } restarts.` - ) - ); - } - - if ( verbose ) { - console.log( - `\nRefreshing worker ${ worker.id } for test ${ reportId }...${ refreshes }` - ); - } - - return refreshWorker(); - } - - function tick() { - setTimeout( checkWorker, POLL_WORKER_TIMEOUT ); - } - - checkWorker(); - } ); -} diff --git a/test/runner/command.js b/test/runner/command.js index 8419625a7..83c90066a 100644 --- a/test/runner/command.js +++ b/test/runner/command.js @@ -7,6 +7,7 @@ import { run } from "./run.js"; const argv = yargs( process.argv.slice( 2 ) ) .version( false ) + .strict() .command( { command: "[options]", describe: "Run jQuery tests in a browser" @@ -48,7 +49,8 @@ const argv = yargs( process.argv.slice( 2 ) ) type: "number", description: "Run tests in parallel in multiple browsers. " + - "Defaults to 8 in normal mode. In browserstack mode, defaults to the maximum available under your BrowserStack plan." + "Defaults to 8 in normal mode. In browserstack mode, " + + "defaults to the maximum available under your BrowserStack plan." } ) .option( "debug", { alias: "d", @@ -65,21 +67,28 @@ const argv = yargs( process.argv.slice( 2 ) ) .option( "retries", { alias: "r", type: "number", - description: "Number of times to retry failed tests.", - default: 0 + description: "Number of times to retry failed tests in BrowserStack.", + implies: [ "browserstack" ] } ) - .option( "no-isolate", { + .option( "run-id", { + type: "string", + description: "A unique identifier for this run." + } ) + .option( "isolate", { type: "boolean", - description: "Run all modules in the same browser instance." + description: "Run each module by itself in the test page. This can extend testing time." } ) .option( "browserstack", { type: "array", description: - "Run tests in BrowserStack.\nRequires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" + + "Run tests in BrowserStack.\n" + + "Requires BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY environment variables.\n" + "The value can be empty for the default configuration, or a string in the format of\n" + "\"browser_[browserVersion | :device]_os_osVersion\" (see --list-browsers).\n" + - "Pass multiple browsers by repeating the option. The --browser option is ignored when --browserstack has a value.\n" + - "Otherwise, the --browser option will be used, with the latest version/device for that browser, on a matching OS." + "Pass multiple browsers by repeating the option.\n" + + "The --browser option is ignored when --browserstack has a value.\n" + + "Otherwise, the --browser option will be used, " + + "with the latest version/device for that browser, on a matching OS." } ) .option( "list-browsers", { type: "string", @@ -88,8 +97,11 @@ const argv = yargs( process.argv.slice( 2 ) ) "Leave blank to view all browsers or pass " + "\"browser_[browserVersion | :device]_os_osVersion\" with each parameter " + "separated by an underscore to filter the list (any can be omitted).\n" + + "\"latest\" can be used in place of \"browserVersion\" to find the latest version.\n" + + "\"latest-n\" can be used to find the nth latest browser version.\n" + "Use a colon to indicate a device.\n" + - "Examples: \"chrome__windows_10\", \"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" + + "Examples: \"chrome__windows_10\", \"safari_latest\", " + + "\"Mobile Safari\", \"Android Browser_:Google Pixel 8 Pro\".\n" + "Use quotes if spaces are necessary." } ) .option( "stop-workers", { diff --git a/test/runner/createTestServer.js b/test/runner/createTestServer.js index ebc6bd4bb..b78fed278 100644 --- a/test/runner/createTestServer.js +++ b/test/runner/createTestServer.js @@ -35,7 +35,11 @@ export async function createTestServer( report ) { // Bind the reporter app.post( "/api/report", bodyParser.json( { limit: "50mb" } ), ( req, res ) => { if ( report ) { - report( req.body ); + const response = report( req.body ); + if ( response ) { + res.json( response ); + return; + } } res.sendStatus( 204 ); } ); diff --git a/test/runner/jsdom.js b/test/runner/jsdom.js index d370ac348..d9ff9dda7 100644 --- a/test/runner/jsdom.js +++ b/test/runner/jsdom.js @@ -24,7 +24,7 @@ export async function runJSDOM( url, { reportId, verbose } ) { } ); } -export function cleanupJSDOM( reportId, verbose ) { +export function cleanupJSDOM( reportId, { verbose } ) { const window = windows[ reportId ]; if ( window ) { if ( window.finish ) { @@ -38,7 +38,7 @@ export function cleanupJSDOM( reportId, verbose ) { } } -export function cleanupAllJSDOM( verbose ) { +export function cleanupAllJSDOM( { verbose } ) { const windowsRemaining = Object.keys( windows ).length; if ( windowsRemaining ) { if ( verbose ) { @@ -49,7 +49,7 @@ export function cleanupAllJSDOM( verbose ) { ); } for ( const id in windows ) { - cleanupJSDOM( id, verbose ); + cleanupJSDOM( id, { verbose } ); } } } diff --git a/test/runner/listeners.js b/test/runner/listeners.js index a3c52c21e..cca2bbd62 100644 --- a/test/runner/listeners.js +++ b/test/runner/listeners.js @@ -68,6 +68,7 @@ request.open( "POST", "/api/report", true ); request.setRequestHeader( "Content-Type", "application/json" ); request.send( json ); + return request; } // Send acknowledgement to the server. @@ -83,6 +84,16 @@ // childSuites is large and unused. data.childSuites = undefined; - send( "runEnd", data ); + var request = send( "runEnd", data ); + request.onload = function() { + if ( request.status === 200 && request.responseText ) { + try { + var json = JSON.parse( request.responseText ); + window.location = json.url; + } catch ( e ) { + console.error( e ); + } + } + }; } ); } )(); diff --git a/test/runner/run.js b/test/runner/run.js index 89b0c5eb7..2c90863b0 100644 --- a/test/runner/run.js +++ b/test/runner/run.js @@ -4,20 +4,20 @@ import { getLatestBrowser } from "./browserstack/api.js"; import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; import { localTunnel } from "./browserstack/local.js"; import { reportEnd, reportTest } from "./reporter.js"; -import { - cleanupAllWorkers, - cleanupWorker, - debugWorker, - retryTest, - touchWorker -} from "./browserstack/workers.js"; import { createTestServer } from "./createTestServer.js"; import { buildTestUrl } from "./lib/buildTestUrl.js"; import { generateHash, printModuleHashes } from "./lib/generateHash.js"; import { getBrowserString } from "./lib/getBrowserString.js"; -import { addRun, runFullQueue } from "./queue.js"; import { cleanupAllJSDOM, cleanupJSDOM } from "./jsdom.js"; import { modules as allModules } from "./modules.js"; +import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js"; +import { + addBrowserStackRun, + getNextBrowserTest, + retryTest, + runAllBrowserStack +} from "./browserstack/queue.js"; +import { addSeleniumRun, runAllSelenium } from "./selenium/queue.js"; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; @@ -31,11 +31,12 @@ export async function run( { debug, esm, headless, - isolate = true, + isolate, modules = [], - retries = 3, + retries = 0, + runId, verbose -} = {} ) { +} ) { if ( !browserNames || !browserNames.length ) { browserNames = [ "chrome" ]; } @@ -57,24 +58,28 @@ export async function run( { // Convert browser names to browser objects let browsers = browserNames.map( ( b ) => ( { browser: b } ) ); - - // A unique identifier for this run - const runId = generateHash( + const tunnelId = generateHash( `${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] ) .concat( browserNames ) .join( ":" ) }` ); + // A unique identifier for this run + if ( !runId ) { + runId = tunnelId; + } + // Create the test app and // hook it up to the reporter const reports = Object.create( null ); - const app = await createTestServer( async( message ) => { + const app = await createTestServer( ( message ) => { switch ( message.type ) { case "testEnd": { const reportId = message.id; - touchWorker( reportId ); - const errors = reportTest( message.data, reportId, reports[ reportId ] ); - pendingErrors[ reportId ] ||= {}; + const report = reports[ reportId ]; + touchBrowser( report.browser ); + const errors = reportTest( message.data, reportId, report ); + pendingErrors[ reportId ] ??= Object.create( null ); if ( errors ) { pendingErrors[ reportId ][ message.data.name ] = errors; } else { @@ -85,6 +90,7 @@ export async function run( { case "runEnd": { const reportId = message.id; const report = reports[ reportId ]; + touchBrowser( report.browser ); const { failed, total } = reportEnd( message.data, message.id, @@ -92,31 +98,34 @@ export async function run( { ); report.total = total; + cleanupJSDOM( reportId, { verbose } ); + + // Handle failure if ( failed ) { - if ( !retryTest( reportId, retries ) ) { - if ( debug ) { - debugWorker( reportId ); - } - errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); - } - } else { - if ( Object.keys( pendingErrors[ reportId ] ).length ) { - console.warn( "Detected flaky tests:" ); - for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) { - console.warn( chalk.italic( chalk.gray( error ) ) ); - } - delete pendingErrors[ reportId ]; + const retry = retryTest( reportId, retries ); + if ( retry ) { + return retry; } + errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) ); + return getNextBrowserTest( reportId ); } - await cleanupWorker( reportId, verbose ); - cleanupJSDOM( reportId, verbose ); - break; + + // Handle success + if ( + pendingErrors[ reportId ] && + Object.keys( pendingErrors[ reportId ] ).length + ) { + console.warn( "Detected flaky tests:" ); + for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) { + console.warn( chalk.italic( chalk.gray( error ) ) ); + } + delete pendingErrors[ reportId ]; + } + return getNextBrowserTest( reportId ); } case "ack": { - touchWorker( message.id ); - if ( verbose ) { - console.log( `\nWorker for test ${ message.id } acknowledged.` ); - } + const report = reports[ message.id ]; + touchBrowser( report.browser ); break; } default: @@ -165,8 +174,8 @@ export async function run( { } } - await cleanupAllWorkers( verbose ); - cleanupAllJSDOM( verbose ); + await cleanupAllBrowsers( { verbose } ); + cleanupAllJSDOM( { verbose } ); } asyncExitHook( @@ -210,13 +219,16 @@ export async function run( { const latestMatch = await getLatestBrowser( browser ); if ( !latestMatch ) { - throw new Error( `Browser not found: ${ getBrowserString( browser ) }.` ); + console.error( + chalk.red( `Browser not found: ${ getBrowserString( browser ) }.` ) + ); + gracefulExit( 1 ); } return latestMatch; } ) ); - tunnel = await localTunnel( runId ); + tunnel = await localTunnel( tunnelId ); if ( verbose ) { console.log( "Started BrowserStackLocal." ); @@ -237,15 +249,21 @@ export async function run( { reportId } ); - addRun( url, browser, { + const options = { debug, headless, modules, reportId, - retries, runId, + tunnelId, verbose - } ); + }; + + if ( browserstack ) { + addBrowserStackRun( url, browser, options ); + } else { + addSeleniumRun( url, browser, options ); + } } for ( const browser of browsers ) { @@ -260,7 +278,11 @@ export async function run( { try { console.log( `Starting Run ${ runId }...` ); - await runFullQueue( { browserstack, concurrency, verbose } ); + if ( browserstack ) { + await runAllBrowserStack( { verbose } ); + } else { + await runAllSelenium( { concurrency, verbose } ); + } } catch ( error ) { console.error( error ); if ( !debug ) { diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js index 765ebb847..d1680b22d 100644 --- a/test/runner/selenium/createDriver.js +++ b/test/runner/selenium/createDriver.js @@ -55,7 +55,8 @@ export default async function createDriver( { browserName, headless, verbose } ) edgeOptions.addArguments( "--headless=new" ); if ( !browserSupportsHeadless( browserName ) ) { console.log( - `Headless mode is not supported for ${ browserName }. Running in normal mode instead.` + `Headless mode is not supported for ${ browserName }.` + + "Running in normal mode instead." ); } } diff --git a/test/runner/queue.js b/test/runner/selenium/queue.js similarity index 57% rename from test/runner/queue.js rename to test/runner/selenium/queue.js index 4c9d66d8f..863db4d9b 100644 --- a/test/runner/queue.js +++ b/test/runner/selenium/queue.js @@ -3,33 +3,25 @@ // and refills the queue when one promise resolves. import chalk from "chalk"; -import { getAvailableSessions } from "./browserstack/api.js"; -import { runWorker } from "./browserstack/workers.js"; -import { getBrowserString } from "./lib/getBrowserString.js"; -import { runSelenium } from "./selenium/runSelenium.js"; -import { runJSDOM } from "./jsdom.js"; +import { getBrowserString } from "../lib/getBrowserString.js"; +import { runSelenium } from "./runSelenium.js"; +import { runJSDOM } from "../jsdom.js"; -const queue = []; const promises = []; +const queue = []; const SELENIUM_WAIT_TIME = 100; -const BROWSERSTACK_WAIT_TIME = 5000; -const WORKER_WAIT_TIME = 30000; // Limit concurrency to 8 by default in selenium // BrowserStack defaults to the max allowed by the plan // More than this will log MaxListenersExceededWarning const MAX_CONCURRENCY = 8; -export function addRun( url, browser, options ) { +export function addSeleniumRun( url, browser, options ) { queue.push( { url, browser, options } ); } -export async function runFullQueue( { - browserstack, - concurrency: defaultConcurrency, - verbose -} ) { +export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) { while ( queue.length ) { const next = queue.shift(); const { url, browser, options } = next; @@ -43,39 +35,15 @@ export async function runFullQueue( { // Wait enough time between requests // to give concurrency a chance to update. // In selenium, this helps avoid undici connect timeout errors. - await new Promise( ( resolve ) => - setTimeout( - resolve, - browserstack ? BROWSERSTACK_WAIT_TIME : SELENIUM_WAIT_TIME - ) - ); - - const concurrency = - browserstack && !defaultConcurrency ? - await getAvailableSessions() : - defaultConcurrency || MAX_CONCURRENCY; + await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) ); if ( verbose ) { - console.log( - `\nConcurrency: ${ concurrency }. Tests remaining: ${ queue.length + 1 }.` - ); - } - - // If concurrency is 0, wait a bit and try again - if ( concurrency <= 0 ) { - if ( verbose ) { - console.log( "\nWaiting for available sessions..." ); - } - queue.unshift( next ); - await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); - continue; + console.log( `\nTests remaining: ${ queue.length + 1 }.` ); } let promise; if ( browser.browser === "jsdom" ) { promise = runJSDOM( url, options ); - } else if ( browserstack ) { - promise = runWorker( url, browser, options ); } else { promise = runSelenium( url, browser, options ); } diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js index 247cd8472..848db36c7 100644 --- a/test/runner/selenium/runSelenium.js +++ b/test/runner/selenium/runSelenium.js @@ -1,4 +1,3 @@ -import chalk from "chalk"; import createDriver from "./createDriver.js"; export async function runSelenium(