From 53ad94f319930a5bf8cb9bd935ebd4e028741903 Mon Sep 17 00:00:00 2001 From: Timmy Willison Date: Wed, 17 Jul 2024 10:13:53 -0400 Subject: [PATCH] Release: correct build date in verification; other improvements - the date is actually the date of the commit *prior* to the tag commit, as the files are built and then committed. - also, the CDN should still be checked for non-stable releases, and should use different filenames (including in the map files). - certain files should be skipped when checking the CDN. - removed file diffing because it ended up being far too noisy, making it difficult to find the info I needed. - because the build script required an addition, release verification will not work until the next release. - print all files in failure case and whether each matched - avoid npm script log in GH release notes changelog - exclude changelog.md from release:clean command - separate the post-release script from release-it for now, so we can keep manual verification before each push. The exact command is printed at the ened for convenience. Closes gh-5521 --- .github/workflows/verify-release.yml | 8 +- .gitignore | 4 +- .release-it.cjs | 10 +- build/release/README.md | 8 ++ build/release/post-release.sh | 2 +- build/release/verify.js | 138 +++++++++++++++++---------- build/tasks/build.js | 5 +- package.json | 3 +- 8 files changed, 120 insertions(+), 58 deletions(-) diff --git a/.github/workflows/verify-release.yml b/.github/workflows/verify-release.yml index cd43efac2..1c1b191e4 100644 --- a/.github/workflows/verify-release.yml +++ b/.github/workflows/verify-release.yml @@ -1,16 +1,17 @@ name: Reproducible Builds on: - # On tags push: + # On tags tags: - '*' # Or manually workflow_dispatch: inputs: version: - description: 'Version to verify (>= 4.0.0-beta.2)' + description: 'Version to verify (>= 4.0.0-rc.1)' required: false + jobs: run: name: Verify release @@ -28,6 +29,9 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} + - name: Install dependencies + run: npm ci + - run: npm run release:verify env: VERSION: ${{ github.event.inputs.version || github.ref_name }} diff --git a/.gitignore b/.gitignore index b3cc97d99..2b984efe7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,8 +31,8 @@ npm-debug.log* /test/data/qunit-fixture.js # Release artifacts -changelog.* -contributors.* +changelog.html +contributors.html # Ignore BrowserStack testing files local.log diff --git a/.release-it.cjs b/.release-it.cjs index 1de0de548..ff55b0ef1 100644 --- a/.release-it.cjs +++ b/.release-it.cjs @@ -14,16 +14,22 @@ module.exports = { "sed -i 's/main\\/AUTHORS.txt/${version}\\/AUTHORS.txt/' package.json", "after:bump": "cross-env VERSION=${version} npm run build:all", "before:git:release": "git add -f dist/ dist-module/ changelog.md", - "after:release": `bash ./build/release/post-release.sh \${version} ${ blogURL }` + "after:release": "echo 'Run the following to complete the release:' && " + + `echo './build/release/post-release.sh $\{version} ${ blogURL }'` }, git: { - changelog: "npm run release:changelog -- ${from} ${to}", + + // Use the node script directly to avoid an npm script + // command log entry in the GH release notes + changelog: "node build/release/changelog.js ${from} ${to}", commitMessage: "Release: ${version}", getLatestTagFromAllRefs: true, + pushRepo: "git@github.com:jquery/jquery.git", requireBranch: "main", requireCleanWorkingDir: true }, github: { + pushRepo: "git@github.com:jquery/jquery.git", release: true, tokenRef: "JQUERY_GITHUB_TOKEN" }, diff --git a/build/release/README.md b/build/release/README.md index 7d050d336..9aef670a1 100644 --- a/build/release/README.md +++ b/build/release/README.md @@ -84,6 +84,14 @@ The release script will not run without this token. **Note**: `preReleaseBase` is set in the npm script to `1` to ensure any pre-releases start at `.1` instead of `.0`. This does not interfere with stable releases. +1. Run the post-release script: + + ```sh + ./build/release/post-release.sh $VERSION $BLOG_URL + ``` + + This will push the release files to the CDN and jquery-dist repos, and push the commit to the jQuery repo to remove the release files and update the AUTHORS.txt URL in the package.json. + 1. Once the release is complete, publish the blog post. ## Stable releases diff --git a/build/release/post-release.sh b/build/release/post-release.sh index 2bc4e2657..6cc4d6517 100644 --- a/build/release/post-release.sh +++ b/build/release/post-release.sh @@ -51,7 +51,7 @@ git add package.json # Leave the tmp folder as some files are needed # after the release (such as for emailing archives). npm run build:clean -git rm --cached -r dist/ dist-module +git rm --cached -r dist/ dist-module/ git add dist/package.json dist/wrappers dist-module/package.json dist-module/wrappers git commit -m "Release: remove dist files from main branch" diff --git a/build/release/verify.js b/build/release/verify.js index f167666b2..423a63c8c 100644 --- a/build/release/verify.js +++ b/build/release/verify.js @@ -1,9 +1,6 @@ /** * Verify the latest release is reproducible - * Works with versions 4.0.0-beta.2 and later */ -import chalk from "chalk"; -import * as Diff from "diff"; import { exec as nodeExec } from "node:child_process"; import crypto from "node:crypto"; import { createWriteStream } from "node:fs"; @@ -22,7 +19,12 @@ const SRC_REPO = "https://github.com/jquery/jquery.git"; const CDN_URL = "https://code.jquery.com"; const REGISTRY_URL = "https://registry.npmjs.org/jquery"; -const rstable = /^(\d+\.\d+\.\d+)$/; +const excludeFromCDN = [ + /^package\.json$/, + /^jquery\.factory\./ +]; + +const rjquery = /^jquery/; async function verifyRelease( { version } = {} ) { if ( !version ) { @@ -33,24 +35,34 @@ async function verifyRelease( { version } = {} ) { console.log( `Verifying jQuery ${ version }...` ); let verified = true; + const matchingFiles = []; + const mismatchingFiles = []; - // Only check stable versions against the CDN - if ( rstable.test( version ) ) { - await Promise.all( - release.files.map( async( file ) => { - const cdnContents = await fetch( new URL( file.name, CDN_URL ) ).then( - ( res ) => res.text() - ); - if ( cdnContents !== file.contents ) { - console.log( `${ file.name } is different from the CDN:` ); - diffFiles( file.contents, cdnContents ); + // Check all files against the CDN + await Promise.all( + release.files + .filter( ( file ) => excludeFromCDN.every( ( re ) => !re.test( file.name ) ) ) + .map( async( file ) => { + const url = new URL( file.cdnName, CDN_URL ); + const response = await fetch( url ); + if ( !response.ok ) { + throw new Error( + `Failed to download ${ + file.cdnName + } from the CDN: ${ response.statusText }` + ); + } + const cdnContents = await response.text(); + if ( cdnContents !== file.cdnContents ) { + mismatchingFiles.push( url.href ); verified = false; + } else { + matchingFiles.push( url.href ); } } ) - ); - } + ); - // Check all releases against npm. + // Check all files against npm. // First, download npm tarball for version const npmPackage = await fetch( REGISTRY_URL ).then( ( res ) => res.json() ); @@ -66,9 +78,10 @@ async function verifyRelease( { version } = {} ) { // Check the tarball checksum const tgzSum = await sumTarball( npmTarballPath ); if ( tgzSum !== release.tgz.contents ) { - console.log( `${ version }.tgz is different from npm:` ); - diffFiles( release.tgz.contents, tgzSum ); + mismatchingFiles.push( `npm:${ version }.tgz` ); verified = false; + } else { + matchingFiles.push( `npm:${ version }.tgz` ); } await Promise.all( @@ -80,16 +93,26 @@ async function verifyRelease( { version } = {} ) { ); if ( npmContents !== file.contents ) { - console.log( `${ file.name } is different from the CDN:` ); - diffFiles( file.contents, npmContents ); + mismatchingFiles.push( `npm:${ file.path }/${ file.name }` ); verified = false; + } else { + matchingFiles.push( `npm:${ file.path }/${ file.name }` ); } } ) ); if ( verified ) { - console.log( `jQuery ${ version } is reproducible!` ); + console.log( `jQuery ${ version } is reproducible! All files match!` ); } else { + console.log(); + for ( const file of matchingFiles ) { + console.log( `✅ ${ file }` ); + } + console.log(); + for ( const file of mismatchingFiles ) { + console.log( `❌ ${ file }` ); + } + throw new Error( `jQuery ${ version } is NOT reproducible!` ); } } @@ -101,19 +124,32 @@ async function buildRelease( { version } ) { console.log( `Cloning jQuery ${ version }...` ); await rimraf( releaseFolder ); await mkdir( releaseFolder, { recursive: true } ); + + // Uses a depth of 2 so we can get the commit date of + // the commit used to build, which is the commit before the tag await exec( - `git clone -q -b ${ version } --depth=1 ${ SRC_REPO } ${ releaseFolder }` + `git clone -q -b ${ version } --depth=2 ${ SRC_REPO } ${ releaseFolder }` ); // Install node dependencies console.log( `Installing dependencies for jQuery ${ version }...` ); await exec( "npm ci", { cwd: releaseFolder } ); + // Find the date of the commit just before the release, + // which was used as the date in the built files + const { stdout: date } = await exec( "git log -1 --format=%ci HEAD~1", { + cwd: releaseFolder + } ); + // Build the release console.log( `Building jQuery ${ version }...` ); const { stdout: buildOutput } = await exec( "npm run build:all", { cwd: releaseFolder, env: { + + // Keep existing environment variables + ...process.env, + RELEASE_DATE: date, VERSION: version } } ); @@ -125,24 +161,35 @@ async function buildRelease( { version } ) { console.log( packOutput ); // Get all top-level /dist and /dist-module files - const distFiles = await readdir( path.join( releaseFolder, "dist" ), { - withFileTypes: true - } ); + const distFiles = await readdir( + path.join( releaseFolder, "dist" ), + { withFileTypes: true } + ); const distModuleFiles = await readdir( path.join( releaseFolder, "dist-module" ), - { - withFileTypes: true - } + { withFileTypes: true } ); const files = await Promise.all( [ ...distFiles, ...distModuleFiles ] .filter( ( dirent ) => dirent.isFile() ) - .map( async( dirent ) => ( { - name: dirent.name, - path: path.basename( dirent.parentPath ), - contents: await readFile( path.join( dirent.parentPath, dirent.name ), "utf8" ) - } ) ) + .map( async( dirent ) => { + const contents = await readFile( + path.join( dirent.parentPath, dirent.name ), + "utf8" + ); + return { + name: dirent.name, + path: path.basename( dirent.parentPath ), + contents, + cdnName: dirent.name.replace( rjquery, `jquery-${ version }` ), + cdnContents: dirent.name.endsWith( ".map" ) ? + + // The CDN has versioned filenames in the maps + convertMapToVersioned( contents, version ) : + contents + }; + } ) ); // Get checksum of the tarball @@ -166,20 +213,6 @@ async function downloadFile( url, dest ) { return finished( stream ); } -async function diffFiles( a, b ) { - const diff = Diff.diffLines( a, b ); - - diff.forEach( ( part ) => { - if ( part.added ) { - console.log( chalk.green( part.value ) ); - } else if ( part.removed ) { - console.log( chalk.red( part.value ) ); - } else { - console.log( part.value ); - } - } ); -} - async function getLatestVersion() { const { stdout: sha } = await exec( "git rev-list --tags --max-count=1" ); const { stdout: tag } = await exec( `git describe --tags ${ sha.trim() }` ); @@ -198,4 +231,13 @@ async function sumTarball( filepath ) { return shasum( unzipped ); } +function convertMapToVersioned( contents, version ) { + const map = JSON.parse( contents ); + return JSON.stringify( { + ...map, + file: map.file.replace( rjquery, `jquery-${ version }` ), + sources: map.sources.map( ( source ) => source.replace( rjquery, `jquery-${ version }` ) ) + } ); +} + verifyRelease(); diff --git a/build/tasks/build.js b/build/tasks/build.js index 1a3ed1d82..d05f7daf0 100644 --- a/build/tasks/build.js +++ b/build/tasks/build.js @@ -148,7 +148,10 @@ async function getLastModifiedDate() { async function writeCompiled( { code, dir, filename, version } ) { // Use the last modified date so builds are reproducible - const date = await getLastModifiedDate(); + const date = process.env.RELEASE_DATE ? + new Date( process.env.RELEASE_DATE ) : + await getLastModifiedDate(); + const compiledContents = code // Embed Version diff --git a/package.json b/package.json index 36bc85462..9f29c124e 100644 --- a/package.json +++ b/package.json @@ -61,8 +61,7 @@ "qunit-fixture": "node build/tasks/qunit-fixture.js", "release": "release-it", "release:cdn": "node build/release/cdn.js", - "release:changelog": "node build/release/changelog.js", - "release:clean": "rimraf tmp --glob changelog.{md,html} contributors.html", + "release:clean": "rimraf tmp changelog.html contributors.html", "release:dist": "node build/release/dist.js", "release:verify": "node build/release/verify.js", "start": "node -e \"(async () => { const { buildDefaultFiles } = await import('./build/tasks/build.js'); buildDefaultFiles({ watch: true }) })()\"",