Release: correct build date in verification; other improvements

- the date is actually the date of the commit *prior*
  to the tag commit, as the files are built and then committed.
- also, the CDN should still be checked for non-stable releases,
  and should use different filenames (including in the map files).
- certain files should be skipped when checking the CDN.
- removed file diffing because it ended up being far too noisy,
  making it difficult to find the info I needed.
- because the build script required an addition, release
  verification will not work until the next release.
- print all files in failure case and whether each matched
- avoid npm script log in GH release notes changelog
- exclude changelog.md from release:clean command
- separate the post-release script from release-it for now, so we
  can keep manual verification before each push. The exact command is
  printed at the ened for convenience.

Closes gh-5521
This commit is contained in:
Timmy Willison 2024-07-17 10:13:53 -04:00
parent be048a027d
commit 53ad94f319
8 changed files with 120 additions and 58 deletions

View File

@ -1,16 +1,17 @@
name: Reproducible Builds name: Reproducible Builds
on: on:
# On tags
push: push:
# On tags
tags: tags:
- '*' - '*'
# Or manually # Or manually
workflow_dispatch: workflow_dispatch:
inputs: inputs:
version: version:
description: 'Version to verify (>= 4.0.0-beta.2)' description: 'Version to verify (>= 4.0.0-rc.1)'
required: false required: false
jobs: jobs:
run: run:
name: Verify release name: Verify release
@ -28,6 +29,9 @@ jobs:
with: with:
node-version: ${{ env.NODE_VERSION }} node-version: ${{ env.NODE_VERSION }}
- name: Install dependencies
run: npm ci
- run: npm run release:verify - run: npm run release:verify
env: env:
VERSION: ${{ github.event.inputs.version || github.ref_name }} VERSION: ${{ github.event.inputs.version || github.ref_name }}

4
.gitignore vendored
View File

@ -31,8 +31,8 @@ npm-debug.log*
/test/data/qunit-fixture.js /test/data/qunit-fixture.js
# Release artifacts # Release artifacts
changelog.* changelog.html
contributors.* contributors.html
# Ignore BrowserStack testing files # Ignore BrowserStack testing files
local.log local.log

View File

@ -14,16 +14,22 @@ module.exports = {
"sed -i 's/main\\/AUTHORS.txt/${version}\\/AUTHORS.txt/' package.json", "sed -i 's/main\\/AUTHORS.txt/${version}\\/AUTHORS.txt/' package.json",
"after:bump": "cross-env VERSION=${version} npm run build:all", "after:bump": "cross-env VERSION=${version} npm run build:all",
"before:git:release": "git add -f dist/ dist-module/ changelog.md", "before:git:release": "git add -f dist/ dist-module/ changelog.md",
"after:release": `bash ./build/release/post-release.sh \${version} ${ blogURL }` "after:release": "echo 'Run the following to complete the release:' && " +
`echo './build/release/post-release.sh $\{version} ${ blogURL }'`
}, },
git: { git: {
changelog: "npm run release:changelog -- ${from} ${to}",
// Use the node script directly to avoid an npm script
// command log entry in the GH release notes
changelog: "node build/release/changelog.js ${from} ${to}",
commitMessage: "Release: ${version}", commitMessage: "Release: ${version}",
getLatestTagFromAllRefs: true, getLatestTagFromAllRefs: true,
pushRepo: "git@github.com:jquery/jquery.git",
requireBranch: "main", requireBranch: "main",
requireCleanWorkingDir: true requireCleanWorkingDir: true
}, },
github: { github: {
pushRepo: "git@github.com:jquery/jquery.git",
release: true, release: true,
tokenRef: "JQUERY_GITHUB_TOKEN" tokenRef: "JQUERY_GITHUB_TOKEN"
}, },

View File

@ -84,6 +84,14 @@ The release script will not run without this token.
**Note**: `preReleaseBase` is set in the npm script to `1` to ensure any pre-releases start at `.1` instead of `.0`. This does not interfere with stable releases. **Note**: `preReleaseBase` is set in the npm script to `1` to ensure any pre-releases start at `.1` instead of `.0`. This does not interfere with stable releases.
1. Run the post-release script:
```sh
./build/release/post-release.sh $VERSION $BLOG_URL
```
This will push the release files to the CDN and jquery-dist repos, and push the commit to the jQuery repo to remove the release files and update the AUTHORS.txt URL in the package.json.
1. Once the release is complete, publish the blog post. 1. Once the release is complete, publish the blog post.
## Stable releases ## Stable releases

View File

@ -51,7 +51,7 @@ git add package.json
# Leave the tmp folder as some files are needed # Leave the tmp folder as some files are needed
# after the release (such as for emailing archives). # after the release (such as for emailing archives).
npm run build:clean npm run build:clean
git rm --cached -r dist/ dist-module git rm --cached -r dist/ dist-module/
git add dist/package.json dist/wrappers dist-module/package.json dist-module/wrappers git add dist/package.json dist/wrappers dist-module/package.json dist-module/wrappers
git commit -m "Release: remove dist files from main branch" git commit -m "Release: remove dist files from main branch"

View File

@ -1,9 +1,6 @@
/** /**
* Verify the latest release is reproducible * Verify the latest release is reproducible
* Works with versions 4.0.0-beta.2 and later
*/ */
import chalk from "chalk";
import * as Diff from "diff";
import { exec as nodeExec } from "node:child_process"; import { exec as nodeExec } from "node:child_process";
import crypto from "node:crypto"; import crypto from "node:crypto";
import { createWriteStream } from "node:fs"; import { createWriteStream } from "node:fs";
@ -22,7 +19,12 @@ const SRC_REPO = "https://github.com/jquery/jquery.git";
const CDN_URL = "https://code.jquery.com"; const CDN_URL = "https://code.jquery.com";
const REGISTRY_URL = "https://registry.npmjs.org/jquery"; const REGISTRY_URL = "https://registry.npmjs.org/jquery";
const rstable = /^(\d+\.\d+\.\d+)$/; const excludeFromCDN = [
/^package\.json$/,
/^jquery\.factory\./
];
const rjquery = /^jquery/;
async function verifyRelease( { version } = {} ) { async function verifyRelease( { version } = {} ) {
if ( !version ) { if ( !version ) {
@ -33,24 +35,34 @@ async function verifyRelease( { version } = {} ) {
console.log( `Verifying jQuery ${ version }...` ); console.log( `Verifying jQuery ${ version }...` );
let verified = true; let verified = true;
const matchingFiles = [];
const mismatchingFiles = [];
// Only check stable versions against the CDN // Check all files against the CDN
if ( rstable.test( version ) ) {
await Promise.all( await Promise.all(
release.files.map( async( file ) => { release.files
const cdnContents = await fetch( new URL( file.name, CDN_URL ) ).then( .filter( ( file ) => excludeFromCDN.every( ( re ) => !re.test( file.name ) ) )
( res ) => res.text() .map( async( file ) => {
const url = new URL( file.cdnName, CDN_URL );
const response = await fetch( url );
if ( !response.ok ) {
throw new Error(
`Failed to download ${
file.cdnName
} from the CDN: ${ response.statusText }`
); );
if ( cdnContents !== file.contents ) { }
console.log( `${ file.name } is different from the CDN:` ); const cdnContents = await response.text();
diffFiles( file.contents, cdnContents ); if ( cdnContents !== file.cdnContents ) {
mismatchingFiles.push( url.href );
verified = false; verified = false;
} else {
matchingFiles.push( url.href );
} }
} ) } )
); );
}
// Check all releases against npm. // Check all files against npm.
// First, download npm tarball for version // First, download npm tarball for version
const npmPackage = await fetch( REGISTRY_URL ).then( ( res ) => res.json() ); const npmPackage = await fetch( REGISTRY_URL ).then( ( res ) => res.json() );
@ -66,9 +78,10 @@ async function verifyRelease( { version } = {} ) {
// Check the tarball checksum // Check the tarball checksum
const tgzSum = await sumTarball( npmTarballPath ); const tgzSum = await sumTarball( npmTarballPath );
if ( tgzSum !== release.tgz.contents ) { if ( tgzSum !== release.tgz.contents ) {
console.log( `${ version }.tgz is different from npm:` ); mismatchingFiles.push( `npm:${ version }.tgz` );
diffFiles( release.tgz.contents, tgzSum );
verified = false; verified = false;
} else {
matchingFiles.push( `npm:${ version }.tgz` );
} }
await Promise.all( await Promise.all(
@ -80,16 +93,26 @@ async function verifyRelease( { version } = {} ) {
); );
if ( npmContents !== file.contents ) { if ( npmContents !== file.contents ) {
console.log( `${ file.name } is different from the CDN:` ); mismatchingFiles.push( `npm:${ file.path }/${ file.name }` );
diffFiles( file.contents, npmContents );
verified = false; verified = false;
} else {
matchingFiles.push( `npm:${ file.path }/${ file.name }` );
} }
} ) } )
); );
if ( verified ) { if ( verified ) {
console.log( `jQuery ${ version } is reproducible!` ); console.log( `jQuery ${ version } is reproducible! All files match!` );
} else { } else {
console.log();
for ( const file of matchingFiles ) {
console.log( `${ file }` );
}
console.log();
for ( const file of mismatchingFiles ) {
console.log( `${ file }` );
}
throw new Error( `jQuery ${ version } is NOT reproducible!` ); throw new Error( `jQuery ${ version } is NOT reproducible!` );
} }
} }
@ -101,19 +124,32 @@ async function buildRelease( { version } ) {
console.log( `Cloning jQuery ${ version }...` ); console.log( `Cloning jQuery ${ version }...` );
await rimraf( releaseFolder ); await rimraf( releaseFolder );
await mkdir( releaseFolder, { recursive: true } ); await mkdir( releaseFolder, { recursive: true } );
// Uses a depth of 2 so we can get the commit date of
// the commit used to build, which is the commit before the tag
await exec( await exec(
`git clone -q -b ${ version } --depth=1 ${ SRC_REPO } ${ releaseFolder }` `git clone -q -b ${ version } --depth=2 ${ SRC_REPO } ${ releaseFolder }`
); );
// Install node dependencies // Install node dependencies
console.log( `Installing dependencies for jQuery ${ version }...` ); console.log( `Installing dependencies for jQuery ${ version }...` );
await exec( "npm ci", { cwd: releaseFolder } ); await exec( "npm ci", { cwd: releaseFolder } );
// Find the date of the commit just before the release,
// which was used as the date in the built files
const { stdout: date } = await exec( "git log -1 --format=%ci HEAD~1", {
cwd: releaseFolder
} );
// Build the release // Build the release
console.log( `Building jQuery ${ version }...` ); console.log( `Building jQuery ${ version }...` );
const { stdout: buildOutput } = await exec( "npm run build:all", { const { stdout: buildOutput } = await exec( "npm run build:all", {
cwd: releaseFolder, cwd: releaseFolder,
env: { env: {
// Keep existing environment variables
...process.env,
RELEASE_DATE: date,
VERSION: version VERSION: version
} }
} ); } );
@ -125,24 +161,35 @@ async function buildRelease( { version } ) {
console.log( packOutput ); console.log( packOutput );
// Get all top-level /dist and /dist-module files // Get all top-level /dist and /dist-module files
const distFiles = await readdir( path.join( releaseFolder, "dist" ), { const distFiles = await readdir(
withFileTypes: true path.join( releaseFolder, "dist" ),
} ); { withFileTypes: true }
);
const distModuleFiles = await readdir( const distModuleFiles = await readdir(
path.join( releaseFolder, "dist-module" ), path.join( releaseFolder, "dist-module" ),
{ { withFileTypes: true }
withFileTypes: true
}
); );
const files = await Promise.all( const files = await Promise.all(
[ ...distFiles, ...distModuleFiles ] [ ...distFiles, ...distModuleFiles ]
.filter( ( dirent ) => dirent.isFile() ) .filter( ( dirent ) => dirent.isFile() )
.map( async( dirent ) => ( { .map( async( dirent ) => {
const contents = await readFile(
path.join( dirent.parentPath, dirent.name ),
"utf8"
);
return {
name: dirent.name, name: dirent.name,
path: path.basename( dirent.parentPath ), path: path.basename( dirent.parentPath ),
contents: await readFile( path.join( dirent.parentPath, dirent.name ), "utf8" ) contents,
} ) ) cdnName: dirent.name.replace( rjquery, `jquery-${ version }` ),
cdnContents: dirent.name.endsWith( ".map" ) ?
// The CDN has versioned filenames in the maps
convertMapToVersioned( contents, version ) :
contents
};
} )
); );
// Get checksum of the tarball // Get checksum of the tarball
@ -166,20 +213,6 @@ async function downloadFile( url, dest ) {
return finished( stream ); return finished( stream );
} }
async function diffFiles( a, b ) {
const diff = Diff.diffLines( a, b );
diff.forEach( ( part ) => {
if ( part.added ) {
console.log( chalk.green( part.value ) );
} else if ( part.removed ) {
console.log( chalk.red( part.value ) );
} else {
console.log( part.value );
}
} );
}
async function getLatestVersion() { async function getLatestVersion() {
const { stdout: sha } = await exec( "git rev-list --tags --max-count=1" ); const { stdout: sha } = await exec( "git rev-list --tags --max-count=1" );
const { stdout: tag } = await exec( `git describe --tags ${ sha.trim() }` ); const { stdout: tag } = await exec( `git describe --tags ${ sha.trim() }` );
@ -198,4 +231,13 @@ async function sumTarball( filepath ) {
return shasum( unzipped ); return shasum( unzipped );
} }
function convertMapToVersioned( contents, version ) {
const map = JSON.parse( contents );
return JSON.stringify( {
...map,
file: map.file.replace( rjquery, `jquery-${ version }` ),
sources: map.sources.map( ( source ) => source.replace( rjquery, `jquery-${ version }` ) )
} );
}
verifyRelease(); verifyRelease();

View File

@ -148,7 +148,10 @@ async function getLastModifiedDate() {
async function writeCompiled( { code, dir, filename, version } ) { async function writeCompiled( { code, dir, filename, version } ) {
// Use the last modified date so builds are reproducible // Use the last modified date so builds are reproducible
const date = await getLastModifiedDate(); const date = process.env.RELEASE_DATE ?
new Date( process.env.RELEASE_DATE ) :
await getLastModifiedDate();
const compiledContents = code const compiledContents = code
// Embed Version // Embed Version

View File

@ -61,8 +61,7 @@
"qunit-fixture": "node build/tasks/qunit-fixture.js", "qunit-fixture": "node build/tasks/qunit-fixture.js",
"release": "release-it", "release": "release-it",
"release:cdn": "node build/release/cdn.js", "release:cdn": "node build/release/cdn.js",
"release:changelog": "node build/release/changelog.js", "release:clean": "rimraf tmp changelog.html contributors.html",
"release:clean": "rimraf tmp --glob changelog.{md,html} contributors.html",
"release:dist": "node build/release/dist.js", "release:dist": "node build/release/dist.js",
"release:verify": "node build/release/verify.js", "release:verify": "node build/release/verify.js",
"start": "node -e \"(async () => { const { buildDefaultFiles } = await import('./build/tasks/build.js'); buildDefaultFiles({ watch: true }) })()\"", "start": "node -e \"(async () => { const { buildDefaultFiles } = await import('./build/tasks/build.js'); buildDefaultFiles({ watch: true }) })()\"",