diff --git a/README.md b/README.md index e2e3c27..89eea5d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # WarpBuild Checkout This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in -replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's -bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git -alternates, so the fetch from GitHub downloads only the delta instead of the whole -repository. On a cache miss, the run hydrates the mirror once (full clone + upload); -mirrors expire on a server-configured TTL and re-hydrate. +replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` +objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA. +A later job checking out the same commit restores that snapshot and skips the fetch from +GitHub entirely — cutting request load (the main source of GitHub rate-limiting on +matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. +- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS); + everything else runs exactly like upstream. - Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. - Fail-open — any cache error degrades to stock `actions/checkout` behavior. - All fork code lives in `src/warpbuild/` diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 8525a44..cbaa272 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,8 +6,13 @@ import { afterEach, afterAll } from '@jest/globals' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' import { SKIP_NOT_WARPBUILD, + assertSafeTarMembers, computeDestinationRef, getMirrorCacheSkipReason } from '../src/warpbuild/mirror-cache.js' @@ -131,6 +136,69 @@ describe('warpbuild snapshot cache', () => { }) }) + describe('assertSafeTarMembers', () => { + async function makeTar( + dir: string, + members: string[], + writeFiles = true + ): Promise { + if (writeFiles) { + for (const m of members) { + const abs = path.join(dir, m) + await fs.promises.mkdir(path.dirname(abs), {recursive: true}) + await fs.promises.writeFile(abs, 'x') + } + } + const tar = path.join(dir, 'out.tar') + await exec.exec('tar', ['-cf', tar, '-C', dir, ...members]) + return tar + } + + it('accepts objects/ and shallow members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow']) + await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined() + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects members outside objects/ and shallow (e.g. hooks)', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit']) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects path traversal members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + await fs.promises.mkdir(path.join(dir, 'sub')) + const tar = path.join(dir, 'evil.tar') + // -P preserves the ../ member instead of stripping it. + await fs.promises.writeFile(path.join(dir, 'evil'), 'x') + await exec.exec('tar', [ + '-cPf', + tar, + '-C', + path.join(dir, 'sub'), + '../evil' + ]) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + }) + describe('computeDestinationRef', () => { it('maps the refs the fetch would have created', () => { expect(computeDestinationRef('refs/heads/main')).toBe( diff --git a/dist/index.js b/dist/index.js index 241550e..c91eee1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41915,6 +41915,28 @@ async function contribute(settings) { endGroup(); } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +async function assertSafeTarMembers(tar) { + let listing = ''; + await exec_exec('tar', ['-tf', tar], { + silent: true, + listeners: { stdout: (d) => (listing += d.toString()) } + }); + for (const raw of listing.split('\n')) { + const member = raw.trim(); + if (!member) { + continue; + } + const top = member.replace(/^\.\//, '').split('/')[0]; + if (member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow')) { + throw new Error(`unexpected snapshot tar member: ${member}`); + } + } +} async function restoreSnapshot(settings, url, sha) { const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); @@ -41926,6 +41948,7 @@ async function restoreSnapshot(settings, url, sha) { throw new Error(`snapshot download answered ${res.status}`); } await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await assertSafeTarMembers(tmpTar); await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); if (check !== 0) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 0bfd32c..dbeb476 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -171,6 +171,31 @@ export async function contribute(settings: IGitSourceSettings): Promise { } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +export async function assertSafeTarMembers(tar: string): Promise { + let listing = '' + await exec.exec('tar', ['-tf', tar], { + silent: true, + listeners: {stdout: (d: Buffer) => (listing += d.toString())} + }) + for (const raw of listing.split('\n')) { + const member = raw.trim() + if (!member) { + continue + } + const top = member.replace(/^\.\//, '').split('/')[0] + if ( + member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow') + ) { + throw new Error(`unexpected snapshot tar member: ${member}`) + } + } +} + async function restoreSnapshot( settings: IGitSourceSettings, url: string, @@ -189,6 +214,7 @@ async function restoreSnapshot( Readable.fromWeb(res.body as import('stream/web').ReadableStream), fs.createWriteStream(tmpTar) ) + await assertSafeTarMembers(tmpTar) await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) const check = await exec.exec(