mirror of
https://github.com/actions/checkout.git
synced 2026-07-08 22:25:38 +00:00
address pr comments
This commit is contained in:
parent
0dbe61b437
commit
9caa1a022a
12
README.md
12
README.md
@ -1,12 +1,14 @@
|
||||
# WarpBuild Checkout
|
||||
|
||||
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
||||
replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's
|
||||
bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git
|
||||
alternates, so the fetch from GitHub downloads only the delta instead of the whole
|
||||
repository. On a cache miss, the run hydrates the mirror once (full clone + upload);
|
||||
mirrors expire on a server-configured TTL and re-hydrate.
|
||||
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
|
||||
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
|
||||
A later job checking out the same commit restores that snapshot and skips the fetch from
|
||||
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
|
||||
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
|
||||
|
||||
- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS);
|
||||
everything else runs exactly like upstream.
|
||||
- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners.
|
||||
- Fail-open — any cache error degrades to stock `actions/checkout` behavior.
|
||||
- All fork code lives in `src/warpbuild/`
|
||||
|
||||
@ -6,8 +6,13 @@ import {
|
||||
afterEach,
|
||||
afterAll
|
||||
} from '@jest/globals'
|
||||
import * as exec from '@actions/exec'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import {
|
||||
SKIP_NOT_WARPBUILD,
|
||||
assertSafeTarMembers,
|
||||
computeDestinationRef,
|
||||
getMirrorCacheSkipReason
|
||||
} from '../src/warpbuild/mirror-cache.js'
|
||||
@ -131,6 +136,69 @@ describe('warpbuild snapshot cache', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSafeTarMembers', () => {
|
||||
async function makeTar(
|
||||
dir: string,
|
||||
members: string[],
|
||||
writeFiles = true
|
||||
): Promise<string> {
|
||||
if (writeFiles) {
|
||||
for (const m of members) {
|
||||
const abs = path.join(dir, m)
|
||||
await fs.promises.mkdir(path.dirname(abs), {recursive: true})
|
||||
await fs.promises.writeFile(abs, 'x')
|
||||
}
|
||||
}
|
||||
const tar = path.join(dir, 'out.tar')
|
||||
await exec.exec('tar', ['-cf', tar, '-C', dir, ...members])
|
||||
return tar
|
||||
}
|
||||
|
||||
it('accepts objects/ and shallow members', async () => {
|
||||
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
|
||||
try {
|
||||
const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow'])
|
||||
await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined()
|
||||
} finally {
|
||||
await fs.promises.rm(dir, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects members outside objects/ and shallow (e.g. hooks)', async () => {
|
||||
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
|
||||
try {
|
||||
const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit'])
|
||||
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
|
||||
/unexpected snapshot tar member/
|
||||
)
|
||||
} finally {
|
||||
await fs.promises.rm(dir, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects path traversal members', async () => {
|
||||
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
|
||||
try {
|
||||
await fs.promises.mkdir(path.join(dir, 'sub'))
|
||||
const tar = path.join(dir, 'evil.tar')
|
||||
// -P preserves the ../ member instead of stripping it.
|
||||
await fs.promises.writeFile(path.join(dir, 'evil'), 'x')
|
||||
await exec.exec('tar', [
|
||||
'-cPf',
|
||||
tar,
|
||||
'-C',
|
||||
path.join(dir, 'sub'),
|
||||
'../evil'
|
||||
])
|
||||
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
|
||||
/unexpected snapshot tar member/
|
||||
)
|
||||
} finally {
|
||||
await fs.promises.rm(dir, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('computeDestinationRef', () => {
|
||||
it('maps the refs the fetch would have created', () => {
|
||||
expect(computeDestinationRef('refs/heads/main')).toBe(
|
||||
|
||||
23
dist/index.js
vendored
23
dist/index.js
vendored
@ -41915,6 +41915,28 @@ async function contribute(settings) {
|
||||
endGroup();
|
||||
}
|
||||
}
|
||||
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
|
||||
// component — the tar is remote and extracted into .git, so a crafted member could
|
||||
// escape (e.g. hooks/, ../) and run code during checkout.
|
||||
async function assertSafeTarMembers(tar) {
|
||||
let listing = '';
|
||||
await exec_exec('tar', ['-tf', tar], {
|
||||
silent: true,
|
||||
listeners: { stdout: (d) => (listing += d.toString()) }
|
||||
});
|
||||
for (const raw of listing.split('\n')) {
|
||||
const member = raw.trim();
|
||||
if (!member) {
|
||||
continue;
|
||||
}
|
||||
const top = member.replace(/^\.\//, '').split('/')[0];
|
||||
if (member.startsWith('/') ||
|
||||
member.split('/').includes('..') ||
|
||||
(top !== 'objects' && top !== 'shallow')) {
|
||||
throw new Error(`unexpected snapshot tar member: ${member}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function restoreSnapshot(settings, url, sha) {
|
||||
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
|
||||
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
|
||||
@ -41926,6 +41948,7 @@ async function restoreSnapshot(settings, url, sha) {
|
||||
throw new Error(`snapshot download answered ${res.status}`);
|
||||
}
|
||||
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar));
|
||||
await assertSafeTarMembers(tmpTar);
|
||||
await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]);
|
||||
const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true });
|
||||
if (check !== 0) {
|
||||
|
||||
@ -171,6 +171,31 @@ export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
|
||||
// component — the tar is remote and extracted into .git, so a crafted member could
|
||||
// escape (e.g. hooks/, ../) and run code during checkout.
|
||||
export async function assertSafeTarMembers(tar: string): Promise<void> {
|
||||
let listing = ''
|
||||
await exec.exec('tar', ['-tf', tar], {
|
||||
silent: true,
|
||||
listeners: {stdout: (d: Buffer) => (listing += d.toString())}
|
||||
})
|
||||
for (const raw of listing.split('\n')) {
|
||||
const member = raw.trim()
|
||||
if (!member) {
|
||||
continue
|
||||
}
|
||||
const top = member.replace(/^\.\//, '').split('/')[0]
|
||||
if (
|
||||
member.startsWith('/') ||
|
||||
member.split('/').includes('..') ||
|
||||
(top !== 'objects' && top !== 'shallow')
|
||||
) {
|
||||
throw new Error(`unexpected snapshot tar member: ${member}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreSnapshot(
|
||||
settings: IGitSourceSettings,
|
||||
url: string,
|
||||
@ -189,6 +214,7 @@ async function restoreSnapshot(
|
||||
Readable.fromWeb(res.body as import('stream/web').ReadableStream),
|
||||
fs.createWriteStream(tmpTar)
|
||||
)
|
||||
await assertSafeTarMembers(tmpTar)
|
||||
await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
|
||||
|
||||
const check = await exec.exec(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user