move to using sha uploads

This commit is contained in:
darshanime 2026-07-08 13:29:51 +05:30
parent 8879f31369
commit 7484b46911
5 changed files with 460 additions and 520 deletions

View File

@ -6,17 +6,12 @@ import {
afterEach, afterEach,
afterAll afterAll
} from '@jest/globals' } from '@jest/globals'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { import {
ALTERNATES_CONTENT,
SKIP_NOT_WARPBUILD, SKIP_NOT_WARPBUILD,
mirrorPath, computeDestinationRef,
getMirrorCacheSkipReason, getMirrorCacheSkipReason
writeAlternates
} from '../src/warpbuild/mirror-cache.js' } from '../src/warpbuild/mirror-cache.js'
import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js'
import {IGitSourceSettings} from '../src/git-source-settings.js' import {IGitSourceSettings} from '../src/git-source-settings.js'
const WB_ENV = [ const WB_ENV = [
@ -30,16 +25,23 @@ for (const key of WB_ENV) {
savedEnv[key] = process.env[key] savedEnv[key] = process.env[key]
} }
const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7'
function settingsFor( function settingsFor(
owner: string, overrides: Partial<IGitSourceSettings> = {}
repo: string,
serverUrl?: string
): IGitSourceSettings { ): IGitSourceSettings {
return { return {
repositoryOwner: owner, repositoryOwner: 'octocat',
repositoryName: repo, repositoryName: 'hello-world',
repositoryPath: '/tmp/does-not-matter', repositoryPath: '/tmp/does-not-matter',
githubServerUrl: serverUrl ref: 'refs/heads/main',
commit: SHA,
fetchDepth: 1,
fetchTags: false,
filter: undefined,
sparseCheckout: undefined,
lfs: false,
...overrides
} as unknown as IGitSourceSettings } as unknown as IGitSourceSettings
} }
@ -50,7 +52,7 @@ function setWarpBuildEnv(): void {
process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world'
} }
describe('warpbuild mirror cache', () => { describe('warpbuild snapshot cache', () => {
beforeEach(() => { beforeEach(() => {
setWarpBuildEnv() setWarpBuildEnv()
}) })
@ -68,130 +70,130 @@ describe('warpbuild mirror cache', () => {
describe('getMirrorCacheSkipReason', () => { describe('getMirrorCacheSkipReason', () => {
const winOnly = process.platform === 'win32' ? it.skip : it const winOnly = process.platform === 'win32' ? it.skip : it
winOnly('returns null for the workflow repo on a WarpBuild runner', () => { winOnly('returns null for the default checkout shape', () => {
expect( expect(getMirrorCacheSkipReason(settingsFor())).toBeNull()
getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world'))
).toBeNull()
}) })
it('reports a non-WarpBuild runner', () => { it('reports a non-WarpBuild runner', () => {
delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN']
expect( expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD)
getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world'))
).toBe(SKIP_NOT_WARPBUILD)
}) })
it('reports repository: inputs that are not the workflow repo', () => { it('reports repository: inputs that are not the workflow repo', () => {
expect( expect(
getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'}))
).toBe( ).toBe(
"repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'"
) )
}) })
it('reports a missing GITHUB_REPOSITORY_ID', () => { it('requires an exact commit sha', () => {
delete process.env['GITHUB_REPOSITORY_ID'] expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe(
expect( 'no exact commit sha to key on'
getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) )
).toBe('GITHUB_REPOSITORY_ID is not set') expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe(
'no exact commit sha to key on'
)
}) })
winOnly('accepts explicit github.com server urls', () => { it('only serves fetch-depth 1', () => {
expect( expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe(
getMirrorCacheSkipReason( 'fetch-depth is 0, cache only serves fetch-depth 1'
settingsFor('octocat', 'hello-world', 'https://github.com') )
) expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe(
).toBeNull() 'fetch-depth is 50, cache only serves fetch-depth 1'
)
}) })
it('reports GHES server urls', () => { it('skips fetch-tags, filters, sparse and lfs checkouts', () => {
expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe(
'fetch-tags is enabled'
)
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
'a fetch filter is configured'
)
expect( expect(
getMirrorCacheSkipReason( getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
settingsFor('octocat', 'hello-world', 'https://ghes.example.com') ).toBe('sparse checkout is configured')
) expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe(
).toBe("server 'https://ghes.example.com' is not github.com") 'lfs is enabled (lfs objects are not in the snapshot)'
)
})
winOnly('accepts sha-only checkouts (empty ref)', () => {
expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull()
})
it('skips unqualified refs', () => {
expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe(
"ref 'main' has no cacheable destination ref"
)
})
})
describe('computeDestinationRef', () => {
it('maps the refs the fetch would have created', () => {
expect(computeDestinationRef('refs/heads/main')).toBe(
'refs/remotes/origin/main'
)
expect(computeDestinationRef('refs/pull/42/merge')).toBe(
'refs/remotes/pull/42/merge'
)
expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3')
expect(computeDestinationRef('')).toBe('')
expect(computeDestinationRef('main')).toBeNull()
}) })
}) })
describe('backend api http contract', () => { describe('backend api http contract', () => {
const realFetch = globalThis.fetch const realFetch = globalThis.fetch
let lastUrl = ''
afterEach(() => { afterEach(() => {
globalThis.fetch = realFetch globalThis.fetch = realFetch
}) })
function stubFetch(status: number, body: unknown): void { function stubFetch(status: number, body: unknown): void {
globalThis.fetch = (async () => globalThis.fetch = (async (input: unknown) => {
new Response(JSON.stringify(body), {status})) as typeof fetch lastUrl = String(input)
return new Response(JSON.stringify(body), {status})
}) as typeof fetch
} }
it('maps 200 to hit', async () => { it('maps 200 to hit and keys by sha', async () => {
stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'})
const result = await lookupMirror('123') const result = await lookupSnapshot('123', SHA)
expect(result.kind).toBe('hit') expect(result.kind).toBe('hit')
if (result.kind === 'hit') { expect(lastUrl).toContain(`sha=${SHA}`)
expect(result.info.url).toBe('https://s3/x')
}
}) })
it('maps 404 to miss (hydrate)', async () => { it('maps 404 to miss (upload after the stock fetch)', async () => {
stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
expect((await lookupMirror('123')).kind).toBe('miss') expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
}) })
it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { it('maps 403 to disabled (backend kill switch)', async () => {
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupMirror('123')).kind).toBe('disabled') expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
expect((await requestUploadURL('123')).kind).toBe('disabled') expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
}) })
it('maps other statuses to error (fall back without hydrating)', async () => { it('maps other statuses and network failures to error', async () => {
stubFetch(500, {}) stubFetch(500, {})
expect((await lookupMirror('123')).kind).toBe('error') expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
expect((await requestUploadURL('123')).kind).toBe('error')
})
it('maps network failures to error', async () => {
globalThis.fetch = (async () => { globalThis.fetch = (async () => {
throw new Error('boom') throw new Error('boom')
}) as typeof fetch }) as typeof fetch
expect((await lookupMirror('123')).kind).toBe('error') expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
expect((await requestUploadURL('123')).kind).toBe('error') expect((await requestUploadURL('123', SHA)).kind).toBe('error')
}) })
it('maps 200 upload responses to ok', async () => { it('maps 200 upload responses to ok', async () => {
stubFetch(200, {url: 'https://s3/put'}) stubFetch(200, {url: 'https://s3/put'})
const result = await requestUploadURL('123') expect(await requestUploadURL('123', SHA)).toEqual({
expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) kind: 'ok',
}) url: 'https://s3/put'
}) })
describe('mirror layout', () => {
it('keeps the mirror inside .git', () => {
expect(mirrorPath('/work/repo')).toBe(
path.join('/work/repo', '.git', 'wb-mirror.git')
)
})
it('uses a relative alternates path that never leaves .git', () => {
expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n')
expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false)
})
it('writeAlternates writes the relative path into objects/info', async () => {
const workspace = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-mirror-test-')
)
try {
await writeAlternates(workspace)
const content = await fs.promises.readFile(
path.join(workspace, '.git', 'objects', 'info', 'alternates'),
'utf8'
)
expect(content).toBe(ALTERNATES_CONTENT)
} finally {
await fs.promises.rm(workspace, {recursive: true, force: true})
}
}) })
}) })
}) })

363
dist/index.js vendored
View File

@ -41695,20 +41695,11 @@ function ref_helper_select(obj, path) {
;// CONCATENATED MODULE: external "stream/promises" ;// CONCATENATED MODULE: external "stream/promises"
const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises");
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
// Thin client for backend-core's /api/v1/git-mirrors endpoints. // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// Auth is the runner verification token every WarpBuild job carries in its env // stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the
// token alone.
//
// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go):
// 200 -> use the presigned URL
// 404 -> cache miss: create the mirror (download from GitHub + tar + upload)
// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror
// creation and upload entirely, behave exactly like stock actions/checkout
// else -> transient trouble: fall back WITHOUT the mirror download
const API_TIMEOUT_MS = 10_000; const API_TIMEOUT_MS = 10_000;
function backend_api_baseUrl() { function backend_api_baseUrl() {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@ -41716,9 +41707,9 @@ function backend_api_baseUrl() {
function authHeader() { function authHeader() {
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`;
} }
async function lookupMirror(repoKey) { async function lookupSnapshot(repoKey, sha) {
try { try {
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, {
headers: { authorization: authHeader() }, headers: { authorization: authHeader() },
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
}); });
@ -41729,18 +41720,18 @@ async function lookupMirror(repoKey) {
return { kind: 'miss' }; return { kind: 'miss' };
} }
if (res.status === 403) { if (res.status === 403) {
core_debug(`[wb-mirror] download-url answered 403 (disabled)`); core_debug(`[wb-cache] download-url answered 403 (disabled)`);
return { kind: 'disabled' }; return { kind: 'disabled' };
} }
core_debug(`[wb-mirror] download-url answered ${res.status}`); core_debug(`[wb-cache] download-url answered ${res.status}`);
return { kind: 'error' }; return { kind: 'error' };
} }
catch (error) { catch (error) {
core_debug(`[wb-mirror] download-url failed: ${error}`); core_debug(`[wb-cache] download-url failed: ${error}`);
return { kind: 'error' }; return { kind: 'error' };
} }
} }
async function requestUploadURL(repoKey) { async function requestUploadURL(repoKey, sha) {
try { try {
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
method: 'POST', method: 'POST',
@ -41748,7 +41739,7 @@ async function requestUploadURL(repoKey) {
authorization: authHeader(), authorization: authHeader(),
'content-type': 'application/json' 'content-type': 'application/json'
}, },
body: JSON.stringify({ repo_key: repoKey }), body: JSON.stringify({ repo_key: repoKey, sha }),
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
}); });
if (res.status === 200) { if (res.status === 200) {
@ -41759,22 +41750,20 @@ async function requestUploadURL(repoKey) {
return { kind: 'error' }; return { kind: 'error' };
} }
if (res.status === 403) { if (res.status === 403) {
core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
return { kind: 'disabled' }; return { kind: 'disabled' };
} }
core_debug(`[wb-mirror] upload-url answered ${res.status}`); core_debug(`[wb-cache] upload-url answered ${res.status}`);
return { kind: 'error' }; return { kind: 'error' };
} }
catch (error) { catch (error) {
core_debug(`[wb-mirror] upload-url failed: ${error}`); core_debug(`[wb-cache] upload-url failed: ${error}`);
return { kind: 'error' }; return { kind: 'error' };
} }
} }
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
.js-suffixed ESM imports both follow upstream's own conventions; the import plugin
has no TS resolver configured in this repo. */
@ -41783,48 +41772,27 @@ async function requestUploadURL(repoKey) {
// WarpBuild git-mirror cache. // WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch
// // produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we // immutable (no expiry). Fail-open: any error degrades to stock behavior.
// restore it to <workspace>/.git/wb-mirror.git and point .git/objects/info/alternates at
// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and
// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers
// 404), THIS run creates the mirror: one full download of all branches + tags from
// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is
// disabled for this org (backend-driven kill switch) and we skip everything — no
// mirror creation, no upload.
//
// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path
// is relative, so it survives every container mount scheme: `container:` jobs (/__w),
// Docker container actions (/github/workspace), and `docker build COPY .`.
//
// Everything here is fail-open: any error or timeout degrades to stock actions/checkout
// behavior with a warning, never a failed checkout.
const MIRROR_DIR = 'wb-mirror.git';
const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`;
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
const UPLOAD_TIMEOUT_MS = 15 * 60_000; const UPLOAD_TIMEOUT_MS = 15 * 60_000;
// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state // "hit <sha>" | "uploaded <sha>", for e2e assertions.
// everywhere outside WarpBuild); every other reason is logged at info. const CACHE_STATE_FILE = 'wb-cache-state';
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
function mirrorPath(repositoryPath) { const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); let decision = 'off';
} // Null = attempt the cache; else a reason to log. Only the default checkout shape
// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache // is served.
// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and
// deliberately strict: anything unexpected means "behave exactly like upstream".
function getMirrorCacheSkipReason(settings) { function getMirrorCacheSkipReason(settings) {
// Only on WarpBuild runners (these are injected into every job's env there).
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
!process.env['WARPBUILD_HOST_URL']) { !process.env['WARPBUILD_HOST_URL']) {
return SKIP_NOT_WARPBUILD; return SKIP_NOT_WARPBUILD;
} }
// Linux + macOS in v1.
if (process.platform === 'win32') { if (process.platform === 'win32') {
return 'Windows is not supported by the mirror cache yet'; return 'Windows is not supported by the snapshot cache yet';
} }
// The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only
// cache when that is what is being checked out (`repository:` inputs fall back).
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '';
if (!repoKey) { if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set'; return 'GITHUB_REPOSITORY_ID is not set';
@ -41833,181 +41801,183 @@ function getMirrorCacheSkipReason(settings) {
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`;
} }
// github.com only (repo ids and mirror keys assume it).
const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, '');
if (server !== 'https://github.com') { if (server !== 'https://github.com') {
return `server '${server}' is not github.com`; return `server '${server}' is not github.com`;
} }
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on';
}
if (settings.fetchDepth !== 1) {
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`;
}
if (settings.fetchTags) {
return 'fetch-tags is enabled';
}
if (settings.filter) {
return 'a fetch filter is configured';
}
if (settings.sparseCheckout) {
return 'sparse checkout is configured';
}
if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the snapshot)';
}
if (settings.ref && computeDestinationRef(settings.ref) === null) {
return `ref '${settings.ref}' has no cacheable destination ref`;
}
return null; return null;
} }
// setup is the single upstream splice point, called right after `git init` + // The local ref the fetch would have created ('' = none needed, null = uncacheable).
// `git remote add` for a fresh repository. It never throws. function computeDestinationRef(ref) {
async function setup(settings, repositoryUrl) { if (!ref) {
return '';
}
const upper = ref.toUpperCase();
if (upper.startsWith('REFS/HEADS/')) {
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`;
}
if (upper.startsWith('REFS/PULL/')) {
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`;
}
if (upper.startsWith('REFS/TAGS/')) {
return ref;
}
return null;
}
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
async function setup(settings) {
decision = 'off';
const skipReason = getMirrorCacheSkipReason(settings); const skipReason = getMirrorCacheSkipReason(settings);
if (skipReason) { if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) { if (skipReason === SKIP_NOT_WARPBUILD) {
core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
} }
else { else {
info(`WarpBuild mirror cache skipped: ${skipReason}`); info(`WarpBuild snapshot cache skipped: ${skipReason}`);
} }
return; return false;
} }
startGroup('WarpBuild: setting up git mirror cache'); startGroup('WarpBuild: checkout snapshot cache');
try { try {
await setupInner(settings, repositoryUrl); return await setupInner(settings);
} }
catch (error) { catch (error) {
warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`);
return false;
} }
finally { finally {
endGroup(); endGroup();
} }
} }
async function setupInner(settings, repositoryUrl) { async function setupInner(settings) {
const repoKey = process.env['GITHUB_REPOSITORY_ID']; const repoKey = process.env['GITHUB_REPOSITORY_ID'];
const mirror = mirrorPath(settings.repositoryPath); const sha = settings.commit;
// A second checkout of the same repo in one job finds the mirror already in place. const lookup = await lookupSnapshot(repoKey, sha);
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) {
info('Mirror already present, reusing it');
await writeAlternates(settings.repositoryPath);
return;
}
const lookup = await lookupMirror(repoKey);
if (lookup.kind === 'disabled') { if (lookup.kind === 'disabled') {
info('Mirror cache is disabled by the backend for this organization; using standard checkout'); info('Snapshot cache is disabled by the backend for this organization');
return; return false;
} }
if (lookup.kind === 'error') { if (lookup.kind === 'error') {
info('Mirror cache backend unavailable; using standard checkout'); info('Snapshot cache backend unavailable; using standard checkout');
return; return false;
} }
if (lookup.kind === 'hit') { if (lookup.kind === 'miss') {
info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`);
if (await restoreMirror(lookup.info.url, mirror)) { decision = 'miss';
await writeAlternates(settings.repositoryPath); return false;
info('Mirror restored; the fetch below downloads only the delta');
}
// Restore failure: fall through to plain checkout. The object exists, so the
// failure was transfer-shaped — re-downloading would just repeat the pain.
return;
} }
// Miss. Probe upload authorization BEFORE the expensive clone so a disabled or info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`);
// unreachable backend never costs a wasted full mirror clone. if (!(await restoreSnapshot(settings, lookup.info.url, sha))) {
const probe = await requestUploadURL(repoKey); return false;
if (probe.kind !== 'ok') {
info(probe.kind === 'disabled'
? 'Mirror cache is disabled by the backend for this organization; using standard checkout'
: 'Mirror cache backend unavailable; skipping mirror creation');
return;
} }
info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); info('Snapshot restored; skipping the GitHub fetch entirely');
await createMirrorFromGitHub(settings, repositoryUrl, mirror); return true;
await writeAlternates(settings.repositoryPath);
// Upload failures only warn: the local mirror still accelerates THIS run.
await uploadMirror(repoKey, mirror);
} }
// writeAlternates points the workspace repo's object lookups at the mirror. The path is // Runs after checkout; uploads the fetch result on a miss. Failures only warn.
// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. async function contribute(settings) {
async function writeAlternates(repositoryPath) { if (decision !== 'miss') {
const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); return;
await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); }
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); startGroup('WarpBuild: uploading checkout snapshot');
info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); try {
await uploadSnapshot(settings);
}
catch (error) {
warning(`Snapshot upload skipped: ${error}`);
}
finally {
endGroup();
}
} }
async function restoreMirror(url, mirror) { async function restoreSnapshot(settings, url, sha) {
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
try { try {
const res = await fetch(url, { const res = await fetch(url, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
}); });
if (!res.ok || !res.body) { if (!res.ok || !res.body) {
throw new Error(`mirror download answered ${res.status}`); throw new Error(`snapshot download answered ${res.status}`);
} }
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar));
await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]);
await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true });
if (check !== 0) {
throw new Error(`restored snapshot does not contain ${sha}`);
}
// The ref the skipped fetch would have created; upstream's verification needs it.
const dstRef = computeDestinationRef(settings.ref);
if (dstRef) {
await exec_exec('git', [
'-C',
settings.repositoryPath,
'update-ref',
dstRef,
sha
]);
}
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
return true; return true;
} }
catch (error) { catch (error) {
warning(`Mirror restore failed: ${error}`); warning(`Snapshot restore failed: ${error}`);
// Never leave a partial mirror behind an alternates file — that is corruption. // Reset .git to its freshly-init state.
await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
recursive: true,
force: true
});
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
recursive: true
});
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
recursive: true
});
return false; return false;
} }
finally { finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
} }
} }
// createMirrorFromGitHub builds the bare mirror by downloading the repository from async function uploadSnapshot(settings) {
// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. const repoKey = process.env['GITHUB_REPOSITORY_ID'];
// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also const sha = settings.commit;
// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
// share of the download on PR-heavy repos). PR-triggered checkouts still work: their const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`);
// head SHA simply arrives as a small delta in the workspace fetch.
//
// The full history download itself cannot be safely avoided: a shallow or partial
// mirror behind an alternates file is an incomplete object store that git assumes is
// complete — the exact corruption Blacksmith hit and reverted. A mirror must be
// complete with respect to the refs it advertises.
async function createMirrorFromGitHub(settings, repositoryUrl, mirror) {
// Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars
// (git >= 2.31) so the credential never appears in any process's argv.
const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64');
core_setSecret(basicCredential);
const env = {};
for (const [key, value] of Object.entries(process.env)) {
if (value !== undefined) {
env[key] = value;
}
}
env['GIT_CONFIG_COUNT'] = '1';
env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader';
env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`;
await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env });
await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], {
env
});
// gc.auto=0: never let the fetch spawn a detached gc that outlives the step.
await exec_exec('git', [
'-c',
'gc.auto=0',
'-C',
mirror,
'fetch',
'--prune',
'--progress',
'origin',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*'
], { env });
}
async function uploadMirror(repoKey, mirror) {
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`);
try { try {
// Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic const members = ['objects'];
// trims; the tar stays a valid bare repo either way. if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) {
await exec_exec('tar', [ members.push('shallow');
'-cf', }
tmpTar, await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]);
'-C',
mirror,
'--exclude',
'./hooks',
'--exclude',
'./description',
'--exclude',
'./FETCH_HEAD',
'.'
]);
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
// Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the const upload = await requestUploadURL(repoKey, sha);
// backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. if (upload.kind !== 'ok') {
const fresh = await requestUploadURL(repoKey); info(upload.kind === 'disabled'
if (fresh.kind !== 'ok') { ? 'Snapshot cache is disabled by the backend; not uploading'
throw new Error(fresh.kind === 'disabled' : 'Snapshot cache backend unavailable; not uploading');
? 'mirror cache was disabled by the backend' return;
: 'upload-url unavailable');
} }
const init = { const init = {
method: 'PUT', method: 'PUT',
@ -42019,14 +41989,12 @@ async function uploadMirror(repoKey, mirror) {
duplex: 'half', duplex: 'half',
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
}; };
const res = await fetch(fresh.url, init); const res = await fetch(upload.url, init);
if (!res.ok) { if (!res.ok) {
throw new Error(`mirror upload answered ${res.status}`); throw new Error(`snapshot upload answered ${res.status}`);
} }
info(`Mirror uploaded (${size} bytes); future runs will restore it`); await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
} info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
catch (error) {
warning(`Mirror upload skipped: ${error}`);
} }
finally { finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
@ -42066,6 +42034,7 @@ async function getSource(settings) {
const git = await getGitCommandManager(settings); const git = await getGitCommandManager(settings);
endGroup(); endGroup();
let authHelper = null; let authHelper = null;
let warpbuildRestored = false;
try { try {
if (git) { if (git) {
authHelper = createAuthHelper(git, settings); authHelper = createAuthHelper(git, settings);
@ -42116,9 +42085,8 @@ async function getSource(settings) {
await git.init(objectFormat); await git.init(objectFormat);
await git.remoteAdd('origin', repositoryUrl); await git.remoteAdd('origin', repositoryUrl);
endGroup(); endGroup();
// WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
// point alternates at it so the fetch below downloads only the delta from GitHub. warpbuildRestored = await setup(settings);
await setup(settings, repositoryUrl);
} }
// Disable automatic garbage collection // Disable automatic garbage collection
startGroup('Disabling automatic garbage collection'); startGroup('Disabling automatic garbage collection');
@ -42158,7 +42126,10 @@ async function getSource(settings) {
else if (settings.sparseCheckout) { else if (settings.sparseCheckout) {
fetchOptions.filter = 'blob:none'; fetchOptions.filter = 'blob:none';
} }
if (settings.fetchDepth <= 0) { if (warpbuildRestored) {
info('Skipping fetch: checkout was restored from the snapshot cache');
}
else if (settings.fetchDepth <= 0) {
// Fetch all branches and tags // Fetch all branches and tags
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
await git.fetch(refSpec, fetchOptions); await git.fetch(refSpec, fetchOptions);
@ -42224,6 +42195,8 @@ async function getSource(settings) {
startGroup('Checking out the ref'); startGroup('Checking out the ref');
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
endGroup(); endGroup();
// WarpBuild snapshot cache: upload the fetch result on a miss
await contribute(settings);
// Submodules // Submodules
if (settings.submodules) { if (settings.submodules) {
// Temporarily override global config // Temporarily override global config

View File

@ -41,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
core.endGroup() core.endGroup()
let authHelper: gitAuthHelper.IGitAuthHelper | null = null let authHelper: gitAuthHelper.IGitAuthHelper | null = null
let warpbuildRestored = false
try { try {
if (git) { if (git) {
authHelper = gitAuthHelper.createAuthHelper(git, settings) authHelper = gitAuthHelper.createAuthHelper(git, settings)
@ -132,9 +133,8 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
await git.remoteAdd('origin', repositoryUrl) await git.remoteAdd('origin', repositoryUrl)
core.endGroup() core.endGroup()
// WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
// point alternates at it so the fetch below downloads only the delta from GitHub. warpbuildRestored = await warpbuildMirror.setup(settings)
await warpbuildMirror.setup(settings, repositoryUrl)
} }
// Disable automatic garbage collection // Disable automatic garbage collection
@ -190,7 +190,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
fetchOptions.filter = 'blob:none' fetchOptions.filter = 'blob:none'
} }
if (settings.fetchDepth <= 0) { if (warpbuildRestored) {
core.info('Skipping fetch: checkout was restored from the snapshot cache')
} else if (settings.fetchDepth <= 0) {
// Fetch all branches and tags // Fetch all branches and tags
let refSpec = refHelper.getRefSpecForAllHistory( let refSpec = refHelper.getRefSpecForAllHistory(
settings.ref, settings.ref,
@ -276,6 +278,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
core.endGroup() core.endGroup()
// WarpBuild snapshot cache: upload the fetch result on a miss
await warpbuildMirror.contribute(settings)
// Submodules // Submodules
if (settings.submodules) { if (settings.submodules) {
// Temporarily override global config // Temporarily override global config

View File

@ -1,29 +1,20 @@
/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
import * as core from '@actions/core' import * as core from '@actions/core'
// Thin client for backend-core's /api/v1/git-mirrors endpoints. // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// Auth is the runner verification token every WarpBuild job carries in its env // stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the
// token alone.
//
// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go):
// 200 -> use the presigned URL
// 404 -> cache miss: create the mirror (download from GitHub + tar + upload)
// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror
// creation and upload entirely, behave exactly like stock actions/checkout
// else -> transient trouble: fall back WITHOUT the mirror download
const API_TIMEOUT_MS = 10_000 const API_TIMEOUT_MS = 10_000
export interface MirrorDownloadInfo { export interface SnapshotDownloadInfo {
url: string url: string
size_bytes: number size_bytes: number
created_at: string created_at: string
} }
export type MirrorLookup = export type SnapshotLookup =
| {kind: 'hit'; info: MirrorDownloadInfo} | {kind: 'hit'; info: SnapshotDownloadInfo}
| {kind: 'miss'} | {kind: 'miss'}
| {kind: 'disabled'} | {kind: 'disabled'}
| {kind: 'error'} | {kind: 'error'}
@ -41,35 +32,41 @@ function authHeader(): string {
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`
} }
export async function lookupMirror(repoKey: string): Promise<MirrorLookup> { export async function lookupSnapshot(
repoKey: string,
sha: string
): Promise<SnapshotLookup> {
try { try {
const res = await fetch( const res = await fetch(
`${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(
repoKey
)}&sha=${encodeURIComponent(sha)}`,
{ {
headers: {authorization: authHeader()}, headers: {authorization: authHeader()},
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
} }
) )
if (res.status === 200) { if (res.status === 200) {
return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo}
} }
if (res.status === 404) { if (res.status === 404) {
return {kind: 'miss'} return {kind: 'miss'}
} }
if (res.status === 403) { if (res.status === 403) {
core.debug(`[wb-mirror] download-url answered 403 (disabled)`) core.debug(`[wb-cache] download-url answered 403 (disabled)`)
return {kind: 'disabled'} return {kind: 'disabled'}
} }
core.debug(`[wb-mirror] download-url answered ${res.status}`) core.debug(`[wb-cache] download-url answered ${res.status}`)
return {kind: 'error'} return {kind: 'error'}
} catch (error) { } catch (error) {
core.debug(`[wb-mirror] download-url failed: ${error}`) core.debug(`[wb-cache] download-url failed: ${error}`)
return {kind: 'error'} return {kind: 'error'}
} }
} }
export async function requestUploadURL( export async function requestUploadURL(
repoKey: string repoKey: string,
sha: string
): Promise<UploadURLResult> { ): Promise<UploadURLResult> {
try { try {
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
@ -78,7 +75,7 @@ export async function requestUploadURL(
authorization: authHeader(), authorization: authHeader(),
'content-type': 'application/json' 'content-type': 'application/json'
}, },
body: JSON.stringify({repo_key: repoKey}), body: JSON.stringify({repo_key: repoKey, sha}),
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
}) })
if (res.status === 200) { if (res.status === 200) {
@ -89,13 +86,13 @@ export async function requestUploadURL(
return {kind: 'error'} return {kind: 'error'}
} }
if (res.status === 403) { if (res.status === 403) {
core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
return {kind: 'disabled'} return {kind: 'disabled'}
} }
core.debug(`[wb-mirror] upload-url answered ${res.status}`) core.debug(`[wb-cache] upload-url answered ${res.status}`)
return {kind: 'error'} return {kind: 'error'}
} catch (error) { } catch (error) {
core.debug(`[wb-mirror] upload-url failed: ${error}`) core.debug(`[wb-cache] upload-url failed: ${error}`)
return {kind: 'error'} return {kind: 'error'}
} }
} }

View File

@ -1,6 +1,4 @@
/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
.js-suffixed ESM imports both follow upstream's own conventions; the import plugin
has no TS resolver configured in this repo. */
import * as core from '@actions/core' import * as core from '@actions/core'
import * as exec from '@actions/exec' import * as exec from '@actions/exec'
import * as fs from 'fs' import * as fs from 'fs'
@ -11,58 +9,38 @@ import {pipeline} from 'stream/promises'
import {IGitSourceSettings} from '../git-source-settings.js' import {IGitSourceSettings} from '../git-source-settings.js'
import * as api from './backend-api.js' import * as api from './backend-api.js'
// WarpBuild git-mirror cache. // WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch
// // produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we // immutable (no expiry). Fail-open: any error degrades to stock behavior.
// restore it to <workspace>/.git/wb-mirror.git and point .git/objects/info/alternates at
// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and
// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers
// 404), THIS run creates the mirror: one full download of all branches + tags from
// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is
// disabled for this org (backend-driven kill switch) and we skip everything — no
// mirror creation, no upload.
//
// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path
// is relative, so it survives every container mount scheme: `container:` jobs (/__w),
// Docker container actions (/github/workspace), and `docker build COPY .`.
//
// Everything here is fail-open: any error or timeout degrades to stock actions/checkout
// behavior with a warning, never a failed checkout.
const MIRROR_DIR = 'wb-mirror.git'
export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
const UPLOAD_TIMEOUT_MS = 15 * 60_000 const UPLOAD_TIMEOUT_MS = 15 * 60_000
// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state // "hit <sha>" | "uploaded <sha>", for e2e assertions.
// everywhere outside WarpBuild); every other reason is logged at info. export const CACHE_STATE_FILE = 'wb-cache-state'
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
export const SKIP_NOT_WARPBUILD = export const SKIP_NOT_WARPBUILD =
'not running on a WarpBuild runner (WARPBUILD_* env not present)' 'not running on a WarpBuild runner (WARPBUILD_* env not present)'
export function mirrorPath(repositoryPath: string): string { const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/
return path.join(repositoryPath, '.git', MIRROR_DIR)
}
// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache let decision: 'off' | 'miss' = 'off'
// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and
// deliberately strict: anything unexpected means "behave exactly like upstream". // Null = attempt the cache; else a reason to log. Only the default checkout shape
// is served.
export function getMirrorCacheSkipReason( export function getMirrorCacheSkipReason(
settings: IGitSourceSettings settings: IGitSourceSettings
): string | null { ): string | null {
// Only on WarpBuild runners (these are injected into every job's env there).
if ( if (
!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
!process.env['WARPBUILD_HOST_URL'] !process.env['WARPBUILD_HOST_URL']
) { ) {
return SKIP_NOT_WARPBUILD return SKIP_NOT_WARPBUILD
} }
// Linux + macOS in v1.
if (process.platform === 'win32') { if (process.platform === 'win32') {
return 'Windows is not supported by the mirror cache yet' return 'Windows is not supported by the snapshot cache yet'
} }
// The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only
// cache when that is what is being checked out (`repository:` inputs fall back).
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''
if (!repoKey) { if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set' return 'GITHUB_REPOSITORY_ID is not set'
@ -71,7 +49,6 @@ export function getMirrorCacheSkipReason(
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`
} }
// github.com only (repo ids and mirror keys assume it).
const server = (settings.githubServerUrl || 'https://github.com').replace( const server = (settings.githubServerUrl || 'https://github.com').replace(
/\/+$/, /\/+$/,
'' ''
@ -79,226 +56,208 @@ export function getMirrorCacheSkipReason(
if (server !== 'https://github.com') { if (server !== 'https://github.com') {
return `server '${server}' is not github.com` return `server '${server}' is not github.com`
} }
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on'
}
if (settings.fetchDepth !== 1) {
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`
}
if (settings.fetchTags) {
return 'fetch-tags is enabled'
}
if (settings.filter) {
return 'a fetch filter is configured'
}
if (settings.sparseCheckout) {
return 'sparse checkout is configured'
}
if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the snapshot)'
}
if (settings.ref && computeDestinationRef(settings.ref) === null) {
return `ref '${settings.ref}' has no cacheable destination ref`
}
return null return null
} }
// setup is the single upstream splice point, called right after `git init` + // The local ref the fetch would have created ('' = none needed, null = uncacheable).
// `git remote add` for a fresh repository. It never throws. export function computeDestinationRef(ref: string): string | null {
export async function setup( if (!ref) {
settings: IGitSourceSettings, return ''
repositoryUrl: string }
): Promise<void> { const upper = ref.toUpperCase()
if (upper.startsWith('REFS/HEADS/')) {
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`
}
if (upper.startsWith('REFS/PULL/')) {
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`
}
if (upper.startsWith('REFS/TAGS/')) {
return ref
}
return null
}
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
decision = 'off'
const skipReason = getMirrorCacheSkipReason(settings) const skipReason = getMirrorCacheSkipReason(settings)
if (skipReason) { if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) { if (skipReason === SKIP_NOT_WARPBUILD) {
core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
} else { } else {
core.info(`WarpBuild mirror cache skipped: ${skipReason}`) core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
} }
return return false
} }
core.startGroup('WarpBuild: setting up git mirror cache') core.startGroup('WarpBuild: checkout snapshot cache')
try { try {
await setupInner(settings, repositoryUrl) return await setupInner(settings)
} catch (error) { } catch (error) {
core.warning( core.warning(
`WarpBuild mirror cache unavailable, using standard checkout: ${error}` `WarpBuild snapshot cache unavailable, using standard checkout: ${error}`
) )
return false
} finally { } finally {
core.endGroup() core.endGroup()
} }
} }
async function setupInner( async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
settings: IGitSourceSettings,
repositoryUrl: string
): Promise<void> {
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
const mirror = mirrorPath(settings.repositoryPath) const sha = settings.commit
// A second checkout of the same repo in one job finds the mirror already in place. const lookup = await api.lookupSnapshot(repoKey, sha)
if (fs.existsSync(path.join(mirror, 'objects'))) {
core.info('Mirror already present, reusing it')
await writeAlternates(settings.repositoryPath)
return
}
const lookup = await api.lookupMirror(repoKey)
if (lookup.kind === 'disabled') { if (lookup.kind === 'disabled') {
core.info( core.info('Snapshot cache is disabled by the backend for this organization')
'Mirror cache is disabled by the backend for this organization; using standard checkout' return false
)
return
} }
if (lookup.kind === 'error') { if (lookup.kind === 'error') {
core.info('Mirror cache backend unavailable; using standard checkout') core.info('Snapshot cache backend unavailable; using standard checkout')
return return false
} }
if (lookup.kind === 'hit') { if (lookup.kind === 'miss') {
core.info( core.info(
`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`
) )
if (await restoreMirror(lookup.info.url, mirror)) { decision = 'miss'
await writeAlternates(settings.repositoryPath) return false
core.info('Mirror restored; the fetch below downloads only the delta')
}
return
}
// Miss. Probe upload authorization BEFORE the expensive clone so a disabled or
// unreachable backend never costs a wasted full mirror clone.
const probe = await api.requestUploadURL(repoKey)
if (probe.kind !== 'ok') {
core.info(
probe.kind === 'disabled'
? 'Mirror cache is disabled by the backend for this organization; using standard checkout'
: 'Mirror cache backend unavailable; skipping mirror creation'
)
return
} }
core.info( core.info(
'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`
) )
await createMirrorFromGitHub(settings, repositoryUrl, mirror) if (!(await restoreSnapshot(settings, lookup.info.url, sha))) {
await writeAlternates(settings.repositoryPath) return false
// Upload failures only warn: the local mirror still accelerates THIS run. }
await uploadMirror(repoKey, mirror) core.info('Snapshot restored; skipping the GitHub fetch entirely')
return true
} }
// writeAlternates points the workspace repo's object lookups at the mirror. The path is // Runs after checkout; uploads the fetch result on a miss. Failures only warn.
// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. export async function contribute(settings: IGitSourceSettings): Promise<void> {
export async function writeAlternates(repositoryPath: string): Promise<void> { if (decision !== 'miss') {
const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') return
await fs.promises.mkdir(infoDir, {recursive: true}) }
await fs.promises.writeFile( core.startGroup('WarpBuild: uploading checkout snapshot')
path.join(infoDir, 'alternates'), try {
ALTERNATES_CONTENT await uploadSnapshot(settings)
) } catch (error) {
core.info( core.warning(`Snapshot upload skipped: ${error}`)
`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` } finally {
) core.endGroup()
}
} }
async function restoreMirror(url: string, mirror: string): Promise<boolean> { async function restoreSnapshot(
const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) settings: IGitSourceSettings,
url: string,
sha: string
): Promise<boolean> {
const gitDir = path.join(settings.repositoryPath, '.git')
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`)
try { try {
const res = await fetch(url, { const res = await fetch(url, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
}) })
if (!res.ok || !res.body) { if (!res.ok || !res.body) {
throw new Error(`mirror download answered ${res.status}`) throw new Error(`snapshot download answered ${res.status}`)
} }
await pipeline( await pipeline(
Readable.fromWeb(res.body as import('stream/web').ReadableStream), Readable.fromWeb(res.body as import('stream/web').ReadableStream),
fs.createWriteStream(tmpTar) fs.createWriteStream(tmpTar)
) )
await fs.promises.mkdir(mirror, {recursive: true}) await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
await exec.exec('tar', ['-xf', tmpTar, '-C', mirror])
const check = await exec.exec(
'git',
['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`],
{ignoreReturnCode: true}
)
if (check !== 0) {
throw new Error(`restored snapshot does not contain ${sha}`)
}
// The ref the skipped fetch would have created; upstream's verification needs it.
const dstRef = computeDestinationRef(settings.ref)
if (dstRef) {
await exec.exec('git', [
'-C',
settings.repositoryPath,
'update-ref',
dstRef,
sha
])
}
await fs.promises.writeFile(
path.join(gitDir, CACHE_STATE_FILE),
`hit ${sha}\n`
)
return true return true
} catch (error) { } catch (error) {
core.warning(`Mirror restore failed: ${error}`) core.warning(`Snapshot restore failed: ${error}`)
// Never leave a partial mirror behind an alternates file — that is corruption. // Reset .git to its freshly-init state.
await fs.promises.rm(mirror, {recursive: true, force: true}) await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true,
force: true
})
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
recursive: true
})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
return false return false
} finally { } finally {
await fs.promises.rm(tmpTar, {force: true}) await fs.promises.rm(tmpTar, {force: true})
} }
} }
// createMirrorFromGitHub builds the bare mirror by downloading the repository from async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also const sha = settings.commit
// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large const gitDir = path.join(settings.repositoryPath, '.git')
// share of the download on PR-heavy repos). PR-triggered checkouts still work: their const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
// head SHA simply arrives as a small delta in the workspace fetch.
//
// The full history download itself cannot be safely avoided: a shallow or partial
// mirror behind an alternates file is an incomplete object store that git assumes is
// complete — the exact corruption Blacksmith hit and reverted. A mirror must be
// complete with respect to the refs it advertises.
async function createMirrorFromGitHub(
settings: IGitSourceSettings,
repositoryUrl: string,
mirror: string
): Promise<void> {
// Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars
// (git >= 2.31) so the credential never appears in any process's argv.
const basicCredential = Buffer.from(
`x-access-token:${settings.authToken}`,
'utf8'
).toString('base64')
core.setSecret(basicCredential)
const env: {[key: string]: string} = {}
for (const [key, value] of Object.entries(process.env)) {
if (value !== undefined) {
env[key] = value
}
}
env['GIT_CONFIG_COUNT'] = '1'
env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'
env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`
await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env})
await exec.exec(
'git',
['-C', mirror, 'remote', 'add', 'origin', repositoryUrl],
{
env
}
)
// gc.auto=0: never let the fetch spawn a detached gc that outlives the step.
await exec.exec(
'git',
[
'-c',
'gc.auto=0',
'-C',
mirror,
'fetch',
'--prune',
'--progress',
'origin',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*'
],
{env}
)
}
async function uploadMirror(repoKey: string, mirror: string): Promise<void> {
const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`)
try { try {
// Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic const members = ['objects']
// trims; the tar stays a valid bare repo either way. if (fs.existsSync(path.join(gitDir, 'shallow'))) {
await exec.exec('tar', [ members.push('shallow')
'-cf', }
tmpTar, await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
'-C',
mirror,
'--exclude',
'./hooks',
'--exclude',
'./description',
'--exclude',
'./FETCH_HEAD',
'.'
])
const size = (await fs.promises.stat(tmpTar)).size const size = (await fs.promises.stat(tmpTar)).size
// Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the const upload = await api.requestUploadURL(repoKey, sha)
// backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. if (upload.kind !== 'ok') {
const fresh = await api.requestUploadURL(repoKey) core.info(
if (fresh.kind !== 'ok') { upload.kind === 'disabled'
throw new Error( ? 'Snapshot cache is disabled by the backend; not uploading'
fresh.kind === 'disabled' : 'Snapshot cache backend unavailable; not uploading'
? 'mirror cache was disabled by the backend'
: 'upload-url unavailable'
) )
return
} }
const init: RequestInit & {duplex: 'half'} = { const init: RequestInit & {duplex: 'half'} = {
@ -313,13 +272,17 @@ async function uploadMirror(repoKey: string, mirror: string): Promise<void> {
duplex: 'half', duplex: 'half',
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
} }
const res = await fetch(fresh.url, init) const res = await fetch(upload.url, init)
if (!res.ok) { if (!res.ok) {
throw new Error(`mirror upload answered ${res.status}`) throw new Error(`snapshot upload answered ${res.status}`)
} }
core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) await fs.promises.writeFile(
} catch (error) { path.join(gitDir, CACHE_STATE_FILE),
core.warning(`Mirror upload skipped: ${error}`) `uploaded ${sha}\n`
)
core.info(
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
)
} finally { } finally {
await fs.promises.rm(tmpTar, {force: true}) await fs.promises.rm(tmpTar, {force: true})
} }