diff --git a/src/commands.ts b/src/commands.ts index 82454b1..67e2966 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -66,12 +66,16 @@ const NET_ENV = { * * @param cwd Working directory for the command, or undefined for the default. */ -function netExec(cwd?: string) { +function netExec(cwd?: string, opts?: { captureOutput?: boolean }) { + const captureOutput = opts?.captureOutput === true; return $({ ...(cwd ? { cwd } : {}), env: NET_ENV, timeout: GIT_NET_TIMEOUT_MS, - stdio: ['ignore', 'inherit', 'inherit'], + stdio: captureOutput + ? ['ignore', 'pipe', 'pipe'] + : ['ignore', 'inherit', 'inherit'], + ...(captureOutput ? { buffer: false } : {}), }); } @@ -104,6 +108,207 @@ async function runNetOp( p.log.success(stopMsg); } +/** `-c` flags that make a large seed push survive: gh credentials over + * HTTPS, a big post buffer + HTTP/1.1 (avoids chunked-encoding issues on + * big POSTs), and a stalled-transfer abort. */ +const SEED_PUSH_CONFIG = [ + '-c', + 'credential.https://github.com.helper=!gh auth git-credential', + '-c', + 'http.postBuffer=524288000', + '-c', + 'http.version=HTTP/1.1', + '-c', + 'http.lowSpeedLimit=1000', + '-c', + 'http.lowSpeedTime=60', +]; + +/** + * Whether a failed seed push is worth retrying. Retry only recognized + * server-side/transport failures during a large transfer (timeouts, + * gateways, dropped connections). Auth/permission/not-found/protected-branch + * and any unrecognized failure fail fast — retrying those just delays a + * permanent setup error. + */ +function isTransientPushError(err: unknown): boolean { + if (err instanceof GitError) return false; // hard timeout + const e = err as { + stderr?: unknown; + stdout?: unknown; + shortMessage?: unknown; + message?: unknown; + }; + const text = [e.stderr, e.stdout, e.shortMessage, e.message] + .map((v) => (typeof v === 'string' ? v : '')) + .join('\n') + .toLowerCase(); + if (!text) return false; + + const permanent = [ + 'http 401', + 'http 403', + 'http 404', + 'authentication failed', + 'permission denied', + 'access denied', + 'repository not found', + 'could not read from remote repository', + 'protected branch', + 'pre-receive hook declined', + 'remote rejected', + 'remote: error: gh', + ]; + if (permanent.some((p) => text.includes(p))) return false; + + const transient = [ + 'http 408', + 'error: 408', + 'http 500', + 'http 502', + 'http 503', + 'http 504', + 'bad gateway', + 'gateway time', + 'service unavailable', + 'the remote end hung up', + 'unexpected disconnect', + 'early eof', + 'connection reset', + 'connection timed out', + 'operation timed out', + 'failed to connect', + 'recv failure', + 'transfer closed', + ]; + return transient.some((p) => text.includes(p)); +} + +const PUSH_OUTPUT_TAIL_CHARS = 16_384; + +function appendTail(text: string, chunk: string): string { + const next = text + chunk; + return next.length > PUSH_OUTPUT_TAIL_CHARS + ? next.slice(-PUSH_OUTPUT_TAIL_CHARS) + : next; +} + +async function pushSeedRef( + tempDir: string, + httpsUrl: string, + src: string, + branch: string +): Promise { + const push = netExec(tempDir, { + captureOutput: true, + })`git ${SEED_PUSH_CONFIG} push --force --no-thin --progress ${httpsUrl} ${src}:refs/heads/${branch}`; + let stdoutTail = ''; + let stderrTail = ''; + push.stdout?.on('data', (chunk: string | Buffer) => { + const text = typeof chunk === 'string' ? chunk : chunk.toString(); + process.stdout.write(text); + stdoutTail = appendTail(stdoutTail, text); + }); + push.stderr?.on('data', (chunk: string | Buffer) => { + const text = typeof chunk === 'string' ? chunk : chunk.toString(); + process.stderr.write(text); + stderrTail = appendTail(stderrTail, text); + }); + + try { + await push; + } catch (err) { + const e = err as { stdout?: unknown; stderr?: unknown }; + if ( + (typeof e.stdout !== 'string' || e.stdout.length === 0) && + stdoutTail.length > 0 + ) { + e.stdout = stdoutTail; + } + if ( + (typeof e.stderr !== 'string' || e.stderr.length === 0) && + stderrTail.length > 0 + ) { + e.stderr = stderrTail; + } + throw err; + } +} + +/** + * Seed a fresh private mirror from a local upstream clone. + * + * A single `git push` of a large repo's full history 408s — GitHub enforces + * a request-duration limit and the monolithic pack POST exceeds it. So push + * the default branch in commit batches (each POST stays small), then push the + * real tip. History/SHAs are identical to upstream, which `sync`/`stage`/PRs + * require. Each push is retried a few times to ride out transient 408s. + * + * Batch size is `VENFORK_SEED_CHUNK` commits (default 1000); small repos take + * a single push (loop body is skipped). + * + * @param tempDir Local upstream clone. + * @param httpsUrl HTTPS URL of the private mirror. + * @param branch Default branch to seed. + */ +async function seedMirrorInChunks( + tempDir: string, + httpsUrl: string, + branch: string +): Promise { + const rawChunk = Number(process.env.VENFORK_SEED_CHUNK ?? 1000); + // Fall back to the default chunk size for invalid values: 0/negative + // (would infinite-loop) or NaN (non-numeric env). + const chunk = Number.isInteger(rawChunk) && rawChunk > 0 ? rawChunk : 1000; + + // Select chunk tips along the branch's first-parent ancestry so each pushed + // tip is guaranteed to descend from the previous one. + const revList = await $({ + cwd: tempDir, + })`git rev-list --first-parent --reverse ${branch}`; + const commits = revList.stdout.split('\n').filter(Boolean); + const total = commits.length; + + // Treat unset or empty/whitespace-only env as "use default" so an empty + // string doesn't coerce to 0 (Number('') === 0) and silently disable + // backoff. An explicit '0' stays valid (0ms backoff). + const rawRetryMs = process.env.VENFORK_SEED_RETRY_MS; + const rawBackoff = + rawRetryMs != null && rawRetryMs.trim() !== '' ? Number(rawRetryMs) : 8000; + // Fall back to the default for invalid values (negative or NaN). + const backoffMs = + Number.isFinite(rawBackoff) && rawBackoff >= 0 ? rawBackoff : 8000; + + const pushRef = async (src: string, label: string): Promise => { + const attempts = 4; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + await runNetOp( + `Pushing ${label} to private mirror repository` + + (attempt > 1 ? ` (attempt ${attempt})` : ''), + `Pushed ${label}`, + () => pushSeedRef(tempDir, httpsUrl, src, branch) + ); + return; + } catch (err) { + // Fail fast on the last attempt, hard timeouts, and permanent + // errors (bad creds, missing access, protected branch, …); only + // recognized transient failures (e.g. HTTP 408) are retried. + if (attempt === attempts || !isTransientPushError(err)) throw err; + p.log.warn( + `Push of ${label} failed; retrying ` + `(${attempt}/${attempts - 1})` + ); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + } + } + }; + + for (let i = chunk; i < total; i += chunk) { + await pushRef(commits[i - 1], `commits 1–${i}/${total}`); + } + await pushRef(branch, `${branch} (final)`); +} + /** * `p.confirm` wrapper that can return `true` immediately when * `VENFORK_NONINTERACTIVE=1` is set in the environment, but only for @@ -1027,15 +1232,7 @@ export async function setupCommand( } s.stop(`Default branch: ${defaultBranch}`); - const ghCredentialHelper = '!gh auth git-credential'; - await runNetOp( - `Pushing ${defaultBranch} to private mirror repository`, - 'Pushed to private mirror repository', - () => - netExec( - tempDir - )`git -c credential.https://github.com.helper=${ghCredentialHelper} -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=60 push --progress ${privateHttpsUrl} ${defaultBranch}:${defaultBranch}` - ); + await seedMirrorInChunks(tempDir, privateHttpsUrl, defaultBranch); } // Step 6: Local clone of the private mirror diff --git a/tests/commands.test.ts b/tests/commands.test.ts index 4ef7e5b..7e66137 100644 --- a/tests/commands.test.ts +++ b/tests/commands.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; +import { PassThrough } from 'node:stream'; /** * Command tests with execution-based mocking @@ -150,6 +151,27 @@ function getMockExecaResponse(command: string) { return Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }); } +/** + * Simulate execa with `buffer: false`: push fails with no buffered + * stdout/stderr on the error object, while stderr is only available via the + * live stream attached to the returned promise. + */ +function streamRejectedPush(stderr: string): Promise & { + stdout: PassThrough; + stderr: PassThrough; +} { + const stdout = new PassThrough(); + const stderrStream = new PassThrough(); + stdout.end(); + const promise = (async (): Promise => { + await new Promise((resolve) => setTimeout(resolve, 0)); + stderrStream.write(stderr); + stderrStream.end(); + throw Object.assign(new Error('Command failed'), { exitCode: 128 }); + })(); + return Object.assign(promise, { stdout, stderr: stderrStream }); +} + // Mock fs.rm and fs.access BEFORE any imports mock.module('node:fs/promises', () => ({ mkdtemp: mock((prefix: string) => { @@ -418,11 +440,179 @@ describe('setupCommand - execution tests', () => { expect(seedPush).toBeDefined(); expect(seedPush).toContain('credential.https://github.com.helper'); expect(seedPush).toContain('--progress'); - expect(seedPush).toContain('main:main'); + expect(seedPush).toContain('main:refs/heads/main'); // The hang fix: the seeding push must not use the raw SSH URL. expect(seedPush).not.toContain('git@github.com:'); }); + test('seeds large history in commit-batched pushes, then the real tip', async () => { + process.env.VENFORK_SEED_CHUNK = '2'; + mockResponses.set('git rev-list --first-parent --reverse main', { + exitCode: 0, + stdout: 'aaa\nbbb\nccc\nddd\neee', + stderr: '', + }); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } finally { + delete process.env.VENFORK_SEED_CHUNK; + } + + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes(' push ') && + cmd.includes('https://github.com/testuser/test-vendor.git') && + cmd.includes(':refs/heads/main') + ); + + // 5 commits, chunk 2 -> push at commit 2 (bbb), commit 4 (ddd), + // then the real branch tip. + expect(seedPushes.length).toBe(3); + expect(seedPushes.some((c) => c.includes('bbb:refs/heads/main'))).toBe( + true + ); + expect(seedPushes.some((c) => c.includes('ddd:refs/heads/main'))).toBe( + true + ); + expect(seedPushes.some((c) => c.includes('main:refs/heads/main'))).toBe( + true + ); + expect(seedPushes.every((c) => c.includes('--force'))).toBe(true); + expect( + execaCalls.some((c) => + c.includes('git rev-list --first-parent --reverse main') + ) + ).toBe(true); + }); + + test('invalid VENFORK_SEED_CHUNK falls back to default (no infinite loop)', async () => { + process.env.VENFORK_SEED_CHUNK = '0'; + mockResponses.set('git rev-list --first-parent --reverse main', { + exitCode: 0, + stdout: 'aaa\nbbb\nccc\nddd\neee', + stderr: '', + }); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } finally { + delete process.env.VENFORK_SEED_CHUNK; + } + + // chunk=0 -> guarded to 1000 -> 5 commits skip the loop -> single tip push. + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes(' push ') && + cmd.includes('https://github.com/testuser/test-vendor.git') && + cmd.includes(':refs/heads/main') + ); + expect(seedPushes.length).toBe(1); + expect(seedPushes[0]).toContain('main:refs/heads/main'); + }); + + test('does not retry permanent seed-push failures (fails fast)', async () => { + mockResponses.set('push --force --no-thin', () => + Promise.reject( + Object.assign(new Error('Command failed'), { + stderr: + 'remote: Permission to testuser/test-vendor.git denied.\n' + + 'fatal: unable to access: The requested URL returned error: HTTP 403', + exitCode: 128, + }) + ) + ); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } + + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes('push --force --no-thin') && + cmd.includes('https://github.com/testuser/test-vendor.git') + ); + expect(seedPushes.length).toBe(1); // no retries for a permanent error + }); + + test('retries transient seed-push failures up to the attempt limit', async () => { + process.env.VENFORK_SEED_RETRY_MS = '0'; + mockResponses.set('push --force --no-thin', () => + Promise.reject( + Object.assign(new Error('Command failed'), { + stderr: + 'error: RPC failed; HTTP 408 curl 22\n' + + 'fatal: the remote end hung up unexpectedly', + exitCode: 128, + }) + ) + ); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } finally { + delete process.env.VENFORK_SEED_RETRY_MS; + } + + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes('push --force --no-thin') && + cmd.includes('https://github.com/testuser/test-vendor.git') + ); + expect(seedPushes.length).toBe(4); // 1 + 3 retries from transient stderr + }); + + test('retries transient seed-push failures from streamed stderr when error has no buffered output', async () => { + process.env.VENFORK_SEED_RETRY_MS = '0'; + mockResponses.set('push --force --no-thin', () => + streamRejectedPush( + 'error: RPC failed; HTTP 408 curl 22\n' + + 'fatal: the remote end hung up unexpectedly' + ) + ); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } finally { + delete process.env.VENFORK_SEED_RETRY_MS; + } + + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes('push --force --no-thin') && + cmd.includes('https://github.com/testuser/test-vendor.git') + ); + expect(seedPushes.length).toBe(4); // 1 + 3 retries from streamed stderr tail + }); + + test('does not retry generic rpc-failed wrappers without transient status', async () => { + mockResponses.set('push --force --no-thin', () => + Promise.reject( + Object.assign(new Error('Command failed'), { + stderr: 'error: RPC failed; HTTP 413 curl 22', + exitCode: 128, + }) + ) + ); + try { + await setupCommand('git@github.com:test/repo.git', 'test-vendor'); + } catch { + // Expected + } + + const seedPushes = execaCalls.filter( + (cmd) => + cmd.includes('push --force --no-thin') && + cmd.includes('https://github.com/testuser/test-vendor.git') + ); + expect(seedPushes.length).toBe(1); // no retries for unrecognized wrappers + }); + test('passes --progress to upstream and private mirror clones', async () => { try { await setupCommand('git@github.com:test/repo.git', 'test-vendor');