From a3c60e2b7e24e44d44c654bd1c2b86b6eb275fc5 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 24 Mar 2026 18:50:09 -1000 Subject: [PATCH 1/5] Generate Netlify upload scaffolding --- apps/example/microblog.schema.json | 8 + packages/cli/src/index.ts | 459 +++++++++++++++++- .../schema/schemas/tokenhost-ths.schema.json | 39 ++ packages/schema/src/lint.ts | 11 + packages/schema/src/types.ts | 17 + schemas/tokenhost-ths.schema.json | 39 ++ test/testCliBuildArtifacts.js | 47 ++ test/testThsSchema.js | 44 ++ 8 files changed, 663 insertions(+), 1 deletion(-) diff --git a/apps/example/microblog.schema.json b/apps/example/microblog.schema.json index 00959c1..9574839 100644 --- a/apps/example/microblog.schema.json +++ b/apps/example/microblog.schema.json @@ -9,6 +9,14 @@ "accentText": "blog" }, "primaryCollection": "Post", + "deploy": { + "netlify": { + "uploads": { + "provider": "filecoin_onchain_cloud", + "runner": "background-function" + } + } + }, "theme": { "preset": "cyber-grid" }, diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 8e7ad1c..e5e3ebb 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2183,6 +2183,21 @@ type RelayConfig = { type UploadRunnerMode = 'local' | 'remote' | 'foc-process'; type UploadProvider = 'local_file' | 'filecoin_onchain_cloud'; +type NetlifyUploadBuildConfig = { + provider: 'filecoin_onchain_cloud'; + runner: 'background-function'; + privateKeyEnv: string; + blobsStore: string; + functionsDirectory: string; + endpointUrl: string; + statusUrl: string; + startFunctionName: string; + statusFunctionName: string; + workerFunctionName: string; + accept: string[]; + maxBytes: number; +}; + type UploadManifestConfig = { enabled: boolean; baseUrl: string; @@ -2309,6 +2324,429 @@ function resolveUploadManifestConfig(featuresUploads: boolean): UploadManifestCo }; } +function normalizeFunctionsDirectory(value: string | undefined): string { + const trimmed = String(value ?? '').trim().replace(/^\/+|\/+$/g, ''); + if (!trimmed) return 'netlify/functions'; + return trimmed; +} + +function resolveNetlifyUploadBuildConfig(schema: ThsSchema): NetlifyUploadBuildConfig | null { + const deploy = schema.app.deploy?.netlify?.uploads; + if (!deploy) return null; + return { + provider: 'filecoin_onchain_cloud', + runner: 'background-function', + privateKeyEnv: String(deploy.privateKeyEnv ?? 'TH_UPLOAD_FOC_PRIVATE_KEY').trim() || 'TH_UPLOAD_FOC_PRIVATE_KEY', + blobsStore: String(deploy.blobsStore ?? 'tokenhost-upload-jobs').trim() || 'tokenhost-upload-jobs', + functionsDirectory: normalizeFunctionsDirectory(deploy.functionsDirectory), + endpointUrl: '/__tokenhost/upload', + statusUrl: '/__tokenhost/upload-status', + startFunctionName: 'tokenhost-upload-start', + statusFunctionName: 'tokenhost-upload-status', + workerFunctionName: 'tokenhost-upload-worker-background', + accept: String(process.env.TH_UPLOAD_ACCEPT ?? 'image/png,image/jpeg,image/gif,image/webp,image/svg+xml') + .split(',') + .map((x) => x.trim()) + .filter(Boolean), + maxBytes: parsePositiveIntEnv(process.env.TH_UPLOAD_MAX_BYTES, 10 * 1024 * 1024) + }; +} + +function renderNetlifyPackageJson(schema: ThsSchema): string { + return JSON.stringify( + { + name: `${schema.app.slug}-netlify`, + private: true, + type: 'module', + dependencies: { + '@netlify/blobs': '^8.1.0' + } + }, + null, + 2 + ); +} + +function renderNetlifyToml(config: NetlifyUploadBuildConfig): string { + return `# Generated by Token Host +[build] + publish = "ui-site" + +[functions] + directory = "${config.functionsDirectory}" + node_bundler = "esbuild" + external_node_modules = ["@netlify/blobs"] + +[[redirects]] + from = "${config.endpointUrl}" + to = "/.netlify/functions/${config.startFunctionName}" + status = 200 + force = true + +[[redirects]] + from = "${config.statusUrl}" + to = "/.netlify/functions/${config.statusFunctionName}" + status = 200 + force = true +`; +} + +function renderNetlifyUploadShared(config: NetlifyUploadBuildConfig): string { + return `import { getStore } from '@netlify/blobs'; +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +export const UPLOAD_PROVIDER = ${JSON.stringify(config.provider)}; +export const UPLOAD_RUNNER_MODE = 'netlify-background'; +export const BLOBS_STORE_NAME = ${JSON.stringify(config.blobsStore)}; +export const PRIVATE_KEY_ENV = ${JSON.stringify(config.privateKeyEnv)}; +export const STATUS_BASE_PATH = ${JSON.stringify(config.statusUrl)}; +export const MAX_BYTES = ${config.maxBytes}; +export const DEFAULT_FOC_COMMAND = 'npx -y foc-cli'; +export const DEFAULT_CHAIN_ID = 314159; +export const DEFAULT_COPIES = 1; + +export function getUploadStore() { + return getStore(BLOBS_STORE_NAME); +} + +export function jobStatusKey(jobId) { + return \`status:\${jobId}\`; +} + +export function jobRequestKey(jobId) { + return \`request:\${jobId}\`; +} + +export function createJobId() { + return crypto.randomUUID(); +} + +export function normalizeUploadResult(parsed) { + const result = parsed?.data?.result ?? parsed?.result ?? parsed; + const copyResults = Array.isArray(result?.copyResults) ? result.copyResults : []; + const firstCopy = copyResults.find((entry) => entry && typeof entry.url === 'string' && entry.url.trim()) ?? null; + const url = firstCopy ? String(firstCopy.url) : ''; + if (!url) { + throw new Error('foc-cli upload did not return a usable copyResults[].url value.'); + } + return { + url, + cid: result?.pieceCid ? String(result.pieceCid) : null, + size: Number.isFinite(Number(result?.size)) ? Number(result.size) : null, + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + contentType: result?.mimeType ? String(result.mimeType) : null, + metadata: { + pieceScannerUrl: result?.pieceScannerUrl ? String(result.pieceScannerUrl) : null, + copyResults, + copyFailures: Array.isArray(result?.copyFailures) ? result.copyFailures : [], + processLog: Array.isArray(parsed?.data?.processLog) ? parsed.data.processLog : [] + } + }; +} + +export async function setJobStatus(jobId, value) { + const store = getUploadStore(); + await store.setJSON(jobStatusKey(jobId), value); +} + +export async function getJobStatus(jobId) { + const store = getUploadStore(); + return (await store.get(jobStatusKey(jobId), { type: 'json' })) ?? null; +} + +export async function getJobRequest(jobId) { + const store = getUploadStore(); + return (await store.get(jobRequestKey(jobId), { type: 'json' })) ?? null; +} + +export async function setJobRequest(jobId, value) { + const store = getUploadStore(); + await store.setJSON(jobRequestKey(jobId), value); +} + +export async function deleteJobRequest(jobId) { + const store = getUploadStore(); + await store.delete(jobRequestKey(jobId)); +} + +export function parsePositiveInt(value, fallback) { + const parsed = Number.parseInt(String(value ?? ''), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +export function shellQuote(value) { + return \`'\${String(value).replace(/'/g, \`'"'"'\`)}'\`; +} + +export async function runShellCommand(command, env) { + return await new Promise((resolve, reject) => { + const child = spawn('/bin/bash', ['-lc', command], { env }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('error', reject); + child.on('close', (code) => resolve({ code, stdout, stderr })); + }); +} + +export async function runFocUploadFromJob(job) { + const privateKey = String(process.env[PRIVATE_KEY_ENV] ?? '').trim(); + if (!privateKey) { + throw new Error(\`Missing required Netlify env var \${PRIVATE_KEY_ENV}.\`); + } + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tokenhost-netlify-upload-')); + const tmpFile = path.join(tmpDir, job.fileName || 'upload.bin'); + + try { + await fs.writeFile(tmpFile, Buffer.from(String(job.bodyBase64 || ''), 'base64')); + const command = String(process.env.TH_UPLOAD_FOC_COMMAND ?? DEFAULT_FOC_COMMAND).trim() || DEFAULT_FOC_COMMAND; + const chainId = parsePositiveInt(process.env.TH_UPLOAD_FOC_CHAIN, DEFAULT_CHAIN_ID); + const copies = parsePositiveInt(process.env.TH_UPLOAD_FOC_COPIES, DEFAULT_COPIES); + const withCDN = String(process.env.TH_UPLOAD_FOC_WITH_CDN ?? '').trim().toLowerCase(); + const debug = String(process.env.TH_UPLOAD_FOC_DEBUG ?? '').trim().toLowerCase(); + const shellCommand = + \`\${command} upload \${shellQuote(tmpFile)} --format json --chain \${chainId} --copies \${copies}\` + + ((withCDN === '1' || withCDN === 'true' || withCDN === 'yes' || withCDN === 'on') ? ' --withCDN true' : '') + + ((debug === '1' || debug === 'true' || debug === 'yes' || debug === 'on') ? ' --debug true --verbose' : ''); + + const result = await runShellCommand(shellCommand, { + ...process.env, + PRIVATE_KEY: privateKey, + TH_UPLOAD_FOC_PRIVATE_KEY: privateKey + }); + + if (result.code !== 0) { + throw new Error(String(result.stderr || result.stdout || \`foc-cli failed with status \${result.code}\`).trim()); + } + + let parsed = null; + try { + parsed = JSON.parse(String(result.stdout || '').trim()); + } catch { + throw new Error('foc-cli did not return valid JSON.'); + } + + return normalizeUploadResult(parsed); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +} +`; +} + +function renderNetlifyUploadStart(config: NetlifyUploadBuildConfig): string { + return `import { + MAX_BYTES, + STATUS_BASE_PATH, + UPLOAD_PROVIDER, + UPLOAD_RUNNER_MODE, + createJobId, + setJobRequest, + setJobStatus +} from './_tokenhost-upload-shared.mjs'; + +export default async (request) => { + if (request.method === 'GET') { + return Response.json({ + enabled: true, + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + statusUrl: STATUS_BASE_PATH + }); + } + + if (request.method !== 'POST') { + return new Response('Method Not Allowed', { status: 405 }); + } + + const body = Buffer.from(await request.arrayBuffer()); + if (body.length > MAX_BYTES) { + return Response.json({ ok: false, error: 'Request body too large.' }, { status: 413 }); + } + + const jobId = createJobId(); + const fileName = String(request.headers.get('x-tokenhost-upload-filename') || 'upload.bin').trim() || 'upload.bin'; + const contentType = String(request.headers.get('content-type') || 'application/octet-stream').trim() || 'application/octet-stream'; + const size = body.length; + await setJobRequest(jobId, { + jobId, + fileName, + contentType, + size, + bodyBase64: body.toString('base64'), + createdAt: new Date().toISOString() + }); + await setJobStatus(jobId, { + ok: true, + pending: true, + jobId, + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + createdAt: new Date().toISOString() + }); + + const workerUrl = new URL('/.netlify/functions/${config.workerFunctionName}', request.url).toString(); + const workerRes = await fetch(workerUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jobId }) + }); + + if (!workerRes.ok && workerRes.status !== 202) { + await setJobStatus(jobId, { + ok: false, + pending: false, + jobId, + error: \`Unable to enqueue Netlify background upload (HTTP \${workerRes.status}).\`, + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + updatedAt: new Date().toISOString() + }); + return Response.json({ ok: false, error: 'Unable to enqueue Netlify background upload.' }, { status: 500 }); + } + + return Response.json( + { + ok: true, + pending: true, + jobId, + statusUrl: \`\${STATUS_BASE_PATH}?jobId=\${encodeURIComponent(jobId)}\` + }, + { status: 202 } + ); +}; +`; +} + +function renderNetlifyUploadStatus(): string { + return `import { getJobStatus } from './_tokenhost-upload-shared.mjs'; + +export default async (request) => { + const url = new URL(request.url); + const jobId = String(url.searchParams.get('jobId') || '').trim(); + if (!jobId) { + return Response.json({ ok: false, error: 'Missing jobId query parameter.' }, { status: 400 }); + } + + const status = await getJobStatus(jobId); + if (!status) { + return Response.json({ ok: false, error: 'Unknown upload job.' }, { status: 404 }); + } + + return Response.json(status, { status: 200 }); +}; +`; +} + +function renderNetlifyUploadWorker(): string { + return `import { deleteJobRequest, getJobRequest, runFocUploadFromJob, setJobStatus, UPLOAD_PROVIDER, UPLOAD_RUNNER_MODE } from './_tokenhost-upload-shared.mjs'; + +export default async (request) => { + if (request.method !== 'POST') { + return new Response('Method Not Allowed', { status: 405 }); + } + + let body = null; + try { + body = await request.json(); + } catch { + body = null; + } + const jobId = String(body?.jobId || '').trim(); + if (!jobId) { + return Response.json({ ok: false, error: 'Missing jobId.' }, { status: 400 }); + } + + const job = await getJobRequest(jobId); + if (!job) { + await setJobStatus(jobId, { + ok: false, + pending: false, + jobId, + error: 'Upload payload was not found in Netlify Blobs.', + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + updatedAt: new Date().toISOString() + }); + return Response.json({ ok: true, pending: false }, { status: 202 }); + } + + try { + const uploaded = await runFocUploadFromJob(job); + await setJobStatus(jobId, { + ok: true, + pending: false, + jobId, + upload: uploaded, + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + updatedAt: new Date().toISOString() + }); + await deleteJobRequest(jobId); + } catch (error) { + await setJobStatus(jobId, { + ok: false, + pending: false, + jobId, + error: String(error?.message ?? error), + provider: UPLOAD_PROVIDER, + runnerMode: UPLOAD_RUNNER_MODE, + updatedAt: new Date().toISOString() + }); + } + + return Response.json({ ok: true, pending: false }, { status: 202 }); +}; +`; +} + +function renderNetlifyReadme(config: NetlifyUploadBuildConfig): string { + return `# Netlify Upload Runner + +This build output includes Netlify Functions scaffolding for Token Host uploads. + +Required Netlify runtime environment variables: +- \`${config.privateKeyEnv}\` - private key used for \`foc-cli upload\` +- \`TH_UPLOAD_FOC_CHAIN\` - optional, defaults to \`314159\` +- \`TH_UPLOAD_FOC_COPIES\` - optional, defaults to \`1\` +- \`TH_UPLOAD_FOC_COMMAND\` - optional, defaults to \`npx -y foc-cli\` +- \`TH_UPLOAD_FOC_WITH_CDN\` - optional +- \`TH_UPLOAD_FOC_DEBUG\` - optional + +Generated routes: +- \`${config.endpointUrl}\` -> upload start +- \`${config.statusUrl}\` -> upload status + +Implementation notes: +- The start function stores the request body in Netlify Blobs and returns \`202\` with a job id. +- The background worker runs \`foc-cli\` and writes the final result back to Netlify Blobs. +- The browser polls the status endpoint until the upload completes. +`; +} + +function writeNetlifyUploadArtifacts(outDir: string, schema: ThsSchema, config: NetlifyUploadBuildConfig) { + const functionsDir = path.join(outDir, config.functionsDirectory); + ensureDir(functionsDir); + fs.writeFileSync(path.join(outDir, 'package.json'), renderNetlifyPackageJson(schema)); + fs.writeFileSync(path.join(outDir, 'netlify.toml'), renderNetlifyToml(config)); + fs.writeFileSync(path.join(outDir, 'NETLIFY-UPLOADS.md'), renderNetlifyReadme(config)); + fs.writeFileSync(path.join(functionsDir, '_tokenhost-upload-shared.mjs'), renderNetlifyUploadShared(config)); + fs.writeFileSync(path.join(functionsDir, `${config.startFunctionName}.mjs`), renderNetlifyUploadStart(config)); + fs.writeFileSync(path.join(functionsDir, `${config.statusFunctionName}.mjs`), renderNetlifyUploadStatus()); + fs.writeFileSync(path.join(functionsDir, `${config.workerFunctionName}.mjs`), renderNetlifyUploadWorker()); +} + function shellQuote(s: string): string { return `'${String(s).replace(/'/g, `'\"'\"'`)}'`; } @@ -3406,7 +3844,19 @@ function buildFromSchema( const zeroAddress = '0x0000000000000000000000000000000000000000'; const txMode = resolveTxMode(opts.txMode, Number(opts.targetChainId ?? anvil.id)); const relayBaseUrl = String(opts.relayBaseUrl ?? process.env.TH_RELAY_BASE_URL ?? '/__tokenhost/relay').trim() || '/__tokenhost/relay'; - const uploadConfig = resolveUploadManifestConfig(features.uploads); + const netlifyUploadBuild = resolveNetlifyUploadBuildConfig(schema); + const uploadConfig = netlifyUploadBuild + ? { + enabled: true, + baseUrl: netlifyUploadBuild.endpointUrl, + endpointUrl: netlifyUploadBuild.endpointUrl, + statusUrl: netlifyUploadBuild.statusUrl, + provider: netlifyUploadBuild.provider, + runnerMode: 'remote' as const, + accept: netlifyUploadBuild.accept, + maxBytes: netlifyUploadBuild.maxBytes + } + : resolveUploadManifestConfig(features.uploads); const manifest = { manifestVersion: '0.1.0', @@ -3531,6 +3981,10 @@ function buildFromSchema( publishManifestToUiSite(uiSiteDir, manifestJsonOut); } + if (netlifyUploadBuild) { + writeNetlifyUploadArtifacts(resolvedOutDir, schema, netlifyUploadBuild); + } + if (!opts.quiet) { console.log(`Wrote ${appSol.path}`); console.log(`Wrote compiled/App.json`); @@ -3540,6 +3994,9 @@ function buildFromSchema( console.log(`Wrote ui-bundle/ (digest: ${uiBundleDigest})`); console.log(`Wrote ui-site/ (self-hostable static root)`); } + if (netlifyUploadBuild) { + console.log(`Wrote netlify.toml + Netlify upload functions`); + } console.log(`Wrote manifest.json`); console.log(''); diff --git a/packages/schema/schemas/tokenhost-ths.schema.json b/packages/schema/schemas/tokenhost-ths.schema.json index ba9a062..6327c69 100644 --- a/packages/schema/schemas/tokenhost-ths.schema.json +++ b/packages/schema/schemas/tokenhost-ths.schema.json @@ -38,6 +38,45 @@ } }, "primaryCollection": { "type": "string", "minLength": 1 }, + "deploy": { + "type": "object", + "additionalProperties": false, + "properties": { + "netlify": { + "type": "object", + "additionalProperties": false, + "properties": { + "uploads": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "runner"], + "properties": { + "provider": { + "type": "string", + "enum": ["filecoin_onchain_cloud"] + }, + "runner": { + "type": "string", + "enum": ["background-function"] + }, + "privateKeyEnv": { + "type": "string", + "minLength": 1 + }, + "blobsStore": { + "type": "string", + "minLength": 1 + }, + "functionsDirectory": { + "type": "string", + "minLength": 1 + } + } + } + } + } + } + }, "theme": { "type": "object", "description": "Theme preset selection and future theme options.", diff --git a/packages/schema/src/lint.ts b/packages/schema/src/lint.ts index e34bb33..7c17e0e 100644 --- a/packages/schema/src/lint.ts +++ b/packages/schema/src/lint.ts @@ -72,6 +72,7 @@ export function lintThs(schema: ThsSchema): Issue[] { const issues: Issue[] = []; const themePreset = String(schema.app.theme?.preset ?? '').trim(); const generatedUi = schema.app.ui?.generated; + const netlifyUploadDeploy = schema.app.deploy?.netlify?.uploads; if (themePreset && themePreset !== 'cyber-grid') { issues.push( @@ -100,6 +101,16 @@ export function lintThs(schema: ThsSchema): Issue[] { ); } + if (netlifyUploadDeploy && schema.app.features?.uploads !== true) { + issues.push( + err( + '/app/deploy/netlify/uploads', + 'lint.app.deploy.netlify.uploads_requires_feature', + 'app.deploy.netlify.uploads requires app.features.uploads=true.' + ) + ); + } + const generatedFeedIds = new Set(); const generatedFeeds = Array.isArray(generatedUi?.feeds) ? generatedUi.feeds : []; for (const [index, feed] of generatedFeeds.entries()) { diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts index 4014c6c..c79e016 100644 --- a/packages/schema/src/types.ts +++ b/packages/schema/src/types.ts @@ -92,6 +92,22 @@ export interface ThsAppBrand { accentText?: string; } +export interface ThsNetlifyUploadDeploy { + provider: 'filecoin_onchain_cloud'; + runner: 'background-function'; + privateKeyEnv?: string; + blobsStore?: string; + functionsDirectory?: string; +} + +export interface ThsAppDeployNetlify { + uploads?: ThsNetlifyUploadDeploy; +} + +export interface ThsAppDeploy { + netlify?: ThsAppDeployNetlify; +} + export type ThsThemePreset = 'cyber-grid'; export interface ThsAppTheme { @@ -105,6 +121,7 @@ export interface ThsApp { description?: string; brand?: ThsAppBrand; primaryCollection?: string; + deploy?: ThsAppDeploy; theme?: ThsAppTheme; features?: ThsAppFeatures; ui?: ThsAppUi; diff --git a/schemas/tokenhost-ths.schema.json b/schemas/tokenhost-ths.schema.json index ba9a062..6327c69 100644 --- a/schemas/tokenhost-ths.schema.json +++ b/schemas/tokenhost-ths.schema.json @@ -38,6 +38,45 @@ } }, "primaryCollection": { "type": "string", "minLength": 1 }, + "deploy": { + "type": "object", + "additionalProperties": false, + "properties": { + "netlify": { + "type": "object", + "additionalProperties": false, + "properties": { + "uploads": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "runner"], + "properties": { + "provider": { + "type": "string", + "enum": ["filecoin_onchain_cloud"] + }, + "runner": { + "type": "string", + "enum": ["background-function"] + }, + "privateKeyEnv": { + "type": "string", + "minLength": 1 + }, + "blobsStore": { + "type": "string", + "minLength": 1 + }, + "functionsDirectory": { + "type": "string", + "minLength": 1 + } + } + } + } + } + } + }, "theme": { "type": "object", "description": "Theme preset selection and future theme options.", diff --git a/test/testCliBuildArtifacts.js b/test/testCliBuildArtifacts.js index 43e0d69..b82a536 100644 --- a/test/testCliBuildArtifacts.js +++ b/test/testCliBuildArtifacts.js @@ -110,4 +110,51 @@ describe('th build (artifacts)', function () { expect(fs.existsSync(path.join(outDir, 'ui-bundle', 'index.html'))).to.equal(true); expect(fs.existsSync(path.join(outDir, 'ui-site', 'index.html'))).to.equal(true); }); + + it('emits Netlify upload scaffolding when schema opts into Netlify background uploads', function () { + this.timeout(180000); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-build-netlify-uploads-')); + const schemaPath = path.join(dir, 'schema.json'); + const outDir = path.join(dir, 'out'); + writeJson( + schemaPath, + minimalSchema({ + app: { + name: 'Netlify Upload Test', + slug: 'netlify-upload-test', + features: { uploads: true, onChainIndexing: true }, + deploy: { + netlify: { + uploads: { + provider: 'filecoin_onchain_cloud', + runner: 'background-function' + } + } + } + } + }) + ); + + const res = runTh(['build', schemaPath, '--out', outDir], process.cwd()); + expect(res.status, res.stderr || res.stdout).to.equal(0); + + for (const p of [ + 'netlify.toml', + 'package.json', + 'NETLIFY-UPLOADS.md', + 'netlify/functions/_tokenhost-upload-shared.mjs', + 'netlify/functions/tokenhost-upload-start.mjs', + 'netlify/functions/tokenhost-upload-status.mjs', + 'netlify/functions/tokenhost-upload-worker-background.mjs' + ]) { + expect(fs.existsSync(path.join(outDir, p)), `missing ${p}`).to.equal(true); + } + + const manifest = JSON.parse(fs.readFileSync(path.join(outDir, 'manifest.json'), 'utf-8')); + expect(manifest?.extensions?.uploads?.endpointUrl).to.equal('/__tokenhost/upload'); + expect(manifest?.extensions?.uploads?.statusUrl).to.equal('/__tokenhost/upload-status'); + expect(manifest?.extensions?.uploads?.provider).to.equal('filecoin_onchain_cloud'); + expect(manifest?.extensions?.uploads?.runnerMode).to.equal('remote'); + }); }); diff --git a/test/testThsSchema.js b/test/testThsSchema.js index 2b2662b..698250b 100644 --- a/test/testThsSchema.js +++ b/test/testThsSchema.js @@ -149,6 +149,50 @@ describe('THS schema validation + lint', function () { expect(res.ok).to.equal(true); }); + it('validateThsStructural accepts Netlify background upload deployment config', function () { + const input = minimalSchema({ + app: { + name: 'Test App', + slug: 'test-app', + features: { uploads: true, onChainIndexing: true }, + deploy: { + netlify: { + uploads: { + provider: 'filecoin_onchain_cloud', + runner: 'background-function' + } + } + } + } + }); + + const res = validateThsStructural(input); + expect(res.ok).to.equal(true); + }); + + it('lintThs requires uploads feature when Netlify upload deploy config is used', function () { + const input = minimalSchema({ + app: { + name: 'Test App', + slug: 'test-app', + features: { uploads: false, onChainIndexing: true }, + deploy: { + netlify: { + uploads: { + provider: 'filecoin_onchain_cloud', + runner: 'background-function' + } + } + } + } + }); + + const res = validateThsStructural(input); + expect(res.ok).to.equal(true); + const issues = lintThs(res.data); + expect(issues.some((i) => i.code === 'lint.app.deploy.netlify.uploads_requires_feature')).to.equal(true); + }); + it('lintThs rejects unknown app.primaryCollection values', function () { const input = minimalSchema({ app: { From 10c663a722f0454fa56ac9b4188df8ea4e4e38a8 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 24 Mar 2026 18:53:46 -1000 Subject: [PATCH 2/5] Document Netlify upload deployment --- README.md | 15 ++++ docs/examples-microblog-netlify-upload.md | 88 +++++++++++++++++++++++ docs/examples-microblog-remote-upload.md | 1 - test/testCliBuildArtifacts.js | 16 +++++ 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 docs/examples-microblog-netlify-upload.md diff --git a/README.md b/README.md index 2f2057f..5e2e689 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,18 @@ Local preview endpoints: - `GET/POST /__tokenhost/relay` for sponsored local writes (anvil). - `GET/POST /__tokenhost/faucet` remains available for non-sponsored local mode. + +## Netlify uploads + +Token Host can now emit Netlify upload scaffolding for Filecoin Onchain Cloud: + +- schema surface: `app.deploy.netlify.uploads` +- build output includes: + - `netlify.toml` + - `netlify/functions/` + - `NETLIFY-UPLOADS.md` + +Runtime secrets for Netlify Functions must come from Netlify environment variables, not `netlify.toml`. + +See: +- `docs/examples-microblog-netlify-upload.md` diff --git a/docs/examples-microblog-netlify-upload.md b/docs/examples-microblog-netlify-upload.md new file mode 100644 index 0000000..c51f151 --- /dev/null +++ b/docs/examples-microblog-netlify-upload.md @@ -0,0 +1,88 @@ +# Microblog Example: Netlify Background Uploads + +This document shows how to generate the canonical microblog example so image uploads run through Netlify Background Functions and Filecoin Onchain Cloud instead of a VPS-hosted upload process. + +Relevant files: +- schema: `apps/example/microblog.schema.json` +- generated Netlify artifacts: `out/microblog/netlify.toml`, `out/microblog/netlify/functions/` + +## 1. Build the microblog app + +```bash +th build apps/example/microblog.schema.json --chain filecoin_calibration --out out/microblog +``` + +Because the schema includes: + +```json +{ + "app": { + "deploy": { + "netlify": { + "uploads": { + "provider": "filecoin_onchain_cloud", + "runner": "background-function" + } + } + } + } +} +``` + +the build emits: +- `out/microblog/netlify.toml` +- `out/microblog/netlify/functions/tokenhost-upload-start.mjs` +- `out/microblog/netlify/functions/tokenhost-upload-status.mjs` +- `out/microblog/netlify/functions/tokenhost-upload-worker-background.mjs` +- `out/microblog/NETLIFY-UPLOADS.md` + +The generated manifest is also wired so the browser upload client talks to: +- `POST /__tokenhost/upload` +- `GET /__tokenhost/upload-status?jobId=...` + +## 2. Set Netlify environment variables + +Set these in Netlify with **Functions** scope: + +- `TH_UPLOAD_FOC_PRIVATE_KEY` +- `TH_UPLOAD_FOC_CHAIN=314159` +- `TH_UPLOAD_FOC_COPIES=1` + +Optional: + +- `TH_UPLOAD_FOC_COMMAND=npx -y foc-cli` +- `TH_UPLOAD_FOC_WITH_CDN=true` +- `TH_UPLOAD_FOC_DEBUG=true` + +Important: +- runtime secrets must come from Netlify environment variables +- do not put the private key in `netlify.toml` + +## 3. Deploy to Netlify + +Deploy the generated build root so Netlify sees: +- `ui-site/` as the publish directory +- `netlify/functions/` as the functions directory + +The generated `netlify.toml` already configures those paths and the upload redirects. + +## 4. Expected runtime flow + +1. Browser uploads file bytes to `POST /__tokenhost/upload` +2. Start function stores the request in Netlify Blobs and returns `202` with `jobId` +3. Netlify background worker runs `foc-cli upload` +4. Worker stores success or failure result in Netlify Blobs +5. Browser polls `/__tokenhost/upload-status?jobId=...` until complete + +## Current status + +This generated target is meant to remove the VPS requirement for FOC uploads. + +What is implemented in Token Host now: +- schema/build support for a Netlify upload deployment target +- generated Netlify functions scaffolding +- manifest/runtime wiring compatible with the existing async browser upload client + +What still needs real-world validation: +- Netlify runtime compatibility for `foc-cli` +- final deploy ergonomics and any provider-specific edge cases diff --git a/docs/examples-microblog-remote-upload.md b/docs/examples-microblog-remote-upload.md index 9a720a3..fffe48a 100644 --- a/docs/examples-microblog-remote-upload.md +++ b/docs/examples-microblog-remote-upload.md @@ -4,7 +4,6 @@ This document shows how to run the canonical microblog example against a standal Relevant files: - schema: `apps/example/microblog.schema.json` -- UI overrides: `apps/example/microblog-ui/` - standalone adapter: `examples/upload-adapters/foc-remote-adapter.mjs` ## 1. Start a remote upload adapter diff --git a/test/testCliBuildArtifacts.js b/test/testCliBuildArtifacts.js index b82a536..9eccba0 100644 --- a/test/testCliBuildArtifacts.js +++ b/test/testCliBuildArtifacts.js @@ -152,9 +152,25 @@ describe('th build (artifacts)', function () { } const manifest = JSON.parse(fs.readFileSync(path.join(outDir, 'manifest.json'), 'utf-8')); + const netlifyToml = fs.readFileSync(path.join(outDir, 'netlify.toml'), 'utf-8'); + const generatedPkg = JSON.parse(fs.readFileSync(path.join(outDir, 'package.json'), 'utf-8')); + const startFn = path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-start.mjs'); + const statusFn = path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-status.mjs'); + const workerFn = path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-worker-background.mjs'); + expect(manifest?.extensions?.uploads?.endpointUrl).to.equal('/__tokenhost/upload'); expect(manifest?.extensions?.uploads?.statusUrl).to.equal('/__tokenhost/upload-status'); expect(manifest?.extensions?.uploads?.provider).to.equal('filecoin_onchain_cloud'); expect(manifest?.extensions?.uploads?.runnerMode).to.equal('remote'); + expect(netlifyToml).to.include('from = "/__tokenhost/upload"'); + expect(netlifyToml).to.include('to = "/.netlify/functions/tokenhost-upload-start"'); + expect(netlifyToml).to.include('from = "/__tokenhost/upload-status"'); + expect(netlifyToml).to.include('external_node_modules = ["@netlify/blobs"]'); + expect(generatedPkg?.dependencies?.['@netlify/blobs']).to.be.a('string'); + + for (const fnPath of [startFn, statusFn, workerFn]) { + const syntax = spawnSync(process.execPath, ['--check', fnPath], { encoding: 'utf-8' }); + expect(syntax.status, `${fnPath}\n${syntax.stderr || syntax.stdout}`).to.equal(0); + } }); }); From 8107c6503484915d2d1a606f543c99dcd14e224f Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 24 Mar 2026 18:58:57 -1000 Subject: [PATCH 3/5] Test generated Netlify upload functions --- test/fixtures/fake-foc-cli.mjs | 16 ++ .../testGeneratedNetlifyUploadFunctions.js | 187 ++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 test/fixtures/fake-foc-cli.mjs create mode 100644 test/integration/testGeneratedNetlifyUploadFunctions.js diff --git a/test/fixtures/fake-foc-cli.mjs b/test/fixtures/fake-foc-cli.mjs new file mode 100644 index 0000000..2420aab --- /dev/null +++ b/test/fixtures/fake-foc-cli.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +const payload = { + ok: true, + result: { + pieceCid: 'bafkqaaaafakecidfornetlifyuploadtest', + size: 321, + copyResults: [ + { + url: 'https://calibration.example.invalid/piece/bafkqaaaafakecidfornetlifyuploadtest' + } + ] + } +}; + +process.stdout.write(JSON.stringify(payload)); diff --git a/test/integration/testGeneratedNetlifyUploadFunctions.js b/test/integration/testGeneratedNetlifyUploadFunctions.js new file mode 100644 index 0000000..6f48ecf --- /dev/null +++ b/test/integration/testGeneratedNetlifyUploadFunctions.js @@ -0,0 +1,187 @@ +import { expect } from 'chai'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { pathToFileURL } from 'url'; +import { spawnSync } from 'child_process'; + +function writeJson(filePath, value) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(value, null, 2)); +} + +function runTh(args, cwd) { + return spawnSync('node', [path.resolve('packages/cli/dist/index.js'), ...args], { + cwd, + encoding: 'utf-8' + }); +} + +function minimalSchema(overrides = {}) { + return { + thsVersion: '2025-12', + schemaVersion: '0.0.1', + app: { + name: 'Netlify Upload Function Test', + slug: 'netlify-upload-function-test', + features: { uploads: true, onChainIndexing: true }, + deploy: { + netlify: { + uploads: { + provider: 'filecoin_onchain_cloud', + runner: 'background-function' + } + } + } + }, + collections: [ + { + name: 'Item', + fields: [{ name: 'image', type: 'image' }], + createRules: { required: [], access: 'public' }, + visibilityRules: { gets: ['image'], access: 'public' }, + updateRules: { mutable: ['image'], access: 'owner' }, + deleteRules: { softDelete: true, access: 'owner' }, + indexes: { unique: [], index: [] } + } + ], + ...overrides + }; +} + +function installNetlifyBlobsStub(outDir) { + const pkgDir = path.join(outDir, 'node_modules', '@netlify', 'blobs'); + fs.mkdirSync(pkgDir, { recursive: true }); + fs.writeFileSync( + path.join(pkgDir, 'package.json'), + JSON.stringify( + { + name: '@netlify/blobs', + type: 'module', + exports: './index.js' + }, + null, + 2 + ) + ); + fs.writeFileSync( + path.join(pkgDir, 'index.js'), + `const stores = globalThis.__tokenhostNetlifyBlobStores ?? (globalThis.__tokenhostNetlifyBlobStores = new Map()); + +export function getStore(name) { + if (!stores.has(name)) stores.set(name, new Map()); + const store = stores.get(name); + return { + async setJSON(key, value) { + store.set(key, JSON.parse(JSON.stringify(value))); + }, + async get(key, options = {}) { + if (!store.has(key)) return null; + const value = store.get(key); + return options.type === 'json' ? JSON.parse(JSON.stringify(value)) : value; + }, + async delete(key) { + store.delete(key); + } + }; +} +` + ); +} + +async function readJsonResponse(response) { + const text = await response.text(); + return text ? JSON.parse(text) : null; +} + +describe('Generated Netlify upload functions', function () { + it('processes an async upload job end to end with generated functions', async function () { + this.timeout(180000); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-netlify-functions-')); + const schemaPath = path.join(dir, 'schema.json'); + const outDir = path.join(dir, 'out'); + writeJson(schemaPath, minimalSchema()); + + const build = runTh(['build', schemaPath, '--out', outDir], process.cwd()); + expect(build.status, build.stderr || build.stdout).to.equal(0); + + installNetlifyBlobsStub(outDir); + + const startPath = pathToFileURL(path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-start.mjs')).href; + const statusPath = pathToFileURL(path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-status.mjs')).href; + const workerPath = pathToFileURL(path.join(outDir, 'netlify', 'functions', 'tokenhost-upload-worker-background.mjs')).href; + + const previousFetch = globalThis.fetch; + const previousPrivateKey = process.env.TH_UPLOAD_FOC_PRIVATE_KEY; + const previousCommand = process.env.TH_UPLOAD_FOC_COMMAND; + + process.env.TH_UPLOAD_FOC_PRIVATE_KEY = '0x1234'; + process.env.TH_UPLOAD_FOC_COMMAND = `node ${path.resolve('test/fixtures/fake-foc-cli.mjs')}`; + + try { + const workerModule = await import(workerPath); + globalThis.fetch = async (url, init = {}) => { + if (String(url).includes('/.netlify/functions/tokenhost-upload-worker-background')) { + return await workerModule.default( + new Request(String(url), { + method: init.method || 'POST', + headers: init.headers, + body: init.body + }) + ); + } + throw new Error(`Unexpected fetch in test: ${String(url)}`); + }; + + const startModule = await import(startPath); + const statusModule = await import(statusPath); + + const health = await startModule.default(new Request('https://example.net/__tokenhost/upload', { method: 'GET' })); + const healthJson = await readJsonResponse(health); + expect(health.status).to.equal(200); + expect(healthJson?.enabled).to.equal(true); + expect(healthJson?.provider).to.equal('filecoin_onchain_cloud'); + + const payload = Buffer.from('tokenhost-netlify-upload-test', 'utf-8'); + const startRes = await startModule.default( + new Request('https://example.net/__tokenhost/upload', { + method: 'POST', + headers: { + 'content-type': 'image/png', + 'x-tokenhost-upload-filename': 'test.png', + 'x-tokenhost-upload-size': String(payload.length) + }, + body: payload + }) + ); + const startJson = await readJsonResponse(startRes); + + expect(startRes.status).to.equal(202); + expect(startJson?.ok).to.equal(true); + expect(startJson?.pending).to.equal(true); + expect(startJson?.jobId).to.be.a('string'); + + const pollRes = await statusModule.default( + new Request(`https://example.net/__tokenhost/upload-status?jobId=${encodeURIComponent(startJson.jobId)}`, { + method: 'GET' + }) + ); + const pollJson = await readJsonResponse(pollRes); + + expect(pollRes.status).to.equal(200); + expect(pollJson?.ok).to.equal(true); + expect(pollJson?.pending).to.equal(false); + expect(pollJson?.upload?.url).to.equal('https://calibration.example.invalid/piece/bafkqaaaafakecidfornetlifyuploadtest'); + expect(pollJson?.upload?.cid).to.equal('bafkqaaaafakecidfornetlifyuploadtest'); + expect(pollJson?.upload?.provider).to.equal('filecoin_onchain_cloud'); + expect(pollJson?.upload?.runnerMode).to.equal('netlify-background'); + } finally { + globalThis.fetch = previousFetch; + if (previousPrivateKey === undefined) delete process.env.TH_UPLOAD_FOC_PRIVATE_KEY; + else process.env.TH_UPLOAD_FOC_PRIVATE_KEY = previousPrivateKey; + if (previousCommand === undefined) delete process.env.TH_UPLOAD_FOC_COMMAND; + else process.env.TH_UPLOAD_FOC_COMMAND = previousCommand; + } + }); +}); From b66fb866f536c9188e77a6186608a2e098028581 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 24 Mar 2026 20:15:40 -1000 Subject: [PATCH 4/5] Make Netlify FOC worker self-configuring --- packages/cli/src/index.ts | 28 +++++++++++++++++++----- test/fixtures/fake-foc-cli.mjs | 39 ++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e5e3ebb..ede1965 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2507,24 +2507,42 @@ export async function runFocUploadFromJob(job) { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tokenhost-netlify-upload-')); const tmpFile = path.join(tmpDir, job.fileName || 'upload.bin'); + const runtimeHome = path.join(os.tmpdir(), 'tokenhost-netlify-home'); + const xdgConfigHome = path.join(runtimeHome, '.config'); + const npmCacheDir = path.join(runtimeHome, '.npm'); + const normalizedPrivateKey = privateKey.startsWith('0x') ? privateKey : \`0x\${privateKey}\`; try { + await fs.mkdir(runtimeHome, { recursive: true }); + await fs.mkdir(xdgConfigHome, { recursive: true }); + await fs.mkdir(npmCacheDir, { recursive: true }); await fs.writeFile(tmpFile, Buffer.from(String(job.bodyBase64 || ''), 'base64')); const command = String(process.env.TH_UPLOAD_FOC_COMMAND ?? DEFAULT_FOC_COMMAND).trim() || DEFAULT_FOC_COMMAND; const chainId = parsePositiveInt(process.env.TH_UPLOAD_FOC_CHAIN, DEFAULT_CHAIN_ID); const copies = parsePositiveInt(process.env.TH_UPLOAD_FOC_COPIES, DEFAULT_COPIES); const withCDN = String(process.env.TH_UPLOAD_FOC_WITH_CDN ?? '').trim().toLowerCase(); const debug = String(process.env.TH_UPLOAD_FOC_DEBUG ?? '').trim().toLowerCase(); + const sharedEnv = { + ...process.env, + HOME: runtimeHome, + XDG_CONFIG_HOME: xdgConfigHome, + NPM_CONFIG_CACHE: npmCacheDir, + npm_config_cache: npmCacheDir, + NPM_CONFIG_UPDATE_NOTIFIER: 'false', + PRIVATE_KEY: privateKey, + TH_UPLOAD_FOC_PRIVATE_KEY: privateKey + }; + const focConfigDir = path.join(xdgConfigHome, 'foc-cli-nodejs'); + const focConfigPath = path.join(focConfigDir, 'config.json'); + await fs.mkdir(focConfigDir, { recursive: true }); + await fs.writeFile(focConfigPath, JSON.stringify({ privateKey: normalizedPrivateKey }) + '\\n', 'utf8'); + const shellCommand = \`\${command} upload \${shellQuote(tmpFile)} --format json --chain \${chainId} --copies \${copies}\` + ((withCDN === '1' || withCDN === 'true' || withCDN === 'yes' || withCDN === 'on') ? ' --withCDN true' : '') + ((debug === '1' || debug === 'true' || debug === 'yes' || debug === 'on') ? ' --debug true --verbose' : ''); - const result = await runShellCommand(shellCommand, { - ...process.env, - PRIVATE_KEY: privateKey, - TH_UPLOAD_FOC_PRIVATE_KEY: privateKey - }); + const result = await runShellCommand(shellCommand, sharedEnv); if (result.code !== 0) { throw new Error(String(result.stderr || result.stdout || \`foc-cli failed with status \${result.code}\`).trim()); diff --git a/test/fixtures/fake-foc-cli.mjs b/test/fixtures/fake-foc-cli.mjs index 2420aab..ad6e6a8 100644 --- a/test/fixtures/fake-foc-cli.mjs +++ b/test/fixtures/fake-foc-cli.mjs @@ -1,16 +1,29 @@ #!/usr/bin/env node -const payload = { - ok: true, - result: { - pieceCid: 'bafkqaaaafakecidfornetlifyuploadtest', - size: 321, - copyResults: [ - { - url: 'https://calibration.example.invalid/piece/bafkqaaaafakecidfornetlifyuploadtest' - } - ] - } -}; +const args = process.argv.slice(2); -process.stdout.write(JSON.stringify(payload)); +if (args[0] === 'wallet' && args[1] === 'init') { + process.stdout.write(JSON.stringify({ ok: true, initialized: true })); + process.exit(0); +} + +if (args[0] === 'upload') { + const payload = { + ok: true, + result: { + pieceCid: 'bafkqaaaafakecidfornetlifyuploadtest', + size: 321, + copyResults: [ + { + url: 'https://calibration.example.invalid/piece/bafkqaaaafakecidfornetlifyuploadtest' + } + ] + } + }; + + process.stdout.write(JSON.stringify(payload)); + process.exit(0); +} + +process.stderr.write(`Unexpected fake foc-cli args: ${args.join(' ')}\n`); +process.exit(1); From d8f0fe781f108c2070191f8c65dd3bca1909770f Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 24 Mar 2026 21:29:43 -1000 Subject: [PATCH 5/5] Fix Netlify upload CI regressions --- packages/cli/src/index.ts | 40 +++++++++++++------ test/integration/testGeneratedAppUiTests.js | 20 +++++++++- .../testMicroblogRemoteUploadFlow.js | 22 +++++++++- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ede1965..e97b379 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2324,6 +2324,19 @@ function resolveUploadManifestConfig(featuresUploads: boolean): UploadManifestCo }; } +function hasExplicitUploadManifestOverrideEnv(): boolean { + return [ + 'TH_UPLOAD_REMOTE_BASE_URL', + 'TH_UPLOAD_REMOTE_ENDPOINT_URL', + 'TH_UPLOAD_REMOTE_STATUS_URL', + 'TH_UPLOAD_RUNNER', + 'TH_UPLOAD_PROVIDER', + 'TH_UPLOAD_BASE_URL', + 'TH_UPLOAD_ACCEPT', + 'TH_UPLOAD_MAX_BYTES' + ].some((key) => String(process.env[key] ?? '').trim() !== ''); +} + function normalizeFunctionsDirectory(value: string | undefined): string { const trimmed = String(value ?? '').trim().replace(/^\/+|\/+$/g, ''); if (!trimmed) return 'netlify/functions'; @@ -3862,19 +3875,22 @@ function buildFromSchema( const zeroAddress = '0x0000000000000000000000000000000000000000'; const txMode = resolveTxMode(opts.txMode, Number(opts.targetChainId ?? anvil.id)); const relayBaseUrl = String(opts.relayBaseUrl ?? process.env.TH_RELAY_BASE_URL ?? '/__tokenhost/relay').trim() || '/__tokenhost/relay'; + const explicitUploadManifestConfig = hasExplicitUploadManifestOverrideEnv() ? resolveUploadManifestConfig(features.uploads) : null; const netlifyUploadBuild = resolveNetlifyUploadBuildConfig(schema); - const uploadConfig = netlifyUploadBuild - ? { - enabled: true, - baseUrl: netlifyUploadBuild.endpointUrl, - endpointUrl: netlifyUploadBuild.endpointUrl, - statusUrl: netlifyUploadBuild.statusUrl, - provider: netlifyUploadBuild.provider, - runnerMode: 'remote' as const, - accept: netlifyUploadBuild.accept, - maxBytes: netlifyUploadBuild.maxBytes - } - : resolveUploadManifestConfig(features.uploads); + const uploadConfig = + explicitUploadManifestConfig ?? + (netlifyUploadBuild + ? { + enabled: true, + baseUrl: netlifyUploadBuild.endpointUrl, + endpointUrl: netlifyUploadBuild.endpointUrl, + statusUrl: netlifyUploadBuild.statusUrl, + provider: netlifyUploadBuild.provider, + runnerMode: 'remote' as const, + accept: netlifyUploadBuild.accept, + maxBytes: netlifyUploadBuild.maxBytes + } + : resolveUploadManifestConfig(features.uploads)); const manifest = { manifestVersion: '0.1.0', diff --git a/test/integration/testGeneratedAppUiTests.js b/test/integration/testGeneratedAppUiTests.js index 1ee0574..f974184 100644 --- a/test/integration/testGeneratedAppUiTests.js +++ b/test/integration/testGeneratedAppUiTests.js @@ -1,5 +1,6 @@ import { expect } from 'chai'; import fs from 'fs'; +import net from 'net'; import os from 'os'; import path from 'path'; import { spawn, spawnSync } from 'child_process'; @@ -58,6 +59,23 @@ function waitForOutput(proc, pattern, timeoutMs) { }); } +function getAvailablePort(host) { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, host, () => { + const address = server.address(); + const port = typeof address === 'object' && address ? address.port : null; + server.close((err) => { + if (err) reject(err); + else if (!port) reject(new Error('Unable to determine an available port.')); + else resolve(port); + }); + }); + }); +} + describe('Generated app UI tests', function () { it('emits schema-aware UI smoke tests that pass against canonical job-board preview', async function () { this.timeout(240000); @@ -79,7 +97,7 @@ describe('Generated app UI tests', function () { expect(buildRes.status, buildRes.stderr || buildRes.stdout).to.equal(0); const host = '127.0.0.1'; - const port = 46000 + Math.floor(Math.random() * 1000); + const port = await getAvailablePort(host); const baseUrl = `http://${host}:${port}`; const preview = spawn( 'node', diff --git a/test/integration/testMicroblogRemoteUploadFlow.js b/test/integration/testMicroblogRemoteUploadFlow.js index 76de9af..56626eb 100644 --- a/test/integration/testMicroblogRemoteUploadFlow.js +++ b/test/integration/testMicroblogRemoteUploadFlow.js @@ -1,5 +1,6 @@ import { expect } from 'chai'; import fs from 'fs'; +import net from 'net'; import os from 'os'; import path from 'path'; import { spawn, spawnSync } from 'child_process'; @@ -57,6 +58,23 @@ async function request(url, init) { return { status: res.status, json, buffer, headers: res.headers }; } +function getAvailablePort(host) { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, host, () => { + const address = server.address(); + const port = typeof address === 'object' && address ? address.port : null; + server.close((err) => { + if (err) reject(err); + else if (!port) reject(new Error('Unable to determine an available port.')); + else resolve(port); + }); + }); + }); +} + describe('Microblog example remote upload adapter flow', function () { it('builds the canonical microblog app against a standalone remote upload adapter', async function () { this.timeout(180000); @@ -65,7 +83,7 @@ describe('Microblog example remote upload adapter flow', function () { const outDir = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'th-microblog-remote-')), 'out'); const adapterHost = '127.0.0.1'; - const adapterPort = 48000 + Math.floor(Math.random() * 1000); + const adapterPort = await getAvailablePort(adapterHost); const adapterBaseUrl = `http://${adapterHost}:${adapterPort}`; const adapterEndpointPath = '/api/upload'; const adapterStatusPath = '/api/upload/status'; @@ -87,7 +105,7 @@ describe('Microblog example remote upload adapter flow', function () { }); const previewHost = '127.0.0.1'; - const previewPort = 49000 + Math.floor(Math.random() * 1000); + const previewPort = await getAvailablePort(previewHost); const previewBaseUrl = `http://${previewHost}:${previewPort}`; let preview = null;