From ad4eca6a811b545ce94192786bdd4f18ea4ba7b3 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Tue, 23 Sep 2025 18:04:33 -0400 Subject: [PATCH] fix: limit assets per run --- .github/scripts/sync-assets.js | 71 ++++++++++++++++++----- .github/workflows/sync-release-assets.yml | 10 +++- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/.github/scripts/sync-assets.js b/.github/scripts/sync-assets.js index a25baeb3..d269d712 100644 --- a/.github/scripts/sync-assets.js +++ b/.github/scripts/sync-assets.js @@ -10,10 +10,11 @@ const { downloadAssetWithRetry } = require('./download-utils'); /** * Process a single repository and download its release assets */ -async function processRepository(github, context, repo, repositoryData, totalAssets, isPullRequest = false, releaseLimit = null) { +async function processRepository(github, context, repo, repositoryData, totalAssets, isPullRequest = false, releaseLimit = null, maxNewAssets = 0, newAssetsDownloaded = 0) { console.log(`Processing repository: ${repo.name}`); let processedReleasesWithAssets = 0; + let repoNewAssets = 0; try { // Get releases for the repository with pagination @@ -28,7 +29,7 @@ async function processRepository(github, context, repo, repositoryData, totalAss if (publishedReleases.length === 0) { console.log(`No published releases found for ${repo.name}`); - return { totalAssets, processedReleases: 0 }; + return { totalAssets, processedReleases: 0, newAssetsDownloaded }; } const repoData = { @@ -37,14 +38,24 @@ async function processRepository(github, context, repo, repositoryData, totalAss }; for (const release of publishedReleases) { + // Check if we've reached the asset limit globally + if (maxNewAssets > 0 && (newAssetsDownloaded + repoNewAssets) >= maxNewAssets) { + console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing for ${repo.name}.`); + break; + } + // For pull requests, stop after processing the specified number of releases with assets if (isPullRequest && releaseLimit && processedReleasesWithAssets >= releaseLimit) { console.log(`PR mode: Reached limit of ${releaseLimit} releases with assets for ${repo.name}`); break; } - const assetCount = await processRelease(repo.name, release); + const result = await processRelease(repo.name, release, maxNewAssets, newAssetsDownloaded + repoNewAssets); + const assetCount = result.assetCount; + const newAssets = result.newAssets; + totalAssets += assetCount; + repoNewAssets += newAssets; if (assetCount > 0) { repoData.releases.push({ @@ -63,18 +74,18 @@ async function processRepository(github, context, repo, repositoryData, totalAss console.error(`Error processing repository ${repo.name}: ${error.message}`); } - return { totalAssets, processedReleases: processedReleasesWithAssets }; + return { totalAssets, processedReleases: processedReleasesWithAssets, newAssetsDownloaded: newAssetsDownloaded + repoNewAssets }; } /** * Process a single release and download its assets */ -async function processRelease(repoName, release) { +async function processRelease(repoName, release, maxNewAssets = 0, currentNewAssets = 0) { console.log(`Processing release: ${release.tag_name}`); if (release.assets.length === 0) { console.log(`No assets found for release ${release.tag_name}`); - return 0; + return { assetCount: 0, newAssets: 0 }; } // Create directory structure @@ -82,15 +93,25 @@ async function processRelease(repoName, release) { ensureDir(releaseDir); let assetCount = 0; + let newAssets = 0; for (const asset of release.assets) { - const downloaded = await processAsset(releaseDir, asset); - if (downloaded) { + // Check if we've reached the new assets download limit + if (maxNewAssets > 0 && (currentNewAssets + newAssets) >= maxNewAssets) { + console.log(`Reached maximum new assets limit (${maxNewAssets}) for this run. Stopping asset processing for release ${release.tag_name}.`); + break; + } + + const result = await processAsset(releaseDir, asset); + if (result.downloaded) { assetCount++; + if (result.isNew) { + newAssets++; + } } } - return assetCount; + return { assetCount, newAssets }; } /** @@ -102,7 +123,7 @@ async function processAsset(releaseDir, asset) { // Skip if asset already exists if (fileExists(assetPath)) { console.log(`Asset already exists: ${assetPath}`); - return true; + return { downloaded: true, isNew: false }; } console.log(`Downloading: ${asset.name}`); @@ -119,11 +140,11 @@ async function processAsset(releaseDir, asset) { // Generate hash files const hashSuccess = generateHashFiles(assetPath); - return hashSuccess; + return { downloaded: hashSuccess, isNew: true }; } catch (error) { console.error(`Failed to download ${asset.name}: ${error.message}`); - return false; + return { downloaded: false, isNew: false }; } } @@ -146,13 +167,19 @@ function generateRepositoryDataJson(repositoryData, totalAssets) { /** * Main function to sync all release assets */ -async function syncReleaseAssets(github, context, isPullRequest = false) { +async function syncReleaseAssets(github, context, isPullRequest = false, maxNewAssets = 0) { console.log('Getting repositories from organization...'); if (isPullRequest) { console.log('Running in pull request mode - limiting to 2 releases with assets per repository'); } + if (maxNewAssets > 0) { + console.log(`Asset download limit: ${maxNewAssets} new assets per run`); + } else { + console.log('Asset download limit: unlimited'); + } + // Get all repositories with pagination const repos = await github.paginate(github.rest.repos.listForOrg, { org: context.repo.owner, @@ -165,9 +192,16 @@ async function syncReleaseAssets(github, context, isPullRequest = false) { const repositoryData = []; let totalAssets = 0; let totalProcessedReleases = 0; + let newAssetsDownloaded = 0; // Process each repository for (const repo of repos) { + // Check if we've reached the asset limit + if (maxNewAssets > 0 && newAssetsDownloaded >= maxNewAssets) { + console.log(`Reached maximum new assets limit (${maxNewAssets}). Stopping processing.`); + break; + } + const result = await processRepository( github, context, @@ -175,10 +209,13 @@ async function syncReleaseAssets(github, context, isPullRequest = false) { repositoryData, totalAssets, isPullRequest, - isPullRequest ? 2 : null + isPullRequest ? 2 : null, + maxNewAssets, + newAssetsDownloaded ); totalAssets = result.totalAssets; totalProcessedReleases += result.processedReleases; + newAssetsDownloaded = result.newAssetsDownloaded; } // Generate repository data JSON for the index.html @@ -189,6 +226,12 @@ async function syncReleaseAssets(github, context, isPullRequest = false) { } else { console.log(`Processed ${repositoryData.length} repositories with assets`); } + + if (maxNewAssets > 0) { + console.log(`Downloaded ${newAssetsDownloaded} new assets (limit: ${maxNewAssets})`); + } else { + console.log(`Downloaded ${newAssetsDownloaded} new assets`); + } } module.exports = { diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml index 1038c27d..fc79490a 100644 --- a/.github/workflows/sync-release-assets.yml +++ b/.github/workflows/sync-release-assets.yml @@ -6,6 +6,9 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.ref }} +env: + MAX_NEW_ASSETS: 50 # Set to 0 for unlimited, or any positive number to limit downloads per run + on: workflow_dispatch: pull_request: @@ -51,10 +54,13 @@ jobs: // Check if this is a pull request event const isPullRequest = context.eventName === 'pull_request'; - // Run the asset synchronization with PR limit + // Get the max new assets limit from environment + const maxNewAssets = parseInt(process.env.MAX_NEW_ASSETS) || 0; + + // Run the asset synchronization with PR limit and asset limit // Change working directory to dist for file operations process.chdir('./dist'); - await syncReleaseAssets(github, context, isPullRequest); + await syncReleaseAssets(github, context, isPullRequest, maxNewAssets); - name: Commit and push changes if: github.event_name != 'pull_request'