diff --git a/.github/scripts/download-utils.js b/.github/scripts/download-utils.js new file mode 100644 index 00000000..af54f607 --- /dev/null +++ b/.github/scripts/download-utils.js @@ -0,0 +1,49 @@ +/** + * Download Utilities + * Handles downloading assets from GitHub releases + */ +const { writeFile } = require('./file-utils'); + +/** + * Download file from URL with authentication + */ +async function downloadFile(url, filePath, token) { + const response = await fetch(url, { + headers: { + 'Authorization': `token ${token}`, + 'User-Agent': 'GitHub Actions' + } + }); + + if (!response.ok) { + throw new Error(`Failed to download: ${response.statusText}`); + } + + const buffer = await response.arrayBuffer(); + writeFile(filePath, Buffer.from(buffer)); +} + +/** + * Download asset with retry logic + */ +async function downloadAssetWithRetry(url, filePath, token, maxRetries = 3) { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await downloadFile(url, filePath, token); + return true; + } catch (error) { + console.error(`Download attempt ${attempt} failed: ${error.message}`); + if (attempt === maxRetries) { + throw error; + } + // Wait before retry (exponential backoff) + await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1))); + } + } + return false; +} + +module.exports = { + downloadFile, + downloadAssetWithRetry +}; diff --git a/.github/scripts/file-utils.js b/.github/scripts/file-utils.js new file mode 100644 index 00000000..52232cac --- /dev/null +++ b/.github/scripts/file-utils.js @@ -0,0 +1,43 @@ +/** + * File System Utilities + * Handles directory creation and file operations + */ +const fs = require('fs'); +const path = require('path'); + +/** + * Ensure directory exists, create if it doesn't + */ +function ensureDir(dirPath) { + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } +} + +/** + * Check if file exists + */ +function fileExists(filePath) { + return fs.existsSync(filePath); +} + +/** + * Write file safely + */ +function writeFile(filePath, content) { + fs.writeFileSync(filePath, content); +} + +/** + * Read file safely + */ +function readFile(filePath) { + return fs.readFileSync(filePath); +} + +module.exports = { + ensureDir, + fileExists, + writeFile, + readFile +}; diff --git a/.github/scripts/hash-utils.js b/.github/scripts/hash-utils.js new file mode 100644 index 00000000..55a164d0 --- /dev/null +++ b/.github/scripts/hash-utils.js @@ -0,0 +1,46 @@ +/** + * Hash Generation Utilities + * Handles generating hash files for downloaded assets + */ +const crypto = require('crypto'); +const { readFile, writeFile } = require('./file-utils'); + +/** + * Generate hash for a file using specified algorithm + */ +function generateHash(filePath, algorithm) { + const fileBuffer = readFile(filePath); + const hash = crypto.createHash(algorithm); + hash.update(fileBuffer); + return hash.digest('hex'); +} + +/** + * Generate all hash files for an asset + */ +function generateHashFiles(assetPath) { + console.log(`Generating hash files for: ${assetPath}`); + + try { + // Generate hashes + const sha256Hash = generateHash(assetPath, 'sha256'); + const sha512Hash = generateHash(assetPath, 'sha512'); + const md5Hash = generateHash(assetPath, 'md5'); + + // Write hash files + writeFile(`${assetPath}.sha256`, sha256Hash); + writeFile(`${assetPath}.sha512`, sha512Hash); + writeFile(`${assetPath}.md5`, md5Hash); + + console.log(`Hash files created for: ${assetPath}`); + return true; + } catch (error) { + console.error(`Failed to generate hash files for ${assetPath}: ${error.message}`); + return false; + } +} + +module.exports = { + generateHash, + generateHashFiles +}; diff --git a/.github/scripts/sync-assets.js b/.github/scripts/sync-assets.js new file mode 100644 index 00000000..a25baeb3 --- /dev/null +++ b/.github/scripts/sync-assets.js @@ -0,0 +1,196 @@ +/** + * Asset Synchronization Script + * Main script for downloading and organizing release assets + */ +const path = require('path'); +const { ensureDir, fileExists, writeFile } = require('./file-utils'); +const { generateHashFiles } = require('./hash-utils'); +const { downloadAssetWithRetry } = require('./download-utils'); + +/** + * Process a single repository and download its release assets + */ +async function processRepository(github, context, repo, repositoryData, totalAssets, isPullRequest = false, releaseLimit = null) { + console.log(`Processing repository: ${repo.name}`); + + let processedReleasesWithAssets = 0; + + try { + // Get releases for the repository with pagination + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: context.repo.owner, + repo: repo.name, + per_page: 100 + }); + + // Filter out draft and prerelease + const publishedReleases = releases.filter(release => !release.draft && !release.prerelease); + + if (publishedReleases.length === 0) { + console.log(`No published releases found for ${repo.name}`); + return { totalAssets, processedReleases: 0 }; + } + + const repoData = { + name: repo.name, + releases: [] + }; + + for (const release of publishedReleases) { + // For pull requests, stop after processing the specified number of releases with assets + if (isPullRequest && releaseLimit && processedReleasesWithAssets >= releaseLimit) { + console.log(`PR mode: Reached limit of ${releaseLimit} releases with assets for ${repo.name}`); + break; + } + + const assetCount = await processRelease(repo.name, release); + totalAssets += assetCount; + + if (assetCount > 0) { + repoData.releases.push({ + tag: release.tag_name, + assetCount: assetCount + }); + processedReleasesWithAssets++; + } + } + + if (repoData.releases.length > 0) { + repositoryData.push(repoData); + } + + } catch (error) { + console.error(`Error processing repository ${repo.name}: ${error.message}`); + } + + return { totalAssets, processedReleases: processedReleasesWithAssets }; +} + +/** + * Process a single release and download its assets + */ +async function processRelease(repoName, release) { + console.log(`Processing release: ${release.tag_name}`); + + if (release.assets.length === 0) { + console.log(`No assets found for release ${release.tag_name}`); + return 0; + } + + // Create directory structure + const releaseDir = path.join(repoName, release.tag_name); + ensureDir(releaseDir); + + let assetCount = 0; + + for (const asset of release.assets) { + const downloaded = await processAsset(releaseDir, asset); + if (downloaded) { + assetCount++; + } + } + + return assetCount; +} + +/** + * Process a single asset - download and generate hashes if not exists + */ +async function processAsset(releaseDir, asset) { + const assetPath = path.join(releaseDir, asset.name); + + // Skip if asset already exists + if (fileExists(assetPath)) { + console.log(`Asset already exists: ${assetPath}`); + return true; + } + + console.log(`Downloading: ${asset.name}`); + + try { + await downloadAssetWithRetry( + asset.browser_download_url, + assetPath, + process.env.GITHUB_TOKEN + ); + + console.log(`Successfully downloaded: ${assetPath}`); + + // Generate hash files + const hashSuccess = generateHashFiles(assetPath); + + return hashSuccess; + + } catch (error) { + console.error(`Failed to download ${asset.name}: ${error.message}`); + return false; + } +} + +/** + * Generate repository data JSON for the web interface + */ +function generateRepositoryDataJson(repositoryData, totalAssets) { + const dataJson = { + repositories: repositoryData, + lastUpdated: new Date().toISOString(), + totalRepositories: repositoryData.length, + totalReleases: repositoryData.reduce((sum, repo) => sum + repo.releases.length, 0), + totalAssets: totalAssets + }; + + writeFile('repository-data.json', JSON.stringify(dataJson, null, 2)); + console.log('Generated repository-data.json'); +} + +/** + * Main function to sync all release assets + */ +async function syncReleaseAssets(github, context, isPullRequest = false) { + console.log('Getting repositories from organization...'); + + if (isPullRequest) { + console.log('Running in pull request mode - limiting to 2 releases with assets per repository'); + } + + // Get all repositories with pagination + const repos = await github.paginate(github.rest.repos.listForOrg, { + org: context.repo.owner, + type: 'all', + per_page: 100 + }); + + console.log(`Found ${repos.length} repositories`); + + const repositoryData = []; + let totalAssets = 0; + let totalProcessedReleases = 0; + + // Process each repository + for (const repo of repos) { + const result = await processRepository( + github, + context, + repo, + repositoryData, + totalAssets, + isPullRequest, + isPullRequest ? 2 : null + ); + totalAssets = result.totalAssets; + totalProcessedReleases += result.processedReleases; + } + + // Generate repository data JSON for the index.html + generateRepositoryDataJson(repositoryData, totalAssets); + + if (isPullRequest) { + console.log(`PR mode: Processed ${repositoryData.length} repositories with ${totalProcessedReleases} releases containing assets`); + } else { + console.log(`Processed ${repositoryData.length} repositories with assets`); + } +} + +module.exports = { + syncReleaseAssets +}; diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml new file mode 100644 index 00000000..1038c27d --- /dev/null +++ b/.github/workflows/sync-release-assets.yml @@ -0,0 +1,68 @@ +--- +name: Sync Release Assets +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + workflow_dispatch: + pull_request: + types: + - opened + - reopened + - synchronize + schedule: + - cron: '0 * * * *' + +jobs: + sync-assets: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + token: ${{ secrets.GH_BOT_TOKEN }} + + - name: Create or checkout dist branch into dist directory + run: | + if git rev-parse --verify origin/dist >/dev/null 2>&1; then + echo "Dist branch exists, checking it out" + git worktree add dist origin/dist + else + echo "Dist branch doesn't exist, creating new one" + git worktree add --orphan dist + cd dist + git rm -rf . 2>/dev/null || true + fi + + - name: Copy gh-pages template files to dist directory + run: cp -r gh-pages-template/* dist/ + + - name: Get organization repositories and download assets + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + // Import the sync assets module + const { syncReleaseAssets } = require('./.github/scripts/sync-assets.js'); + + // Check if this is a pull request event + const isPullRequest = context.eventName === 'pull_request'; + + // Run the asset synchronization with PR limit + // Change working directory to dist for file operations + process.chdir('./dist'); + await syncReleaseAssets(github, context, isPullRequest); + + - name: Commit and push changes + if: github.event_name != 'pull_request' + uses: actions-js/push@v1.5 + with: + author_email: ${{ secrets.GH_BOT_EMAIL }} + author_name: ${{ secrets.GH_BOT_NAME }} + branch: dist + directory: dist + github_token: ${{ secrets.GH_BOT_TOKEN }} + message: 'Update release assets - ${{ github.run_id }}' diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e3f4af32 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# JetBrains IDEs +.idea/ diff --git a/README.md b/README.md index fd3e658c..5eb992ee 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,51 @@ -# template-base -Base repository template for LizardByte. +# LizardByte Package Repository + +This repository serves as a centralized storage for all packages and release assets from repositories within the +LizardByte organization. All release assets are automatically downloaded and organized in the `dist` branch for +easy access and distribution. + +## Structure + +The repository is organized as follows: + +``` +dist/ +├── repo-name-1/ +│ ├── v1.0.0/ +│ │ ├── asset1.zip +│ │ ├── asset1.zip.sha256 +│ │ ├── asset1.zip.sha512 +│ │ ├── asset1.zip.md5 +│ │ └── ... +│ └── v1.1.0/ +│ └── ... +├── repo-name-2/ +│ └── ... +└── ... +``` + +## Features + +- **Automated Collection**: GitHub Actions workflow automatically downloads release assets from all org repos +- **Hash Validation**: Each asset includes SHA256, SHA512, and MD5 hash files for integrity verification +- **Organized Structure**: Assets are organized by repository name and release tag +- **Incremental Updates**: Only new assets are downloaded to avoid duplication +- **Scheduled Updates**: Runs every hour to keep assets up-to-date + +## Workflow + +The release asset collection is handled by the `sync-release-assets.yml` workflow which: + +1. Discovers all repositories in the LizardByte organization +2. Fetches release information for each repository +3. Downloads missing release assets +4. Generates hash files for integrity verification +5. Commits changes to the `dist` branch + +## Usage + +All release assets are available in the `dist` branch of this repository. You can: + +- Browse assets directly on GitHub +- Clone the `dist` branch for offline access +- Use the hash files to verify asset integrity diff --git a/gh-pages-template/app.js b/gh-pages-template/app.js new file mode 100644 index 00000000..1c5d0da7 --- /dev/null +++ b/gh-pages-template/app.js @@ -0,0 +1,208 @@ +/** + * Repository Data Manager + * Handles loading and managing repository data from the JSON file + */ +class RepositoryDataManager { + constructor() { + this.repositoryData = []; + } + + /** + * Load repository data from the JSON file or fallback to directory scanning + */ + async loadRepositoryData() { + try { + // Try to load from the generated JSON file + const response = await fetch('./repository-data.json'); + if (response.ok) { + const data = await response.json(); + this.repositoryData = data.repositories || []; + return data; + } else { + // Fallback: scan the directory structure + await this.scanDirectoryStructure(); + return null; + } + } catch (error) { + console.log('Using directory scanning fallback'); + await this.scanDirectoryStructure(); + return null; + } + } + + /** + * Fallback method for scanning directory structure + * In a real GitHub Pages environment, we'd need the JSON data file + */ + async scanDirectoryStructure() { + // This is a simplified version that would work if we can list directories + // In a real GitHub Pages environment, we'd need the JSON data file + this.repositoryData = []; + } + + /** + * Get all repository data + */ + getRepositories() { + return this.repositoryData; + } + + /** + * Filter repositories based on search term + */ + filterRepositories(searchTerm) { + if (!searchTerm) { + return this.repositoryData; + } + + return this.repositoryData.filter(repo => { + const repoMatch = repo.name.toLowerCase().includes(searchTerm.toLowerCase()); + const releaseMatch = repo.releases.some(release => + release.tag.toLowerCase().includes(searchTerm.toLowerCase())); + return repoMatch || releaseMatch; + }); + } +} + +/** + * UI Manager + * Handles all DOM manipulation and rendering + */ +class UIManager { + constructor() { + this.repositoryGrid = document.getElementById('repositoryGrid'); + this.searchInput = document.getElementById('searchInput'); + this.repoCountElement = document.getElementById('repoCount'); + this.releaseCountElement = document.getElementById('releaseCount'); + this.assetCountElement = document.getElementById('assetCount'); + this.updateTimeElement = document.getElementById('updateTime'); + } + + /** + * Render repositories in the grid + */ + renderRepositories(repos) { + if (repos.length === 0) { + this.repositoryGrid.innerHTML = '
Centralized storage for all release assets from LizardByte repositories
+