From 3f8ecfa7f7045c8eaf90acbd0f59e412cb4fb079 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Tue, 23 Sep 2025 16:26:33 -0400 Subject: [PATCH 1/5] feat: initial version --- .github/workflows/sync-release-assets.yml | 147 ++++++++++++++++++++++ .gitignore | 2 + README.md | 53 +++++++- 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/sync-release-assets.yml create mode 100644 .gitignore diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml new file mode 100644 index 00000000..da7752bc --- /dev/null +++ b/.github/workflows/sync-release-assets.yml @@ -0,0 +1,147 @@ +--- +name: Sync Release Assets +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + workflow_dispatch: + schedule: + - cron: '0 * * * *' + +jobs: + sync-assets: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + token: ${{ secrets.GH_BOT_TOKEN }} + + - name: Configure Git + run: | + git config --global user.name "${{ secrets.GH_BOT_NAME }}" + git config --global user.email "${{ secrets.GH_BOT_EMAIL }}" + + - name: Create or checkout dist branch + run: | + if git rev-parse --verify origin/dist >/dev/null 2>&1; then + git checkout dist + else + git checkout --orphan dist + git rm -rf . + echo "# Release Assets" > README.md + git add README.md + git commit -m "Initialize dist branch" + fi + + - name: Get organization repositories + id: get-repos + run: | + echo "Getting repositories from LizardByte organization..." + + # Get all repositories in the organization + repos=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ + "https://api.github.com/orgs/LizardByte/repos?per_page=100" | \ + jq -r '.[].name' | grep -v "^$") + + echo "Found repositories:" + echo "${repos}" + + # Save repos to file for processing + echo "${repos}" > repos.txt + + - name: Download release assets + run: | + ORG="${{ github.repository_owner }}" + + while IFS= read -r repo; do + if [ -z "${repo}" ]; then + continue + fi + + echo "Processing repository: ${repo}" + + # Get releases for the repository + releases=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ + "https://api.github.com/repos/$ORG/$repo/releases" | \ + jq -r '.[] | select(.draft == false and .prerelease == false) | "\(.tag_name)|\(.id)"') + + if [ -z "${releases}" ]; then + echo "No releases found for $repo" + continue + fi + + echo "${releases}" | while IFS='|' read -r tag_name release_id; do + if [ -z "${tag_name}" ] || [ -z "${release_id}" ]; then + continue + fi + + echo "Processing release: ${tag_name}" + + # Create directory structure + release_dir="${repo}/${tag_name}" + mkdir -p "${release_dir}" + + # Get assets for this release + assets=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ + "https://api.github.com/repos/$ORG/$repo/releases/$release_id" | \ + jq -r '.assets[] | "\(.name)|\(.browser_download_url)"') + + if [ -z "${assets}" ]; then + echo "No assets found for release ${tag_name}" + continue + fi + + echo "${assets}" | while IFS='|' read -r asset_name download_url; do + if [ -z "${asset_name}" ] || [ -z "${download_url}" ]; then + continue + fi + + asset_path="${release_dir}/${asset_name}" + if [ -f "${asset_path}" ]; then + echo "Asset already exists: ${asset_path}" + continue + fi + + echo "Downloading: ${asset_name}" + curl -s -L -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ + -o "$asset_path" "${download_url}" + + if [ -f "${asset_path}" ]; then + echo "Successfully downloaded: ${asset_path}" + echo "Generating hash files for: ${asset_name}" + sha256sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.sha256" + sha512sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.sha512" + md5sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.md5" + echo "Hash files created for: ${asset_name}" + else + echo "Failed to download: ${asset_name}" + fi + done + done + done < repos.txt + + - name: Commit and push changes + run: | + # Check if there are any changes + if [ -n "$(git status --porcelain)" ]; then + echo "Changes detected, committing..." + + # Add all changes + git add . + + # Create commit message with timestamp + commit_msg="Update release assets - $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + git commit -m "${commit_msg}" + + # Push changes + git push origin dist + + echo "Changes committed and pushed to dist branch" + else + echo "No changes detected, nothing to commit" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e3f4af32 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# JetBrains IDEs +.idea/ diff --git a/README.md b/README.md index fd3e658c..5eb992ee 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,51 @@ -# template-base -Base repository template for LizardByte. +# LizardByte Package Repository + +This repository serves as a centralized storage for all packages and release assets from repositories within the +LizardByte organization. All release assets are automatically downloaded and organized in the `dist` branch for +easy access and distribution. + +## Structure + +The repository is organized as follows: + +``` +dist/ +├── repo-name-1/ +│ ├── v1.0.0/ +│ │ ├── asset1.zip +│ │ ├── asset1.zip.sha256 +│ │ ├── asset1.zip.sha512 +│ │ ├── asset1.zip.md5 +│ │ └── ... +│ └── v1.1.0/ +│ └── ... +├── repo-name-2/ +│ └── ... +└── ... +``` + +## Features + +- **Automated Collection**: GitHub Actions workflow automatically downloads release assets from all org repos +- **Hash Validation**: Each asset includes SHA256, SHA512, and MD5 hash files for integrity verification +- **Organized Structure**: Assets are organized by repository name and release tag +- **Incremental Updates**: Only new assets are downloaded to avoid duplication +- **Scheduled Updates**: Runs every hour to keep assets up-to-date + +## Workflow + +The release asset collection is handled by the `sync-release-assets.yml` workflow which: + +1. Discovers all repositories in the LizardByte organization +2. Fetches release information for each repository +3. Downloads missing release assets +4. Generates hash files for integrity verification +5. Commits changes to the `dist` branch + +## Usage + +All release assets are available in the `dist` branch of this repository. You can: + +- Browse assets directly on GitHub +- Clone the `dist` branch for offline access +- Use the hash files to verify asset integrity From cc6a8209d44d893ba6f47a6c6f3003d989af66ee Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Tue, 23 Sep 2025 16:47:24 -0400 Subject: [PATCH 2/5] refactor: organize code structure --- .github/scripts/download-utils.js | 49 +++++ .github/scripts/file-utils.js | 43 +++++ .github/scripts/hash-utils.js | 46 +++++ .github/scripts/sync-assets.js | 168 +++++++++++++++++ .github/workflows/sync-release-assets.yml | 131 +++----------- gh-pages-template/app.js | 208 ++++++++++++++++++++++ gh-pages-template/index.html | 46 +++++ gh-pages-template/styles.css | 180 +++++++++++++++++++ 8 files changed, 763 insertions(+), 108 deletions(-) create mode 100644 .github/scripts/download-utils.js create mode 100644 .github/scripts/file-utils.js create mode 100644 .github/scripts/hash-utils.js create mode 100644 .github/scripts/sync-assets.js create mode 100644 gh-pages-template/app.js create mode 100644 gh-pages-template/index.html create mode 100644 gh-pages-template/styles.css diff --git a/.github/scripts/download-utils.js b/.github/scripts/download-utils.js new file mode 100644 index 00000000..af54f607 --- /dev/null +++ b/.github/scripts/download-utils.js @@ -0,0 +1,49 @@ +/** + * Download Utilities + * Handles downloading assets from GitHub releases + */ +const { writeFile } = require('./file-utils'); + +/** + * Download file from URL with authentication + */ +async function downloadFile(url, filePath, token) { + const response = await fetch(url, { + headers: { + 'Authorization': `token ${token}`, + 'User-Agent': 'GitHub Actions' + } + }); + + if (!response.ok) { + throw new Error(`Failed to download: ${response.statusText}`); + } + + const buffer = await response.arrayBuffer(); + writeFile(filePath, Buffer.from(buffer)); +} + +/** + * Download asset with retry logic + */ +async function downloadAssetWithRetry(url, filePath, token, maxRetries = 3) { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + await downloadFile(url, filePath, token); + return true; + } catch (error) { + console.error(`Download attempt ${attempt} failed: ${error.message}`); + if (attempt === maxRetries) { + throw error; + } + // Wait before retry (exponential backoff) + await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1))); + } + } + return false; +} + +module.exports = { + downloadFile, + downloadAssetWithRetry +}; diff --git a/.github/scripts/file-utils.js b/.github/scripts/file-utils.js new file mode 100644 index 00000000..52232cac --- /dev/null +++ b/.github/scripts/file-utils.js @@ -0,0 +1,43 @@ +/** + * File System Utilities + * Handles directory creation and file operations + */ +const fs = require('fs'); +const path = require('path'); + +/** + * Ensure directory exists, create if it doesn't + */ +function ensureDir(dirPath) { + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } +} + +/** + * Check if file exists + */ +function fileExists(filePath) { + return fs.existsSync(filePath); +} + +/** + * Write file safely + */ +function writeFile(filePath, content) { + fs.writeFileSync(filePath, content); +} + +/** + * Read file safely + */ +function readFile(filePath) { + return fs.readFileSync(filePath); +} + +module.exports = { + ensureDir, + fileExists, + writeFile, + readFile +}; diff --git a/.github/scripts/hash-utils.js b/.github/scripts/hash-utils.js new file mode 100644 index 00000000..55a164d0 --- /dev/null +++ b/.github/scripts/hash-utils.js @@ -0,0 +1,46 @@ +/** + * Hash Generation Utilities + * Handles generating hash files for downloaded assets + */ +const crypto = require('crypto'); +const { readFile, writeFile } = require('./file-utils'); + +/** + * Generate hash for a file using specified algorithm + */ +function generateHash(filePath, algorithm) { + const fileBuffer = readFile(filePath); + const hash = crypto.createHash(algorithm); + hash.update(fileBuffer); + return hash.digest('hex'); +} + +/** + * Generate all hash files for an asset + */ +function generateHashFiles(assetPath) { + console.log(`Generating hash files for: ${assetPath}`); + + try { + // Generate hashes + const sha256Hash = generateHash(assetPath, 'sha256'); + const sha512Hash = generateHash(assetPath, 'sha512'); + const md5Hash = generateHash(assetPath, 'md5'); + + // Write hash files + writeFile(`${assetPath}.sha256`, sha256Hash); + writeFile(`${assetPath}.sha512`, sha512Hash); + writeFile(`${assetPath}.md5`, md5Hash); + + console.log(`Hash files created for: ${assetPath}`); + return true; + } catch (error) { + console.error(`Failed to generate hash files for ${assetPath}: ${error.message}`); + return false; + } +} + +module.exports = { + generateHash, + generateHashFiles +}; diff --git a/.github/scripts/sync-assets.js b/.github/scripts/sync-assets.js new file mode 100644 index 00000000..6569eac2 --- /dev/null +++ b/.github/scripts/sync-assets.js @@ -0,0 +1,168 @@ +/** + * Asset Synchronization Script + * Main script for downloading and organizing release assets + */ +const path = require('path'); +const { ensureDir, fileExists, writeFile } = require('./file-utils'); +const { generateHashFiles } = require('./hash-utils'); +const { downloadAssetWithRetry } = require('./download-utils'); + +/** + * Process a single repository and download its release assets + */ +async function processRepository(github, context, repo, repositoryData, totalAssets) { + console.log(`Processing repository: ${repo.name}`); + + try { + // Get releases for the repository with pagination + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: context.repo.owner, + repo: repo.name, + per_page: 100 + }); + + // Filter out draft and prerelease + const publishedReleases = releases.filter(release => !release.draft && !release.prerelease); + + if (publishedReleases.length === 0) { + console.log(`No published releases found for ${repo.name}`); + return totalAssets; + } + + const repoData = { + name: repo.name, + releases: [] + }; + + for (const release of publishedReleases) { + const assetCount = await processRelease(repo.name, release); + totalAssets += assetCount; + + if (assetCount > 0) { + repoData.releases.push({ + tag: release.tag_name, + assetCount: assetCount + }); + } + } + + if (repoData.releases.length > 0) { + repositoryData.push(repoData); + } + + } catch (error) { + console.error(`Error processing repository ${repo.name}: ${error.message}`); + } + + return totalAssets; +} + +/** + * Process a single release and download its assets + */ +async function processRelease(repoName, release) { + console.log(`Processing release: ${release.tag_name}`); + + if (release.assets.length === 0) { + console.log(`No assets found for release ${release.tag_name}`); + return 0; + } + + // Create directory structure + const releaseDir = path.join(repoName, release.tag_name); + ensureDir(releaseDir); + + let assetCount = 0; + + for (const asset of release.assets) { + const downloaded = await processAsset(releaseDir, asset); + if (downloaded) { + assetCount++; + } + } + + return assetCount; +} + +/** + * Process a single asset - download and generate hashes if not exists + */ +async function processAsset(releaseDir, asset) { + const assetPath = path.join(releaseDir, asset.name); + + // Skip if asset already exists + if (fileExists(assetPath)) { + console.log(`Asset already exists: ${assetPath}`); + return true; + } + + console.log(`Downloading: ${asset.name}`); + + try { + await downloadAssetWithRetry( + asset.browser_download_url, + assetPath, + process.env.GITHUB_TOKEN + ); + + console.log(`Successfully downloaded: ${assetPath}`); + + // Generate hash files + const hashSuccess = generateHashFiles(assetPath); + + return hashSuccess; + + } catch (error) { + console.error(`Failed to download ${asset.name}: ${error.message}`); + return false; + } +} + +/** + * Generate repository data JSON for the web interface + */ +function generateRepositoryDataJson(repositoryData, totalAssets) { + const dataJson = { + repositories: repositoryData, + lastUpdated: new Date().toISOString(), + totalRepositories: repositoryData.length, + totalReleases: repositoryData.reduce((sum, repo) => sum + repo.releases.length, 0), + totalAssets: totalAssets + }; + + writeFile('repository-data.json', JSON.stringify(dataJson, null, 2)); + console.log('Generated repository-data.json'); +} + +/** + * Main function to sync all release assets + */ +async function syncReleaseAssets(github, context) { + console.log('Getting repositories from organization...'); + + // Get all repositories with pagination + const repos = await github.paginate(github.rest.repos.listForOrg, { + org: context.repo.owner, + type: 'all', + per_page: 100 + }); + + console.log(`Found ${repos.length} repositories`); + + const repositoryData = []; + let totalAssets = 0; + + // Process each repository + for (const repo of repos) { + totalAssets = await processRepository(github, context, repo, repositoryData, totalAssets); + } + + // Generate repository data JSON for the index.html + generateRepositoryDataJson(repositoryData, totalAssets); + + console.log(`Processed ${repositoryData.length} repositories with assets`); +} + +module.exports = { + syncReleaseAssets +}; diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml index da7752bc..95ae350d 100644 --- a/.github/workflows/sync-release-assets.yml +++ b/.github/workflows/sync-release-assets.yml @@ -21,11 +21,6 @@ jobs: fetch-depth: 0 token: ${{ secrets.GH_BOT_TOKEN }} - - name: Configure Git - run: | - git config --global user.name "${{ secrets.GH_BOT_NAME }}" - git config --global user.email "${{ secrets.GH_BOT_EMAIL }}" - - name: Create or checkout dist branch run: | if git rev-parse --verify origin/dist >/dev/null 2>&1; then @@ -38,110 +33,30 @@ jobs: git commit -m "Initialize dist branch" fi - - name: Get organization repositories - id: get-repos - run: | - echo "Getting repositories from LizardByte organization..." - - # Get all repositories in the organization - repos=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ - "https://api.github.com/orgs/LizardByte/repos?per_page=100" | \ - jq -r '.[].name' | grep -v "^$") - - echo "Found repositories:" - echo "${repos}" - - # Save repos to file for processing - echo "${repos}" > repos.txt - - - name: Download release assets + - name: Copy gh-pages template files from main branch run: | - ORG="${{ github.repository_owner }}" - - while IFS= read -r repo; do - if [ -z "${repo}" ]; then - continue - fi - - echo "Processing repository: ${repo}" - - # Get releases for the repository - releases=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ - "https://api.github.com/repos/$ORG/$repo/releases" | \ - jq -r '.[] | select(.draft == false and .prerelease == false) | "\(.tag_name)|\(.id)"') - - if [ -z "${releases}" ]; then - echo "No releases found for $repo" - continue - fi - - echo "${releases}" | while IFS='|' read -r tag_name release_id; do - if [ -z "${tag_name}" ] || [ -z "${release_id}" ]; then - continue - fi - - echo "Processing release: ${tag_name}" - - # Create directory structure - release_dir="${repo}/${tag_name}" - mkdir -p "${release_dir}" - - # Get assets for this release - assets=$(curl -s -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ - "https://api.github.com/repos/$ORG/$repo/releases/$release_id" | \ - jq -r '.assets[] | "\(.name)|\(.browser_download_url)"') - - if [ -z "${assets}" ]; then - echo "No assets found for release ${tag_name}" - continue - fi - - echo "${assets}" | while IFS='|' read -r asset_name download_url; do - if [ -z "${asset_name}" ] || [ -z "${download_url}" ]; then - continue - fi - - asset_path="${release_dir}/${asset_name}" - if [ -f "${asset_path}" ]; then - echo "Asset already exists: ${asset_path}" - continue - fi - - echo "Downloading: ${asset_name}" - curl -s -L -H "Authorization: token ${{ secrets.GH_BOT_TOKEN }}" \ - -o "$asset_path" "${download_url}" + git checkout main -- gh-pages-template/ + if [ -d "gh-pages-template" ]; then + cp -r gh-pages-template/* . + rm -rf gh-pages-template/ + fi - if [ -f "${asset_path}" ]; then - echo "Successfully downloaded: ${asset_path}" - echo "Generating hash files for: ${asset_name}" - sha256sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.sha256" - sha512sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.sha512" - md5sum "${asset_path}" | cut -d' ' -f1 > "${asset_path}.md5" - echo "Hash files created for: ${asset_name}" - else - echo "Failed to download: ${asset_name}" - fi - done - done - done < repos.txt + - name: Get organization repositories and download assets + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + // Import the sync assets module + const { syncReleaseAssets } = require('./.github/scripts/sync-assets.js'); + + // Run the asset synchronization + await syncReleaseAssets(github, context); - name: Commit and push changes - run: | - # Check if there are any changes - if [ -n "$(git status --porcelain)" ]; then - echo "Changes detected, committing..." - - # Add all changes - git add . - - # Create commit message with timestamp - commit_msg="Update release assets - $(date -u '+%Y-%m-%d %H:%M:%S UTC')" - git commit -m "${commit_msg}" - - # Push changes - git push origin dist - - echo "Changes committed and pushed to dist branch" - else - echo "No changes detected, nothing to commit" - fi + uses: actions-js/push@v1.5 + with: + github_token: ${{ secrets.GH_BOT_TOKEN }} + branch: dist + message: 'Update release assets - ${{ github.run_id }}' + author_name: ${{ secrets.GH_BOT_NAME }} + author_email: ${{ secrets.GH_BOT_EMAIL }} diff --git a/gh-pages-template/app.js b/gh-pages-template/app.js new file mode 100644 index 00000000..1c5d0da7 --- /dev/null +++ b/gh-pages-template/app.js @@ -0,0 +1,208 @@ +/** + * Repository Data Manager + * Handles loading and managing repository data from the JSON file + */ +class RepositoryDataManager { + constructor() { + this.repositoryData = []; + } + + /** + * Load repository data from the JSON file or fallback to directory scanning + */ + async loadRepositoryData() { + try { + // Try to load from the generated JSON file + const response = await fetch('./repository-data.json'); + if (response.ok) { + const data = await response.json(); + this.repositoryData = data.repositories || []; + return data; + } else { + // Fallback: scan the directory structure + await this.scanDirectoryStructure(); + return null; + } + } catch (error) { + console.log('Using directory scanning fallback'); + await this.scanDirectoryStructure(); + return null; + } + } + + /** + * Fallback method for scanning directory structure + * In a real GitHub Pages environment, we'd need the JSON data file + */ + async scanDirectoryStructure() { + // This is a simplified version that would work if we can list directories + // In a real GitHub Pages environment, we'd need the JSON data file + this.repositoryData = []; + } + + /** + * Get all repository data + */ + getRepositories() { + return this.repositoryData; + } + + /** + * Filter repositories based on search term + */ + filterRepositories(searchTerm) { + if (!searchTerm) { + return this.repositoryData; + } + + return this.repositoryData.filter(repo => { + const repoMatch = repo.name.toLowerCase().includes(searchTerm.toLowerCase()); + const releaseMatch = repo.releases.some(release => + release.tag.toLowerCase().includes(searchTerm.toLowerCase())); + return repoMatch || releaseMatch; + }); + } +} + +/** + * UI Manager + * Handles all DOM manipulation and rendering + */ +class UIManager { + constructor() { + this.repositoryGrid = document.getElementById('repositoryGrid'); + this.searchInput = document.getElementById('searchInput'); + this.repoCountElement = document.getElementById('repoCount'); + this.releaseCountElement = document.getElementById('releaseCount'); + this.assetCountElement = document.getElementById('assetCount'); + this.updateTimeElement = document.getElementById('updateTime'); + } + + /** + * Render repositories in the grid + */ + renderRepositories(repos) { + if (repos.length === 0) { + this.repositoryGrid.innerHTML = '
Centralized storage for all release assets from LizardByte repositories
+