diff --git a/.github/workflows/staging-release.yaml b/.github/workflows/staging-release.yaml index 68257382c35..62f930e8087 100644 --- a/.github/workflows/staging-release.yaml +++ b/.github/workflows/staging-release.yaml @@ -274,7 +274,9 @@ jobs: - name: Sync to release bucket on S3 run: | - aws s3 sync "packaging/releases/$CODENAME" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" --delete --follow-symlinks --no-progress + ./packaging/sync-apt-repo-to-s3.sh \ + "packaging/releases/$CODENAME" \ + "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/dockerfiles/Dockerfile.windows b/dockerfiles/Dockerfile.windows index 6168ae84c42..36753254c0a 100644 --- a/dockerfiles/Dockerfile.windows +++ b/dockerfiles/Dockerfile.windows @@ -32,35 +32,41 @@ ENV MSVS_BUILD_TOOLS_VERSION="$MSVS_VERSION" ` RUN $msvs_build_tools_dist_name=\"vs_buildtools.exe\"; ` $msvs_build_tools_dist=\"${env:TMP}\${msvs_build_tools_dist_name}\"; ` - $msvs_build_tools_channel=\"C:\local\VisualStudio.chman\"; ` $msvs_build_tools_dist_url=\"${env:MSVS_BUILD_TOOLS_DOWNLOAD_URL}/${env:MSVS_BUILD_TOOLS_VERSION}/release/${msvs_build_tools_dist_name}\"; ` - $msvs_build_tools_channel_url=\"${env:MSVS_BUILD_TOOLS_DOWNLOAD_URL}/${env:MSVS_BUILD_TOOLS_VERSION}/release/channel\"; ` Write-Host \"Downloading Visual Studio Build Tools...\"; ` Write-Host \"${msvs_build_tools_dist_url} -> ${msvs_build_tools_dist}\"; ` - Write-Host \"${msvs_build_tools_channel_url} -> ${msvs_build_tools_channel}\"; ` - Invoke-WebRequest -OutFile \"${msvs_build_tools_dist}\" \"${msvs_build_tools_dist_url}\"; ` - Invoke-WebRequest -OutFile \"${msvs_build_tools_channel}\" \"${msvs_build_tools_channel_url}\"; ` + $max_retries = 5; ` + for ($retry = 1; $retry -le $max_retries; $retry++) { ` + try { ` + Invoke-WebRequest -OutFile \"${msvs_build_tools_dist}\" \"${msvs_build_tools_dist_url}\"; ` + break; ` + } catch { ` + if ($retry -eq $max_retries) { ` + throw ` + }; ` + Write-Host \"Download attempt $retry failed: $($_.Exception.Message)\"; ` + Start-Sleep -Seconds (10 * $retry); ` + } ` + }; ` Write-Host \"Installing Visual Studio Build Tools into ${env:MSVS_HOME}...\"; ` $p = Start-Process \"${msvs_build_tools_dist}\" ` -ArgumentList '--quiet ', '--wait ', '--norestart ', '--nocache', ` \"--installPath ${env:MSVS_HOME}\", ` - \"--channelUri ${msvs_build_tools_channel}\", ` - \"--installChannelUri ${msvs_build_tools_channel}\", ` '--add Microsoft.VisualStudio.Workload.VCTools', ` '--add Microsoft.VisualStudio.Workload.MSBuildTools', ` '--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64', ` '--add Microsoft.VisualStudio.Component.VC.CoreBuildTools', ` '--add Microsoft.VisualStudio.Component.VC.MSVC.143', ` '--add Microsoft.VisualStudio.Component.Windows10SDK.19041' ` - -NoNewWindow -Wait; ` + -NoNewWindow -Wait -PassThru; ` + Write-Host \"Visual Studio Build Tools installer exit code: $($p.ExitCode)\"; ` if ($p.ExitCode -ne 0 -and $p.ExitCode -ne 3010) { ` throw \"Visual Studio Build Tools installer failed with exit code $($p.ExitCode)\" ` }; ` if (-not (Test-Path \"${env:MSVS_HOME}\VC\Auxiliary\Build\vcvars64.bat\")) { ` throw \"Visual Studio Build Tools installation is incomplete: ${env:MSVS_HOME}\VC\Auxiliary\Build\vcvars64.bat not found\" ` }; ` - Remove-Item -Force \"${msvs_build_tools_dist}\"; ` - Remove-Item -Path \"${msvs_build_tools_channel}\" -Force; + Remove-Item -Force \"${msvs_build_tools_dist}\"; ENV CMAKE_HOME="C:\cmake" ARG CMAKE_VERSION="3.31.6" diff --git a/packaging/sync-apt-repo-to-s3.sh b/packaging/sync-apt-repo-to-s3.sh new file mode 100755 index 00000000000..26b01e7aa59 --- /dev/null +++ b/packaging/sync-apt-repo-to-s3.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +# Upload an APT repo in two phases so Release metadata only becomes visible +# after the referenced package indexes and pool files are already uploaded. + +SOURCE_DIR=${1:?Usage: sync-apt-repo-to-s3.sh } +DESTINATION=${2:?Usage: sync-apt-repo-to-s3.sh } + +if [[ ! -d "$SOURCE_DIR" ]]; then + echo "ERROR: missing source dir: $SOURCE_DIR" + exit 1 +fi + +SOURCE_DIR=$(realpath "$SOURCE_DIR") + +aws s3 sync "$SOURCE_DIR" "$DESTINATION" \ + --delete \ + --follow-symlinks \ + --no-progress \ + --exclude "dists/*/InRelease" \ + --exclude "dists/*/Release" \ + --exclude "dists/*/Release.gpg" + +DIST_DIR="$SOURCE_DIR/dists" +if [[ ! -d "$DIST_DIR" ]]; then + echo "ERROR: missing dists dir in source: $DIST_DIR" + exit 1 +fi + +while IFS= read -r metadata_file; do + relative_path=${metadata_file#"$SOURCE_DIR"/} + aws s3 cp "$metadata_file" "$DESTINATION/$relative_path" --no-progress +done < <(find "$DIST_DIR" -type f \( -name "InRelease" -o -name "Release" -o -name "Release.gpg" \) | sort)