feat: download kubelet with oras in network isolated windows cluster

e2e/scenario_win_test.go

@@ -546,3 +546,32 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
 		},
 	})
 }
+
+func Test_NetworkIsolatedCluster_Windows_OrasKubeletDownload(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "Tests that Windows nodes in network isolated clusters download kubelet binaries via ORAS when BootstrapProfileContainerRegistryServer is set",
+		Tags: Tags{
+			NetworkIsolated: true,
+			NonAnonymousACR: false,
+		},
+		Config: Config{
+			Cluster:         ClusterAzureBootstrapProfileCache,
+			VHD:             config.VHDWindows2025Gen2,
+			VMConfigMutator: EmptyVMConfigMutator,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				Windows2025BootstrapConfigMutator(t, nbc)
+				nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
+					PrivateEgress: &datamodel.PrivateEgress{
+						Enabled:                 true,
+						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
+					},
+				}
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote")
+				// Verify kubelet binaries were downloaded via ORAS instead of HTTP
+				ValidateFileHasContent(ctx, s, "/AzureData/CustomDataSetupScript.log", "Start to download kubelet binaries with oras")
+			},
+		},
+	})
+}

staging/cse/windows/containerdfunc.tests.ps1

@@ -350,4 +350,9 @@ Describe "Set-BootstrapProfileRegistryContainerdHost" {
 
     $script:capturedContent | Should -Match '\[host\."https://myacr.azurecr.io/v2/aaa"\]'
   }
+
+  AfterEach {
+    $global:BootstrapProfileContainerRegistryServer = $null
+    $global:MCRRepositoryBase = $null
+  }
 }

staging/cse/windows/kubeletfunc.ps1

@@ -193,19 +193,40 @@ function Get-KubePackage {
             Write-Log "Did not find $global:KubeBinariesVersion in $mappingFile"
         }
     }
-    Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubletBinaries" -TaskMessage "Start to download kubelet binaries and unzip. KubeBinariesPackageSASURL: $KubeBinariesSASURL"
 
     $zipfile = "c:\k.zip"
-    for ($i = 0; $i -le 10; $i++) {
-        DownloadFileOverHttp -Url $KubeBinariesSASURL -DestinationPath $zipfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE
-        if ($?) {
-            break
+
+
+    if ([string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) {
+        # default path
+        # download kubelet binaries via http if BootstrapProfileContainerRegistryServer is not set
+        Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubeletBinaries" -TaskMessage "Start to download kubelet binaries and unzip. KubeBinariesPackageSASURL: $KubeBinariesSASURL"
+
+        for ($i = 0; $i -le 10; $i++) {
+            DownloadFileOverHttp -Url $KubeBinariesSASURL -DestinationPath $zipfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE
+            if ($?) {
+                break
+            }
+            else {
+                Write-Log $Error[0].Exception.Message
+            }
         }
-        else {
-            Write-Log $Error[0].Exception.Message
+    } else {
+        # ni path
+        # download kubelet binaries via oras if BootstrapProfileContainerRegistryServer is set
+        if (-not (Get-Command 'DownloadFileWithOras' -ErrorAction SilentlyContinue)) {
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL -ErrorMessage "DownloadFileWithOras function is not available. networkisolatedclusterfunc.ps1 may not be sourced."
+        }
+        Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubeletBinariesWithOras" -TaskMessage "Start to download kubelet binaries with oras. KubeBinariesVersion: $global:KubeBinariesVersion, BootstrapProfileContainerRegistryServer: $global:BootstrapProfileContainerRegistryServer"
+        $orasReference = "$($global:BootstrapProfileContainerRegistryServer)/aks/packages/kubernetes/windowszip:v$($global:KubeBinariesVersion)"
+        try {
+            Retry-Command -Command "DownloadFileWithOras" -Args @{Reference=$orasReference; DestinationPath=$zipfile} -Retries 5 -RetryDelaySeconds 10
+        } catch {
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL -ErrorMessage "Exhausted retries for oras pull $orasReference. Error: $_"
         }
     }
-    AKS-Expand-Archive -Path $zipfile -DestinationPath C:\ 
+
+    AKS-Expand-Archive -Path $zipfile -DestinationPath C:\
     Remove-Item $zipfile
 }
 

staging/cse/windows/kubeletfunc.tests.ps1

@@ -1,12 +1,13 @@
 BeforeAll {
     . $PSScriptRoot\..\..\..\parts\windows\windowscsehelper.ps1
+    . $PSScriptRoot\networkisolatedclusterfunc.ps1
     . $PSCommandPath.Replace('.tests.ps1','.ps1')
 }
 
 Describe 'Get-KubePackage' {
     BeforeEach {
         Mock Expand-Archive
-        Mock Remove-Item
+        Mock Remove-Item -ParameterFilter { $Path -eq 'c:\k.zip' }
         Mock Logs-To-Event
         Mock DownloadFileOverHttp -MockWith {
             Param(
@@ -18,6 +19,8 @@ Describe 'Get-KubePackage' {
         } -Verifiable
 
         $global:CacheDir = 'c:\akse-cache'
+        # Ensure no global state leaked from other test files
+        $global:BootstrapProfileContainerRegistryServer = $null
     }
 
     Context 'mapping file exists' {
@@ -62,6 +65,107 @@ Describe 'Get-KubePackage' {
             Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Exactly -Times 1 -ParameterFilter { $Url -eq 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip' }
         }
     }
+
+    Context 'BootstrapProfileContainerRegistryServer is set - downloads with ORAS' {
+        BeforeEach {
+            Mock DownloadFileWithOras -MockWith {}
+            Mock Test-Path -MockWith { $false }
+            Mock AKS-Expand-Archive -MockWith {}
+
+            $global:BootstrapProfileContainerRegistryServer = "myregistry.azurecr.io"
+            $global:KubeBinariesVersion = "1.29.2"
+        }
+
+        AfterEach {
+            $global:BootstrapProfileContainerRegistryServer = $null
+        }
+
+        It "Should not call DownloadFileOverHttp when BootstrapProfileContainerRegistryServer is set" {
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Exactly -Times 0
+        }
+
+        It "Should call DownloadFileWithOras with correct reference when BootstrapProfileContainerRegistryServer is set" {
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'DownloadFileWithOras' -Exactly -Times 1 -ParameterFilter {
+                $Reference -eq 'myregistry.azurecr.io/aks/packages/kubernetes/windowszip:v1.29.2' -and
+                $DestinationPath -eq 'c:\k.zip'
+            }
+        }
+
+        It "Should call Logs-To-Event with ORAS task name when BootstrapProfileContainerRegistryServer is set" {
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'Logs-To-Event' -Exactly -Times 1 -ParameterFilter { $TaskName -eq 'AKS.WindowsCSE.DownloadKubeletBinariesWithOras' }
+        }
+
+        It "Should still expand archive and clean up zip file after ORAS download" {
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'AKS-Expand-Archive' -Exactly -Times 1 -ParameterFilter { $Path -eq 'c:\k.zip' -and $DestinationPath -eq 'C:\' }
+            Assert-MockCalled -CommandName 'Remove-Item' -Exactly -Times 1
+        }
+
+        It "Should call Set-ExitCode after exhausting retries when DownloadFileWithOras keeps failing" {
+            Mock DownloadFileWithOras -MockWith { throw "oras pull failed" }
+            Mock Start-Sleep -MockWith {}
+            Mock Set-ExitCode -MockWith {
+                Param($ExitCode, $ErrorMessage)
+                throw "Set-ExitCode: $ExitCode - $ErrorMessage"
+            }
+
+            { Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip' } | Should -Throw "*Exhausted retries for oras pull*"
+            Assert-MockCalled -CommandName 'DownloadFileWithOras' -Exactly -Times 5
+        }
+    }
+
+    Context 'BootstrapProfileContainerRegistryServer is set but DownloadFileWithOras is not available' {
+        BeforeEach {
+            Mock Test-Path -MockWith { $false }
+            Mock AKS-Expand-Archive -MockWith {}
+            Mock Set-ExitCode -MockWith {
+                Param($ExitCode, $ErrorMessage)
+                throw "Set-ExitCode: $ExitCode - $ErrorMessage"
+            }
+            Mock Get-Command -MockWith { return $null } -ParameterFilter { $Name -eq 'DownloadFileWithOras' }
+
+            $global:BootstrapProfileContainerRegistryServer = "myregistry.azurecr.io"
+            $global:KubeBinariesVersion = "1.29.2"
+        }
+
+        AfterEach {
+            $global:BootstrapProfileContainerRegistryServer = $null
+        }
+
+        It "Should call Set-ExitCode when DownloadFileWithOras function is not available" {
+            { Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip' } | Should -Throw "*DownloadFileWithOras function is not available*"
+        }
+    }
+
+    Context 'BootstrapProfileContainerRegistryServer is not set - falls back to HTTP download' {
+        BeforeEach {
+            $global:BootstrapProfileContainerRegistryServer = $null
+            Mock Test-Path -MockWith { $false }
+            Mock AKS-Expand-Archive -MockWith {}
+        }
+
+        It "Should call DownloadFileOverHttp when BootstrapProfileContainerRegistryServer is not set" {
+            $global:KubeBinariesVersion = '1.29.2'
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Exactly -Times 1 -ParameterFilter { $Url -eq 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip' }
+        }
+
+        It "Should call Logs-To-Event with HTTP download task name" {
+            $global:KubeBinariesVersion = '1.29.2'
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'Logs-To-Event' -Exactly -Times 1 -ParameterFilter { $TaskName -eq 'AKS.WindowsCSE.DownloadKubeletBinaries' }
+        }
+
+        It "Should call DownloadFileOverHttp when BootstrapProfileContainerRegistryServer is empty string" {
+            $global:BootstrapProfileContainerRegistryServer = ""
+            $global:KubeBinariesVersion = '1.29.2'
+            Get-KubePackage -KubeBinariesSASURL 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip'
+            Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Exactly -Times 1 -ParameterFilter { $Url -eq 'https://xxx.blob.core.windows.net/kubernetes/v1.29.2/windowszip/v1.29.2-1int.zip' }
+        }
+    }
 }
 
 Describe 'Configure-KubeletServingCertificateRotation' {
@@ -137,7 +241,7 @@ Describe 'Configure-KubeletServingCertificateRotation' {
     It "Should no-op if kubelet args and node labels are already correct when the opt-out tag is not set" {
         Mock Get-TagValue -MockWith { "false" }
         $kubeletConfigArgs = "--rotate-certificates=true,--rotate-server-certificates=true,--node-ip=10.0.0.1,anonymous-auth=false"
-        $kubeletNodeLabels = "kubernetes.azure.com/agentpool=wp0,kubernetes.azure.com/kubelet-serving-ca=cluster" 
+        $kubeletNodeLabels = "kubernetes.azure.com/agentpool=wp0,kubernetes.azure.com/kubelet-serving-ca=cluster"
         $global:KubeletNodeLabels = $kubeletNodeLabels
         $global:KubeletConfigArgs = $kubeletConfigArgs
         $global:EnableKubeletServingCertificateRotation = $true
@@ -217,8 +321,8 @@ Describe 'Get-TagValue' {
     Context 'Unable to call IMDS' {
         BeforeEach {
             Mock Set-ExitCode
-            Mock Invoke-RestMethod -MockWith { 
-                Throw 'IMDS is down' 
+            Mock Invoke-RestMethod -MockWith {
+                Throw 'IMDS is down'
             }
         }
 
@@ -295,4 +399,4 @@ Describe 'Remove-KubeletNodeLabel' {
         Remove-KubeletNodeLabel -Label $label
         Compare-Object $global:KubeletNodeLabels $expected | Should -Be $null
     }
-}
+}

staging/cse/windows/networkisolatedclusterfunc.ps1

@@ -365,3 +365,73 @@ function Get-BootstrapRegistryDomainName {
     }
     return $registryDomainName
 }
+
+function DownloadFileWithOras {
+    Param(
+        [Parameter(Mandatory = $true)][string]
+        $Reference,
+        [Parameter(Mandatory = $true)][string]
+        $DestinationPath,
+        [Parameter(Mandatory = $false)][string]
+        $Platform = "windows/amd64"
+    )
+
+    Write-Log "Downloading $Reference to $DestinationPath via oras pull (platform=$Platform)"
+
+    # oras pull --output specifies the output directory, not the filename.
+    # Download to a temp directory first, then move the file to DestinationPath.
+    $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid().ToString())
+    New-Item -ItemType Directory -Path $tempDir -Force | Out-Null
+
+    $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew()
+    $orasArgs = @(
+        "pull",
+        $Reference,
+        "--platform=$Platform",
+        "--registry-config=$($global:OrasRegistryConfigFile)",
+        "--output", $tempDir
+    )
+    & $global:OrasPath @orasArgs
+    if ($LASTEXITCODE -ne 0) {
+        $downloadTimer.Stop()
+        Remove-Item -Path $tempDir -Recurse -Force -ErrorAction SilentlyContinue
+        throw "oras pull failed with exit code $LASTEXITCODE for $Reference"
+    }
+    $downloadTimer.Stop()
+    $elapsedMs = $downloadTimer.ElapsedMilliseconds
+
+    # Find the downloaded file in the temp directory and move it to the desired path
+    $downloadedFile = Get-ChildItem -Path $tempDir -File | Select-Object -First 1
+    if (-not $downloadedFile) {
+        Remove-Item -Path $tempDir -Recurse -Force -ErrorAction SilentlyContinue
+        throw "oras pull succeeded but no file found in temp directory for $Reference"
+    }
+
+    Write-Log "Downloaded file name: $($downloadedFile.Name) (size: $($downloadedFile.Length) bytes) in temp directory $tempDir"
+
+    # Ensure the destination parent directory exists
+    $destDir = [System.IO.Path]::GetDirectoryName($DestinationPath)
+    if ($destDir -and -not (Test-Path $destDir)) {
+        New-Item -ItemType Directory -Path $destDir -Force | Out-Null
+    }
+
+    # Remove existing destination if present, then move the downloaded file
+    if (Test-Path $DestinationPath) {
+        Remove-Item -Path $DestinationPath -Force
+    }
+    Write-Log "Moving $($downloadedFile.FullName) to $DestinationPath"
+    Move-Item -Path $downloadedFile.FullName -Destination $DestinationPath -Force
+    Remove-Item -Path $tempDir -Recurse -Force -ErrorAction SilentlyContinue
+
+    if ($global:AppInsightsClient -ne $null) {
+        $event = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry"
+        $event.Name = "FileDownload"
+        $event.Properties["FileName"] = $Reference
+        $event.Properties["Method"] = "oras"
+        $event.Metrics["DurationMs"] = $elapsedMs
+        $global:AppInsightsClient.TrackEvent($event)
+    }
+
+    Write-Log "Downloaded $Reference to $DestinationPath via oras in $elapsedMs ms"
+    Get-Item $DestinationPath -ErrorAction Continue | Format-List | Out-String | Write-Log
+}