feat: oras pull pause image if not cached for windows network isolated cluster

e2e/scenario_win_test.go

@@ -507,7 +507,7 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
 		Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images",
 		Tags: Tags{
 			NetworkIsolated: true,
-			NonAnonymousACR: true,
+			NonAnonymousACR: false,
 		},
 		Config: Config{
 			Cluster: ClusterAzureBootstrapProfileCache,
@@ -516,7 +516,7 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
 				nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
 					PrivateEgress: &datamodel.PrivateEgress{
 						Enabled:                 true,
-						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
+						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
 					},
 				}
 			},

staging/cse/windows/containerdfunc.ps1

@@ -330,5 +330,13 @@ function Install-Containerd {
     -CNIConfDir $CNIConfDir
 
   RegisterContainerDService -KubeDir $KubeDir
+  if (-not [string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) {
+    if (Get-Command -Name Set-PodInfraContainerImage -ErrorAction SilentlyContinue) {
+      Set-PodInfraContainerImage
+    }
+    else {
+      Write-Log "Set-PodInfraContainerImage command not found; skipping pod infra container image configuration."
+    }
+  }
   Enable-Logging
 }

staging/cse/windows/networkisolatedclusterfunc.ps1

@@ -65,3 +65,81 @@ function Install-Oras {
 
     Write-Log "Oras installed successfully at $($global:OrasPath)"
 }
+
+# For network isolated cluster, containerd has issue to pull pause image via credential provider, thus it will not be able to pull pause image during runtime. We have to pull pause image and set pinned label to avoid gc.
+function Set-PodInfraContainerImage {
+    $podInfraContainerImageDownloadDir = "C:\k\pod-infra-container-image\downloads"
+    $podInfraContainerImageTar = "C:\k\pod-infra-container-image\pod-infra-container-image.tar"
+
+    $clusterConfig = ConvertFrom-Json ((Get-Content $global:KubeClusterConfigPath -ErrorAction Stop) | Out-String)
+    $podInfraContainerImage = $clusterConfig.Cri.Images.Pause
+    if ([string]::IsNullOrWhiteSpace($podInfraContainerImage)) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "Failed to recognize pod infra container image"
+    }
+
+    Write-Log "Checking if '$podInfraContainerImage' already exists locally..."
+    $images = & ctr.exe -n k8s.io images list -q 2>$null
+    if (($LASTEXITCODE -eq 0) -and ($images -contains $podInfraContainerImage)) {
+        Write-Log "Image '$podInfraContainerImage' already exists locally, skipping pull"
+        return
+    }
+
+    $baseName = $podInfraContainerImage -replace ':[^:]+$'
+    $tag = "local"
+
+    $image = $podInfraContainerImage
+    if (-not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) {
+        $image = $podInfraContainerImage.Replace("mcr.microsoft.com", $global:BootstrapProfileContainerRegistryServer)
+    }
+
+    if (-not (Test-Path -Path $podInfraContainerImageDownloadDir)) {
+        New-Item -ItemType Directory -Path $podInfraContainerImageDownloadDir -Force | Out-Null
+    }
+
+    Write-Log "Pulling via oras for '$image'"
+    $orasCopySucceeded = $false
+    $orasDestination = '{0}:{1}' -f $podInfraContainerImageDownloadDir, $tag
+    for ($i = 1; $i -le 10; $i++) {
+        if ($i -gt 1) {
+            Start-Sleep -Seconds 5
+        }
+
+        Write-Log "Try $i : oras cp '$image' to '$orasDestination'"
+        $orasOutput = & $global:OrasPath cp $image $orasDestination --to-oci-layout --from-registry-config $global:OrasRegistryConfigFile 2>&1
+        if ($LASTEXITCODE -eq 0) {
+            Write-Log ("Successfully pulled '$image' via oras on attempt $i")
+            $orasCopySucceeded = $true
+            break
+        }
+
+        Write-Log ('oras cp attempt {0} failed (exit code {1}): {2}' -f $i, $LASTEXITCODE, $orasOutput)
+    }
+
+    if (-not $orasCopySucceeded) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "Failed to pull '$image'"
+    }
+    tar -cf $podInfraContainerImageTar -C $podInfraContainerImageDownloadDir .
+    if ($LASTEXITCODE -ne 0) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "failed to create tar for pod infra image from '$podInfraContainerImageDownloadDir'"
+    }
+
+    $importOutput = $(ctr.exe -n k8s.io image import --base-name $baseName $podInfraContainerImageTar 2>&1)
+    if ($LASTEXITCODE -ne 0) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage ('failed to import ''{0}'': {1}' -f $podInfraContainerImage, $importOutput)
+    }
+
+    $finalImage = '{0}:{1}' -f $baseName, $tag
+    $tagOutput = $(ctr.exe -n k8s.io image tag ${finalImage} $podInfraContainerImage 2>&1)
+    if ($LASTEXITCODE -ne 0) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage "failed to tag pod infra image: {0}" -f $tagOutput
+    }
+
+    $labelOutput = $(ctr.exe -n k8s.io images label $podInfraContainerImage io.cri-containerd.pinned=pinned 2>&1)
+    if ($LASTEXITCODE -ne 0) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER -ErrorMessage ('failed to label pod infra image ''{0}'' as pinned: {1}' -f $podInfraContainerImage, $labelOutput)
+    }
+    Write-Log "Successfully imported '$podInfraContainerImage'"
+
+    Remove-Item -Path $podInfraContainerImageDownloadDir -Recurse -Force -ErrorAction SilentlyContinue
+    Remove-Item -Path $podInfraContainerImageTar -Force -ErrorAction SilentlyContinue
+}

staging/cse/windows/networkisolatedclusterfunc.tests.ps1

@@ -105,3 +105,128 @@ Describe "Install-Oras" {
     } | Should -Throw "*Set-ExitCode:$($global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND):Failed to extract oras archive*"
   }
 }
+
+Describe "Set-PodInfraContainerImage" {
+  BeforeEach {
+    $global:KubeClusterConfigPath = "C:\k\kubeclusterconfig.json"
+    $global:BootstrapProfileContainerRegistryServer = "myacr.azurecr.io/aks-managed-repository"
+    $global:OrasRegistryConfigFile = "C:\aks-tools\oras\config.json"
+    $global:OrasPath = "Mock-OrasCli"
+    $global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER = 82
+
+    Mock Start-Sleep
+    Mock New-Item
+    Mock Remove-Item
+    Mock tar -MockWith { $global:LASTEXITCODE = 0 }
+    $script:CtrExeInvocations = @()
+    $script:CtrExeMock = {
+      param($Args)
+      return "ok"
+    }
+    function global:ctr.exe {
+      param([Parameter(ValueFromRemainingArguments = $true)]$Args)
+      $script:CtrExeInvocations += , @($Args)
+      $global:LASTEXITCODE = 0
+      return & $script:CtrExeMock $Args
+    }
+    Mock Test-Path -MockWith { $false }
+    Mock Set-ExitCode -MockWith {
+      param(
+        [Parameter(Mandatory = $true)][int]$ExitCode,
+        [Parameter(Mandatory = $true)][string]$ErrorMessage
+      )
+      throw "Set-ExitCode:${ExitCode}:${ErrorMessage}"
+    }
+
+    Mock Get-Content -MockWith {
+@'
+{
+  "Cri": {
+    "Images": {
+      "Pause": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1"
+    }
+  }
+}
+'@
+    }
+  }
+
+  AfterEach {
+    Remove-Item Function:\global:ctr.exe -ErrorAction SilentlyContinue
+  }
+
+  It "fails when pod infra image is empty" {
+    Mock Get-Content -MockWith {
+@'
+{
+  "Cri": {
+    "Images": {
+      "Pause": ""
+    }
+  }
+}
+'@
+    }
+
+    {
+      Set-PodInfraContainerImage
+    } | Should -Throw "*Set-ExitCode:82:Failed to recognize pod infra container image*"
+  }
+
+  It "returns early when image already exists locally" {
+    $script:CtrExeMock = {
+      param($Args)
+      return @("mcr.microsoft.com/oss/v2/kubernetes/pause:3.10.1")
+    }
+
+    function global:Mock-OrasCli {
+      param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
+      $global:LASTEXITCODE = 0
+      return "ok"
+    }
+
+    { Set-PodInfraContainerImage } | Should -Not -Throw
+    Assert-MockCalled -CommandName 'tar' -Times 0
+    $script:CtrExeInvocations.Count | Should -Be 1
+  }
+
+  It "pulls via oras and imports image when not found locally" {
+    $script:CtrExeMock = {
+      param($Args)
+      if ($Args -contains 'list') {
+        return @()
+      }
+      return "ok"
+    }
+
+    function global:Mock-OrasCli {
+      param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
+      $global:LASTEXITCODE = 0
+      return "oras ok"
+    }
+
+    { Set-PodInfraContainerImage } | Should -Not -Throw
+    Assert-MockCalled -CommandName 'tar' -Times 1
+    $script:CtrExeInvocations.Count | Should -Be 4
+    Assert-MockCalled -CommandName 'Remove-Item' -Times 2
+  }
+
+  It "fails after oras retry exhaustion" {
+    $script:CtrExeMock = {
+      param($Args)
+      return @()
+    }
+
+    function global:Mock-OrasCli {
+      param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Args)
+      $global:LASTEXITCODE = 1
+      return "oras failed"
+    }
+
+    {
+      Set-PodInfraContainerImage
+    } | Should -Throw "*Set-ExitCode:82:Failed to pull*"
+    Assert-MockCalled -CommandName 'Start-Sleep' -Times 9
+    $script:CtrExeInvocations.Count | Should -Be 1
+  }
+}