Merge branch 'kubernetes:master' into master

cluster-autoscaler/cloudprovider/azure/examples/dev/README.md

@@ -0,0 +1,23 @@
+## Overview:
+
+This document, and directory are focused on the ability to deploy and test a working version of autoscaler from a development branch onto an AKS cluster for testing out a set of changes.
+
+## Steps:
+
+1. Create a codespace using one of the devcontainer setups from the `devcontainers` branch of https://github.com/azure/autoscaler
+
+2. In the codespace switch to whatever branch you want to test
+    - Note: for testing an upstream branch use: `git checkout upstream/<branch-name>`
+        - This might require a `git fetch upstream`
+
+5. run `cd cluster-autoscaler/cloudprovider/azure/examples/dev`
+
+6. run `az login`
+
+7. run `./aks-dev-deploy.sh`
+
+8. run `cd ../../../../`
+
+9. run `skaffold run --filename cloudprovider/azure/examples/dev/skaffold.yaml`
+
+10. inspect the cluster with `kubectl`, and scale the `inflate` deployment for testing as desired.

cluster-autoscaler/cloudprovider/azure/examples/dev/aks-dev-deploy.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# This script deploys an AKS cluster with a user node pool and ACR,
+# and workload identity to be used for cluster-autoscaler. It creates
+# the necessary role assignments to allow the cluster-autoscaler to manage VMSS,
+# configures skaffold to use the ACR for the cluster-autoscaler deployment,
+# and updates the cluster-autoscaler deployment with the necessary values,
+# ready to be used with skaffold dev/run/debug.
+
+# assumed logged in (az login) and subscription set (az account set --subscription ...)
+
+# set resource group and ACR name (preferably unique)
+RG=${CODESPACE_NAME:-cluster-autoscaler-test}
+ACR_NAME=$(echo "$RG" | tr -d -) # remove hyphens
+az group create --name "${RG}" --location westus3 --output none
+
+# deploy AKS & ACR
+DEPLOYMENT_JSON=$(az deployment group create --name aks-dev --resource-group "${RG}" \
+  --template-file ./aks-dev.bicep \
+  --parameters acrName="${ACR_NAME}")
+
+# get relevant information
+RESOURCE_GROUP_MC=$(jq -r ".properties.outputs.nodeResourceGroup.value"                <<< "$DEPLOYMENT_JSON")
+USER_POOL_NAME=$(   jq -r ".properties.outputs.userNodePoolName.value"                 <<< "$DEPLOYMENT_JSON")
+AKS_NAME=$(         jq -r ".properties.outputs.aksName.value"                          <<< "$DEPLOYMENT_JSON")
+CAS_UAI_PRINCIPAL=$(jq -r ".properties.outputs.casUserAssignedIdentityPrincipal.value" <<< "$DEPLOYMENT_JSON")
+CAS_UAI_CLIENTID=$( jq -r ".properties.outputs.casUserAssignedIdentityClientId.value"  <<< "$DEPLOYMENT_JSON")
+
+# confgure dev environment
+az aks get-credentials --name "${AKS_NAME}" --resource-group "${RG}"
+az acr login --name "${ACR_NAME}"
+skaffold config set default-repo "${ACR_NAME}.azurecr.io/cluster-autoscaler"
+
+# create role assignments to let CAS manage VMSS
+az role assignment create \
+  --assignee "${CAS_UAI_PRINCIPAL}" \
+  --scope "$(az group show --name "${RESOURCE_GROUP_MC}" --query "id" --output tsv)" \
+  --role "Virtual Machine Contributor" \
+  --output none
+
+# prep values for and update CAS deployment
+VMSS_NAME=$(az resource list \
+  --tag aks-managed-poolName="${USER_POOL_NAME}" \
+  --query "[?resourceGroup=='${RESOURCE_GROUP_MC}'].name" \
+  --output tsv)
+TENANT_ID_B64=$(az account show --query tenantId --output tsv | base64)
+RESOURCE_GROUP_MC_B64=$(base64 <<< "$RESOURCE_GROUP_MC")
+SUBSCRIPTION_ID_B64=$(az account show --query id --output tsv | base64)
+
+export TENANT_ID_B64 RESOURCE_GROUP_MC_B64 VMSS_NAME CAS_UAI_CLIENTID SUBSCRIPTION_ID_B64
+
+yq '(.. | select(tag == "!!str")) |= envsubst(nu)' \
+    cluster-autoscaler-vmss-wi-dynamic.yaml.tpl > \
+    cluster-autoscaler-vmss-wi-dynamic.yaml
+
+# skaffold dev/run/debug
+
+exit
+
+# To recover access after restarting codespace with existing AKS and ACR:
+# az login & az account set -n ...
+# az aks get-credentials -n cas-test -g $CODESPACE_NAME 
+# ACR_NAME=$(echo "$CODESPACE_NAME" | tr -d -)
+# az acr login -n $ACR_NAME
+# skaffold config set default-repo "${ACR_NAME}.azurecr.io/cluster-autoscaler"
+# skaffold dev/run/debug

cluster-autoscaler/cloudprovider/azure/examples/dev/aks-dev.bicep

@@ -0,0 +1,82 @@
+param aksName string = 'cas-test'
+param acrName string = 'castestacr'
+param location string = resourceGroup().location
+param dnsPrefix string = aksName
+param vmSize string = 'Standard_DS2_v2'
+param casUserAssignedIdentityName string = 'cas-msi'
+param casFederatedCredenatialName string = 'cas-federated-credential'
+param casNamespace string = 'kube-system'
+
+resource aks 'Microsoft.ContainerService/managedClusters@2023-11-01' = {
+  location: location
+  name: aksName
+  identity: { type: 'SystemAssigned' }  // --enable-managed-identity
+
+  properties: {
+    dnsPrefix: dnsPrefix
+    oidcIssuerProfile: { enabled: true }  // --enable-oidc-issuer
+    securityProfile: {
+       workloadIdentity: { enabled: true } // --enable-workload-identity 
+    } 
+    agentPoolProfiles: [
+      {
+        count: 1
+        mode: 'System'
+        name: 'nodepool1'
+        type: 'VirtualMachineScaleSets'
+        vmSize: vmSize
+      }
+      {
+        count: 3
+        mode: 'User'
+        name: 'nodepool2'
+        type: 'VirtualMachineScaleSets'
+        vmSize: vmSize
+      }
+    ]
+    networkProfile: {
+      networkPlugin: 'azure'
+      networkPluginMode: 'overlay'
+    } 
+  }
+}
+
+resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' = {
+  location: location
+  name: acrName
+  sku: { name: 'Basic' }
+}
+
+var AcrPull = subscriptionResourceId('Microsoft.Authorization/roleDefinition', '7f951dda-4ed3-4680-a7ca-43fe172d538d')
+
+@description('AKS can pull images from ACR')
+resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(resourceGroup().id, acr.name, aks.name, AcrPull)
+  scope: acr
+  properties: {
+    principalId: aks.properties.identityProfile.kubeletidentity.objectId
+    principalType: 'ServicePrincipal'
+    roleDefinitionId: AcrPull
+  }
+}
+
+@description('CAS user assigned identity')
+resource casUserAssignedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  location: location
+  name: casUserAssignedIdentityName
+  resource caseFederatedCredential 'federatedIdentityCredentials' = {
+    name: casFederatedCredenatialName
+    properties: {
+      issuer: aks.properties.oidcIssuerProfile.issuerURL
+      subject: 'system:serviceaccount:${casNamespace}:cluster-autoscaler' // TODO: parameterize namespace
+      audiences: ['api://AzureADTokenExchange']
+    }
+  }
+}
+
+output acrName string = acr.name
+output aksName string = aks.name
+output nodeResourceGroup string = aks.properties.nodeResourceGroup
+output userNodePoolName string = aks.properties.agentPoolProfiles[1].name
+output casUserAssignedIdentityPrincipal string = casUserAssignedIdentity.properties.principalId
+output casUserAssignedIdentityClientId string = casUserAssignedIdentity.properties.clientId