From dda7a50ba7339c61e47518e074eb02481c84e48a Mon Sep 17 00:00:00 2001 From: Massimo Federico Bonfigli Date: Tue, 28 Jul 2026 13:40:04 +0200 Subject: [PATCH] aws: wire CCM to CCO credentials via CredentialsRequest AWS Cloud Controller Manager currently reads credentials from the EC2 instance metadata service (IMDS) via the master node IAM instance role, which is static from install time. This prevents new IAM permissions required by new CCM features from being granted on upgraded clusters (OCPBUGS-98763). This commit migrates CCM to the Cloud Credential Operator (CCO) model: - Add a CredentialsRequest manifest for CCM with the scoped set of EC2 and ELB permissions it actually needs, including elasticloadbalancing:SetSecurityGroups for BYO Security Group NLB support and elasticloadbalancing:SetIpAddressType for dual-stack NLBs. In Mint mode CCO mints a dedicated IAM user; in Manual+STS mode CCO writes role_arn + web_identity_token_file for IRSA. - Mount the resulting credentials secret and a projected ServiceAccount token (audience: sts.amazonaws.com) into the CCM Deployment, and point the AWS SDK at the credentials file via AWS_SHARED_CREDENTIALS_FILE. - Add unit test assertions verifying the new volumes, mounts, and env var are present in the rendered Deployment. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- ...er-operator_19_credentialsrequest-aws.yaml | 68 +++++++++++++++++++ pkg/cloud/aws/assets/deployment.yaml | 17 +++++ pkg/cloud/aws/aws_test.go | 31 +++++++++ 3 files changed, 116 insertions(+) create mode 100644 manifests/0000_26_cloud-controller-manager-operator_19_credentialsrequest-aws.yaml diff --git a/manifests/0000_26_cloud-controller-manager-operator_19_credentialsrequest-aws.yaml b/manifests/0000_26_cloud-controller-manager-operator_19_credentialsrequest-aws.yaml new file mode 100644 index 000000000..9e91977ed --- /dev/null +++ b/manifests/0000_26_cloud-controller-manager-operator_19_credentialsrequest-aws.yaml @@ -0,0 +1,68 @@ +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: openshift-aws-cloud-controller-manager + namespace: openshift-cloud-credential-operator + annotations: + capability.openshift.io/name: CloudCredential+CloudControllerManager + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true" +spec: + serviceAccountNames: + - cloud-controller-manager + secretRef: + name: cloud-controller-manager-credentials + namespace: openshift-cloud-controller-manager + cloudTokenPath: /var/run/secrets/openshift/serviceaccount/token + providerSpec: + apiVersion: cloudcredential.openshift.io/v1 + kind: AWSProviderSpec + stsIAMRoleARN: "" + statementEntries: + - effect: Allow + action: + # Node lifecycle + - ec2:Describe* + - ec2:CreateSecurityGroup + - ec2:CreateTags + - ec2:DeleteSecurityGroup + - ec2:AuthorizeSecurityGroupIngress + - ec2:RevokeSecurityGroupIngress + - ec2:ModifyInstanceAttribute + # Volume management + - ec2:AttachVolume + - ec2:CreateVolume + - ec2:DeleteVolume + - ec2:DetachVolume + - ec2:ModifyVolume + - kms:DescribeKey + # Load balancer management (CLB) + - elasticloadbalancing:AddTags + - elasticloadbalancing:AttachLoadBalancerToSubnets + - elasticloadbalancing:ApplySecurityGroupsToLoadBalancer + - elasticloadbalancing:CreateLoadBalancer + - elasticloadbalancing:CreateLoadBalancerPolicy + - elasticloadbalancing:CreateLoadBalancerListeners + - elasticloadbalancing:ConfigureHealthCheck + - elasticloadbalancing:DeleteLoadBalancer + - elasticloadbalancing:DeleteLoadBalancerListeners + - elasticloadbalancing:DeregisterInstancesFromLoadBalancer + - elasticloadbalancing:Describe* + - elasticloadbalancing:DetachLoadBalancerFromSubnets + - elasticloadbalancing:ModifyLoadBalancerAttributes + - elasticloadbalancing:RegisterInstancesWithLoadBalancer + - elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer + - elasticloadbalancing:SetLoadBalancerPoliciesOfListener + # Load balancer management (NLB/ALB) + - elasticloadbalancing:CreateListener + - elasticloadbalancing:CreateTargetGroup + - elasticloadbalancing:DeleteListener + - elasticloadbalancing:DeleteTargetGroup + - elasticloadbalancing:DeregisterTargets + - elasticloadbalancing:ModifyListener + - elasticloadbalancing:ModifyTargetGroup + - elasticloadbalancing:ModifyTargetGroupAttributes + - elasticloadbalancing:RegisterTargets + # BYO Security Group for NLB + - elasticloadbalancing:SetSecurityGroups + resource: "*" diff --git a/pkg/cloud/aws/assets/deployment.yaml b/pkg/cloud/aws/assets/deployment.yaml index c66a25171..a10d6fbfb 100644 --- a/pkg/cloud/aws/assets/deployment.yaml +++ b/pkg/cloud/aws/assets/deployment.yaml @@ -54,6 +54,8 @@ spec: env: - name: CLOUD_CONFIG value: /etc/kubernetes-cloud-config/cloud.conf + - name: AWS_SHARED_CREDENTIALS_FILE + value: /etc/aws-credentials/credentials image: {{ .images.CloudControllerManager }} imagePullPolicy: IfNotPresent name: cloud-controller-manager @@ -76,6 +78,12 @@ spec: - name: trusted-ca mountPath: /etc/pki/ca-trust/extracted/pem readOnly: true + - name: aws-credentials + mountPath: /etc/aws-credentials + readOnly: true + - name: bound-sa-token + mountPath: /var/run/secrets/openshift/serviceaccount + readOnly: true hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" @@ -123,3 +131,12 @@ spec: hostPath: path: /etc/kubernetes type: Directory + - name: aws-credentials + secret: + secretName: cloud-controller-manager-credentials + - name: bound-sa-token + projected: + sources: + - serviceAccountToken: + path: token + audience: sts.amazonaws.com diff --git a/pkg/cloud/aws/aws_test.go b/pkg/cloud/aws/aws_test.go index 36ac82ec1..566d2da79 100644 --- a/pkg/cloud/aws/aws_test.go +++ b/pkg/cloud/aws/aws_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" configv1 "github.com/openshift/api/config/v1" @@ -44,6 +47,34 @@ func TestResourcesRenderingSmoke(t *testing.T) { resources := assets.GetRenderedResources() assert.Len(t, resources, 3) + + deploy, ok := resources[0].(*appsv1.Deployment) + require.True(t, ok, "first resource should be a Deployment") + + container := deploy.Spec.Template.Spec.Containers[0] + + assert.Contains(t, container.Env, corev1.EnvVar{ + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/etc/aws-credentials/credentials", + }) + + assert.Contains(t, container.VolumeMounts, corev1.VolumeMount{ + Name: "aws-credentials", + MountPath: "/etc/aws-credentials", + ReadOnly: true, + }) + assert.Contains(t, container.VolumeMounts, corev1.VolumeMount{ + Name: "bound-sa-token", + MountPath: "/var/run/secrets/openshift/serviceaccount", + ReadOnly: true, + }) + + volumeNames := make([]string, 0, len(deploy.Spec.Template.Spec.Volumes)) + for _, v := range deploy.Spec.Template.Spec.Volumes { + volumeNames = append(volumeNames, v.Name) + } + assert.Contains(t, volumeNames, "aws-credentials") + assert.Contains(t, volumeNames, "bound-sa-token") }) } }