Skip to content

Commit e2d3459

Browse files
mvandenburghmvandenburgh
committed
Migrate to AWS Load Balancer Controller
Resource: https://aws.amazon.com/blogs/containers/exposing-kubernetes-applications-part-3-nginx-ingress-controller/
1 parent f1681ae commit e2d3459

File tree

3 files changed

+332
-1
lines changed

3 files changed

+332
-1
lines changed

‎k8s/production/aws-load-balancer-controller/release.yaml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
apiVersion: source.toolkit.fluxcd.io/v1beta2
3+
kind: HelmRepository
4+
metadata:
5+
name: aws-load-balancer-controller
6+
namespace: kube-system
7+
spec:
8+
interval: 10m
9+
url: https://aws.github.io/eks-charts
10+
11+
---
12+
apiVersion: helm.toolkit.fluxcd.io/v2beta2
13+
kind: HelmRelease
14+
metadata:
15+
name: aws-load-balancer-controller
16+
namespace: kube-system
17+
spec:
18+
interval: 10m
19+
chart:
20+
spec:
21+
chart: aws-load-balancer-controller
22+
version: 1.8.1 # [email protected]
23+
sourceRef:
24+
kind: HelmRepository
25+
name: aws-load-balancer-controller
26+
install:
27+
crds: CreateReplace
28+
upgrade:
29+
crds: CreateReplace
30+
valuesFrom:
31+
# See terraform/modules/spack/eks.tf
32+
- kind: ConfigMap
33+
name: aws-lb-controller-config
34+
valuesKey: values.yaml
35+
values:
36+
tolerations:
37+
# The LB Controller should be scheduled on the initial managed nodegroup pods
38+
# so that it is not dependent on Karpenter to be scheduled.
39+
- key: "CriticalAddonsOnly"
40+
operator: "Exists"

‎k8s/production/sealed-secrets/release.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,5 @@ spec:
2828
spack.io/node-pool: base
2929
tolerations:
3030
- key: CriticalAddonsOnly
31-
operator: Equal
31+
operator: Exists
3232
effect: NoSchedule

‎terraform/modules/spack_aws_k8s/eks.tf

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,3 +296,294 @@ resource "kubectl_manifest" "cluster_name_config_map" {
296296
cluster-name: ${module.eks.cluster_name}
297297
YAML
298298
}
299+
300+
301+
resource "aws_iam_role" "load_balancer_controller" {
302+
name = "AmazonEKSLoadBalancerControllerRole-${var.deployment_name}"
303+
assume_role_policy = jsonencode({
304+
"Version" : "2012-10-17",
305+
"Statement" : [
306+
{
307+
"Effect" : "Allow",
308+
"Principal" : {
309+
"Federated" : module.eks.oidc_provider_arn,
310+
},
311+
"Action" : "sts:AssumeRoleWithWebIdentity",
312+
"Condition" : {
313+
"StringEquals" : {
314+
"${module.eks.oidc_provider}:aud" : "sts.amazonaws.com",
315+
"${module.eks.oidc_provider}:sub" : "system:serviceaccount:kube-system:aws-load-balancer-controller"
316+
}
317+
}
318+
}
319+
]
320+
})
321+
}
322+
323+
resource "aws_iam_role_policy" "load_balancer_controller" {
324+
name = "AmazonEKSLoadBalancerControllerPolicy-${var.deployment_name}"
325+
role = aws_iam_role.load_balancer_controller.id
326+
# Copied from https://docs.aws.amazon.com/eks/latest/userguide/lbc-manifest.html#lbc-iam
327+
policy = jsonencode({
328+
"Version" : "2012-10-17",
329+
"Statement" : [
330+
{
331+
"Effect" : "Allow",
332+
"Action" : [
333+
"iam:CreateServiceLinkedRole"
334+
],
335+
"Resource" : "*",
336+
"Condition" : {
337+
"StringEquals" : {
338+
"iam:AWSServiceName" : "elasticloadbalancing.amazonaws.com"
339+
}
340+
}
341+
},
342+
{
343+
"Effect" : "Allow",
344+
"Action" : [
345+
"ec2:DescribeAccountAttributes",
346+
"ec2:DescribeAddresses",
347+
"ec2:DescribeAvailabilityZones",
348+
"ec2:DescribeInternetGateways",
349+
"ec2:DescribeVpcs",
350+
"ec2:DescribeVpcPeeringConnections",
351+
"ec2:DescribeSubnets",
352+
"ec2:DescribeSecurityGroups",
353+
"ec2:DescribeInstances",
354+
"ec2:DescribeNetworkInterfaces",
355+
"ec2:DescribeTags",
356+
"ec2:GetCoipPoolUsage",
357+
"ec2:DescribeCoipPools",
358+
"elasticloadbalancing:DescribeLoadBalancers",
359+
"elasticloadbalancing:DescribeLoadBalancerAttributes",
360+
"elasticloadbalancing:DescribeListeners",
361+
"elasticloadbalancing:DescribeListenerCertificates",
362+
"elasticloadbalancing:DescribeSSLPolicies",
363+
"elasticloadbalancing:DescribeRules",
364+
"elasticloadbalancing:DescribeTargetGroups",
365+
"elasticloadbalancing:DescribeTargetGroupAttributes",
366+
"elasticloadbalancing:DescribeTargetHealth",
367+
"elasticloadbalancing:DescribeTags",
368+
"elasticloadbalancing:DescribeTrustStores"
369+
],
370+
"Resource" : "*"
371+
},
372+
{
373+
"Effect" : "Allow",
374+
"Action" : [
375+
"cognito-idp:DescribeUserPoolClient",
376+
"acm:ListCertificates",
377+
"acm:DescribeCertificate",
378+
"iam:ListServerCertificates",
379+
"iam:GetServerCertificate",
380+
"waf-regional:GetWebACL",
381+
"waf-regional:GetWebACLForResource",
382+
"waf-regional:AssociateWebACL",
383+
"waf-regional:DisassociateWebACL",
384+
"wafv2:GetWebACL",
385+
"wafv2:GetWebACLForResource",
386+
"wafv2:AssociateWebACL",
387+
"wafv2:DisassociateWebACL",
388+
"shield:GetSubscriptionState",
389+
"shield:DescribeProtection",
390+
"shield:CreateProtection",
391+
"shield:DeleteProtection"
392+
],
393+
"Resource" : "*"
394+
},
395+
{
396+
"Effect" : "Allow",
397+
"Action" : [
398+
"ec2:AuthorizeSecurityGroupIngress",
399+
"ec2:RevokeSecurityGroupIngress"
400+
],
401+
"Resource" : "*"
402+
},
403+
{
404+
"Effect" : "Allow",
405+
"Action" : [
406+
"ec2:CreateSecurityGroup"
407+
],
408+
"Resource" : "*"
409+
},
410+
{
411+
"Effect" : "Allow",
412+
"Action" : [
413+
"ec2:CreateTags"
414+
],
415+
"Resource" : "arn:aws:ec2:*:*:security-group/*",
416+
"Condition" : {
417+
"StringEquals" : {
418+
"ec2:CreateAction" : "CreateSecurityGroup"
419+
},
420+
"Null" : {
421+
"aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
422+
}
423+
}
424+
},
425+
{
426+
"Effect" : "Allow",
427+
"Action" : [
428+
"ec2:CreateTags",
429+
"ec2:DeleteTags"
430+
],
431+
"Resource" : "arn:aws:ec2:*:*:security-group/*",
432+
"Condition" : {
433+
"Null" : {
434+
"aws:RequestTag/elbv2.k8s.aws/cluster" : "true",
435+
"aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
436+
}
437+
}
438+
},
439+
{
440+
"Effect" : "Allow",
441+
"Action" : [
442+
"ec2:AuthorizeSecurityGroupIngress",
443+
"ec2:RevokeSecurityGroupIngress",
444+
"ec2:DeleteSecurityGroup"
445+
],
446+
"Resource" : "*",
447+
"Condition" : {
448+
"Null" : {
449+
"aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
450+
}
451+
}
452+
},
453+
{
454+
"Effect" : "Allow",
455+
"Action" : [
456+
"elasticloadbalancing:CreateLoadBalancer",
457+
"elasticloadbalancing:CreateTargetGroup"
458+
],
459+
"Resource" : "*",
460+
"Condition" : {
461+
"Null" : {
462+
"aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
463+
}
464+
}
465+
},
466+
{
467+
"Effect" : "Allow",
468+
"Action" : [
469+
"elasticloadbalancing:CreateListener",
470+
"elasticloadbalancing:DeleteListener",
471+
"elasticloadbalancing:CreateRule",
472+
"elasticloadbalancing:DeleteRule"
473+
],
474+
"Resource" : "*"
475+
},
476+
{
477+
"Effect" : "Allow",
478+
"Action" : [
479+
"elasticloadbalancing:AddTags",
480+
"elasticloadbalancing:RemoveTags"
481+
],
482+
"Resource" : [
483+
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
484+
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
485+
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
486+
],
487+
"Condition" : {
488+
"Null" : {
489+
"aws:RequestTag/elbv2.k8s.aws/cluster" : "true",
490+
"aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
491+
}
492+
}
493+
},
494+
{
495+
"Effect" : "Allow",
496+
"Action" : [
497+
"elasticloadbalancing:AddTags",
498+
"elasticloadbalancing:RemoveTags"
499+
],
500+
"Resource" : [
501+
"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
502+
"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
503+
"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
504+
"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
505+
]
506+
},
507+
{
508+
"Effect" : "Allow",
509+
"Action" : [
510+
"elasticloadbalancing:ModifyLoadBalancerAttributes",
511+
"elasticloadbalancing:SetIpAddressType",
512+
"elasticloadbalancing:SetSecurityGroups",
513+
"elasticloadbalancing:SetSubnets",
514+
"elasticloadbalancing:DeleteLoadBalancer",
515+
"elasticloadbalancing:ModifyTargetGroup",
516+
"elasticloadbalancing:ModifyTargetGroupAttributes",
517+
"elasticloadbalancing:DeleteTargetGroup"
518+
],
519+
"Resource" : "*",
520+
"Condition" : {
521+
"Null" : {
522+
"aws:ResourceTag/elbv2.k8s.aws/cluster" : "false"
523+
}
524+
}
525+
},
526+
{
527+
"Effect" : "Allow",
528+
"Action" : [
529+
"elasticloadbalancing:AddTags"
530+
],
531+
"Resource" : [
532+
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
533+
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
534+
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
535+
],
536+
"Condition" : {
537+
"StringEquals" : {
538+
"elasticloadbalancing:CreateAction" : [
539+
"CreateTargetGroup",
540+
"CreateLoadBalancer"
541+
]
542+
},
543+
"Null" : {
544+
"aws:RequestTag/elbv2.k8s.aws/cluster" : "false"
545+
}
546+
}
547+
},
548+
{
549+
"Effect" : "Allow",
550+
"Action" : [
551+
"elasticloadbalancing:RegisterTargets",
552+
"elasticloadbalancing:DeregisterTargets"
553+
],
554+
"Resource" : "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
555+
},
556+
{
557+
"Effect" : "Allow",
558+
"Action" : [
559+
"elasticloadbalancing:SetWebAcl",
560+
"elasticloadbalancing:ModifyListener",
561+
"elasticloadbalancing:AddListenerCertificates",
562+
"elasticloadbalancing:RemoveListenerCertificates",
563+
"elasticloadbalancing:ModifyRule"
564+
],
565+
"Resource" : "*"
566+
}
567+
]
568+
})
569+
}
570+
571+
resource "kubectl_manifest" "load_balancer_controller" {
572+
yaml_body = <<-YAML
573+
apiVersion: v1
574+
kind: ConfigMap
575+
metadata:
576+
name: aws-lb-controller-config
577+
namespace: kube-system
578+
data:
579+
values.yaml: |
580+
clusterName: ${module.eks.cluster_name}
581+
serviceAccount:
582+
create: true
583+
name: aws-load-balancer-controller
584+
annotations:
585+
eks.amazonaws.com/role-arn: ${aws_iam_role.load_balancer_controller.arn}
586+
region: ${data.aws_region.current.name}
587+
vpcId: ${module.vpc.vpc_id}
588+
YAML
589+
}

0 commit comments

Comments
 (0)