Skip to content

Commit c1ccda5

Browse files
zackgalbreathmvandenburgh
zackgalbreath
authored and
mvandenburgh
committed
Add CDash s3 bucket/postgres db, use helm in staging
Update our terraform config to: * Use postgres instead of mysql * Provision an S3 bucket for CDash to read/write files to * Create a new ServiceAccount to allow CDash to access this S3 bucket
1 parent 26b9e7e commit c1ccda5

File tree

4 files changed

+277
-27
lines changed

4 files changed

+277
-27
lines changed

‎k8s/staging/cdash/kustomization.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
resources:
5+
- ../../production/cdash/certificates.yaml
6+
- ../../production/cdash/ingress.yaml
7+
- ../../production/cdash/namespace.yaml
8+
9+
patches:
10+
- target:
11+
kind: Certificate
12+
name: tls-cdash
13+
namespace: cdash
14+
patch: |-
15+
- op: replace
16+
path: /spec/commonName
17+
value: cdash.staging.spack.io
18+
- op: replace
19+
path: /spec/dnsNames/0
20+
value: cdash.staging.spack.io
21+
22+
- target:
23+
kind: Ingress
24+
name: cdash
25+
namespace: cdash
26+
patch: |-
27+
- op: replace
28+
path: /spec/rules/0/host
29+
value: cdash.staging.spack.io

‎k8s/staging/cdash/release.yaml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
---
2+
apiVersion: source.toolkit.fluxcd.io/v1
3+
kind: HelmRepository
4+
metadata:
5+
name: kitware
6+
namespace: cdash
7+
spec:
8+
interval: 10m
9+
url: https://kitware.github.io/helm
10+
11+
---
12+
apiVersion: helm.toolkit.fluxcd.io/v2
13+
kind: HelmRelease
14+
metadata:
15+
name: cdash
16+
namespace: cdash
17+
spec:
18+
interval: 10m
19+
chart:
20+
spec:
21+
chart: cdash
22+
version: 0.2.1
23+
sourceRef:
24+
kind: HelmRepository
25+
name: kitware
26+
values:
27+
nodeSelector:
28+
spack.io/node-pool: base
29+
30+
cdash:
31+
host: cdash.staging.spack.io
32+
serviceAccountName: cdash
33+
website:
34+
resources:
35+
requests:
36+
memory: 50Mi
37+
worker:
38+
replicas: 1
39+
resources:
40+
requests:
41+
memory: 50Mi
42+
43+
postgresql:
44+
enabled: false
45+
46+
minio:
47+
enabled: false

‎terraform/modules/spack_aws_k8s/cdash_db.tf

Lines changed: 146 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
locals {
2+
cdash_db_name = "cdash"
3+
cdash_db_user = "cdash"
4+
cdash_db_port = "5432"
5+
}
6+
17
resource "aws_db_subnet_group" "cdash_db" {
28
name = "spack-cdash${local.suffix}"
39
subnet_ids = module.vpc.private_subnets
@@ -12,17 +18,17 @@ module "cdash_db" {
1218
source = "terraform-aws-modules/rds/aws"
1319
version = "6.10.0"
1420

15-
identifier = "spack-cdash${local.suffix}"
21+
identifier = "spack-cdash-postgres${local.suffix}"
1622

17-
engine = "mysql"
18-
engine_version = "8.0.40"
19-
family = "mysql8.0"
20-
major_engine_version = "8.0"
23+
engine = "postgres"
24+
family = "postgres17"
25+
major_engine_version = "17"
2126
instance_class = var.cdash_db_instance_class
2227

23-
username = "admin"
24-
port = "3306"
28+
db_name = local.cdash_db_name
29+
username = local.cdash_db_user
2530
password = random_password.cdash_db_password.result
31+
port = local.cdash_db_port
2632
manage_master_user_password = false
2733

2834
publicly_accessible = false
@@ -37,29 +43,142 @@ module "cdash_db" {
3743
skip_final_snapshot = true
3844
deletion_protection = true
3945

40-
allocated_storage = 300
46+
allocated_storage = 400
4147
storage_type = "gp3"
42-
iops = 3000 # 3,000 is the minimum IOPs for <400 GB storage. We can increase this as needed.
43-
storage_throughput = 125 # 125 is the minimum throughput for <400 GB storage. We can increase this as needed.
48+
iops = 12000 # 3,000 is the minimum IOPs for <400 GB storage. We can increase this as needed.
49+
storage_throughput = 500 # 500 is the minimum throughput for >=400 GB storage. We can increase this as needed.
50+
51+
vpc_security_group_ids = [module.postgres_security_group.security_group_id]
52+
}
53+
54+
resource "aws_s3_bucket" "cdash" {
55+
bucket = "spack-cdash${local.suffix}"
56+
lifecycle {
57+
prevent_destroy = true
58+
}
59+
}
60+
61+
# Bucket policy that prevents deletion of CDash bucket.
62+
resource "aws_s3_bucket_policy" "cdash" {
63+
bucket = aws_s3_bucket.cdash.id
64+
65+
policy = jsonencode({
66+
"Version" : "2012-10-17",
67+
"Statement" : [
68+
{
69+
"Principal" : "*"
70+
"Effect" : "Deny",
71+
"Action" : [
72+
"s3:DeleteBucket",
73+
],
74+
"Resource" : aws_s3_bucket.cdash.arn
75+
}
76+
]
77+
})
78+
}
4479

45-
vpc_security_group_ids = [module.mysql_security_group.security_group_id]
80+
resource "aws_iam_role" "cdash" {
81+
name = "CDashS3Role-${var.deployment_name}-${var.deployment_stage}"
82+
description = "Managed by Terraform. Role for CDash to assume so that it can access relevant S3 buckets."
83+
assume_role_policy = jsonencode({
84+
"Version" : "2012-10-17",
85+
"Statement" : [
86+
{
87+
"Effect" : "Allow",
88+
"Principal" : {
89+
"Federated" : module.eks.oidc_provider_arn,
90+
},
91+
"Action" : "sts:AssumeRoleWithWebIdentity",
92+
"Condition" : {
93+
"StringEquals" : {
94+
"${module.eks.oidc_provider}:aud" : "sts.amazonaws.com"
95+
}
96+
}
97+
}
98+
]
99+
})
46100
}
47101

48-
module "mysql_security_group" {
49-
source = "terraform-aws-modules/security-group/aws"
50-
version = "5.2.0"
51-
52-
name = "mysql_sg"
53-
description = "Security group for RDS MySQL database"
54-
vpc_id = module.vpc.vpc_id
55-
56-
ingress_with_cidr_blocks = [
57-
{
58-
from_port = 3306
59-
to_port = 3306
60-
protocol = "tcp"
61-
description = "MySQL access from within VPC"
62-
cidr_blocks = module.vpc.vpc_cidr_block
63-
},
102+
resource "aws_iam_policy" "cdash" {
103+
name = "CDashS3Role-${var.deployment_name}-${var.deployment_stage}"
104+
description = "Managed by Terraform. Grants required permissions for CDash to read/write to relevant S3 buckets."
105+
policy = jsonencode({
106+
"Version" : "2012-10-17",
107+
"Statement" : [
108+
{
109+
"Effect" : "Allow",
110+
"Action" : [
111+
"s3:GetBucketLocation",
112+
"s3:ListBucket"
113+
],
114+
"Resource" : aws_s3_bucket.cdash.arn
115+
},
116+
{
117+
"Effect" : "Allow",
118+
"Action" : [
119+
"s3:DeleteObject",
120+
"s3:DeleteObjectVersion",
121+
"s3:GetObject",
122+
"s3:GetObjectAcl",
123+
"s3:GetObjectVersion",
124+
"s3:PutObject",
125+
"s3:PutObjectAcl",
126+
"s3:ReplicateObject"
127+
],
128+
"Resource" : [
129+
aws_s3_bucket.cdash.arn,
130+
"${aws_s3_bucket.cdash.arn}/*"
131+
]
132+
}
133+
]
134+
})
135+
}
136+
137+
resource "aws_iam_role_policy_attachment" "cdash" {
138+
role = aws_iam_role.cdash.name
139+
policy_arn = aws_iam_policy.cdash.arn
140+
}
141+
142+
resource "kubectl_manifest" "cdash_service_account" {
143+
yaml_body = <<-YAML
144+
apiVersion: v1
145+
kind: ServiceAccount
146+
metadata:
147+
name: cdash
148+
namespace: cdash
149+
annotations:
150+
eks.amazonaws.com/role-arn: ${aws_iam_role.cdash.arn}
151+
YAML
152+
depends_on = [
153+
aws_iam_role_policy_attachment.cdash,
64154
]
65155
}
156+
157+
resource "kubectl_manifest" "cdash_s3_secret" {
158+
yaml_body = <<-YAML
159+
apiVersion: v1
160+
kind: Secret
161+
metadata:
162+
name: cdash-s3
163+
namespace: cdash
164+
stringData:
165+
region: "${data.aws_region.current.name}"
166+
bucket: "${aws_s3_bucket.cdash.id}"
167+
YAML
168+
}
169+
170+
resource "kubectl_manifest" "cdash_db_secret" {
171+
yaml_body = <<-YAML
172+
apiVersion: v1
173+
kind: Secret
174+
metadata:
175+
name: cdash-db
176+
namespace: cdash
177+
stringData:
178+
host: "${module.cdash_db.db_instance_address}"
179+
database: "${local.cdash_db_name}"
180+
username: "${local.cdash_db_user}"
181+
password: "${random_password.cdash_db_password.result}"
182+
port: "${local.cdash_db_port}"
183+
YAML
184+
}

‎terraform/modules/spack_aws_k8s/cdash_db_old.tf

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
module "cdash_db_old" {
2+
source = "terraform-aws-modules/rds/aws"
3+
version = "6.10.0"
4+
5+
identifier = "spack-cdash${local.suffix}"
6+
7+
engine = "mysql"
8+
engine_version = "8.0.40"
9+
family = "mysql8.0"
10+
major_engine_version = "8.0"
11+
instance_class = var.cdash_db_instance_class
12+
13+
username = "admin"
14+
port = "3306"
15+
password = random_password.cdash_db_password.result
16+
manage_master_user_password = false
17+
18+
publicly_accessible = false
19+
db_subnet_group_name = aws_db_subnet_group.cdash_db.name
20+
21+
maintenance_window = "Sun:00:00-Sun:03:00"
22+
backup_window = "03:00-06:00"
23+
create_cloudwatch_log_group = true
24+
performance_insights_enabled = var.deployment_name == "prod"
25+
26+
backup_retention_period = 7
27+
skip_final_snapshot = true
28+
deletion_protection = true
29+
30+
allocated_storage = 300
31+
storage_type = "gp3"
32+
iops = 3000 # 3,000 is the minimum IOPs for <400 GB storage. We can increase this as needed.
33+
storage_throughput = 125 # 125 is the minimum throughput for <400 GB storage. We can increase this as needed.
34+
35+
vpc_security_group_ids = [module.mysql_security_group.security_group_id]
36+
}
37+
38+
module "mysql_security_group" {
39+
source = "terraform-aws-modules/security-group/aws"
40+
version = "5.2.0"
41+
42+
name = "mysql_sg"
43+
description = "Security group for RDS MySQL database"
44+
vpc_id = module.vpc.vpc_id
45+
46+
ingress_with_cidr_blocks = [
47+
{
48+
from_port = 3306
49+
to_port = 3306
50+
protocol = "tcp"
51+
description = "MySQL access from within VPC"
52+
cidr_blocks = module.vpc.vpc_cidr_block
53+
},
54+
]
55+
}

0 commit comments

Comments
 (0)