k8s/production/aws-load-balancer-controller/release.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: source.toolkit.fluxcd.io/v1beta2
+apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
   name: aws-load-balancer-controller
@@ -9,7 +9,7 @@ spec:
   url: https://aws.github.io/eks-charts
 
 ---
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
+apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
   name: aws-load-balancer-controller
@@ -19,7 +19,7 @@ spec:
   chart:
     spec:
       chart: aws-load-balancer-controller
-      version: 1.8.1  # aws-load-balancer-controller@v2.8.1
+      version: 1.9.0  # aws-load-balancer-controller@v2.9.0
       sourceRef:
         kind: HelmRepository
         name: aws-load-balancer-controller

k8s/production/ingress-nginx/release.yaml

@@ -43,7 +43,12 @@ spec:
       #   configMapNamespace: ""
 
       nodeSelector:
-        spack.io/node-pool: beefy
+        spack.io/node-pool: nginx
+
+      tolerations:
+        - key: "nginx"
+          operator: "Exists"
+          effect: "NoSchedule"
 
       resources:
         requests:
@@ -54,8 +59,8 @@ spec:
       minAvailable: 1
 
       autoscaling:
-        enabled: true
-        minReplicas: 2
+        enabled: false
+        minReplicas: 1
         maxReplicas: 40
         targetCPUUtilizationPercentage: 50
         targetMemoryUtilizationPercentage: 50
@@ -70,6 +75,18 @@ spec:
           # NOTE(opadron): Set the idle timeout for the ingress LB
           "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600"
 
+          service.beta.kubernetes.io/aws-load-balancer-name: spack-eks-load-balancer
+          service.beta.kubernetes.io/aws-load-balancer-type: nlb
+          # Disable client IP preservation - see https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-troubleshooting.html#intermittent-connection-failure
+          service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: "preserve_client_ip.enabled=false"
+          service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+          service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+          service.beta.kubernetes.io/aws-load-balancer-target-node-labels: is-nginx-node=true
+          # service.beta.kubernetes.io/aws-load-balancer-target-node-labels: enable-nginx-ingress=true
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: http
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: /healthz
+          # service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: 10254
+
         labels: {}
 
         enableHttp: true
@@ -99,7 +116,7 @@ spec:
 
     ## Default 404 backend
     defaultBackend:
-      enabled: true
+      enabled: false
       replicaCount: 3
       minAvailable: 1
       nodeSelector:

terraform/modules/spack_aws_k8s/binary_mirrors.tf

@@ -25,7 +25,7 @@ resource "aws_cloudfront_cache_policy" "min_ttl_zero" {
 }
 
 module "pr_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "pull-requests-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries-prs${local.bucket_name_suffix}"
@@ -38,7 +38,7 @@ module "pr_binary_mirror" {
 }
 
 module "protected_binary_mirror" {
-  source = "./modules/binary_mirror"
+  source = "../binary_mirror"
 
   bucket_iam_username = "protected-binary-mirror${local.bucket_name_suffix}"
   bucket_name         = "spack-binaries${local.bucket_name_suffix}"

terraform/modules/spack_aws_k8s/eks.tf

@@ -299,7 +299,7 @@ resource "kubectl_manifest" "cluster_name_config_map" {
 
 
 resource "aws_iam_role" "load_balancer_controller" {
-  name = "AmazonEKSLoadBalancerControllerRole-${var.deployment_name}"
+  name = "AmazonEKSLoadBalancerControllerRole-${var.deployment_name}-${var.deployment_stage}"
   assume_role_policy = jsonencode({
     "Version" : "2012-10-17",
     "Statement" : [
@@ -321,7 +321,7 @@ resource "aws_iam_role" "load_balancer_controller" {
 }
 
 resource "aws_iam_role_policy" "load_balancer_controller" {
-  name = "AmazonEKSLoadBalancerControllerPolicy-${var.deployment_name}"
+  name = "AmazonEKSLoadBalancerControllerPolicy-${var.deployment_name}-${var.deployment_stage}"
   role = aws_iam_role.load_balancer_controller.id
   # Copied from https://docs.aws.amazon.com/eks/latest/userguide/lbc-manifest.html#lbc-iam
   policy = jsonencode({
@@ -365,7 +365,8 @@ resource "aws_iam_role_policy" "load_balancer_controller" {
           "elasticloadbalancing:DescribeTargetGroupAttributes",
           "elasticloadbalancing:DescribeTargetHealth",
           "elasticloadbalancing:DescribeTags",
-          "elasticloadbalancing:DescribeTrustStores"
+          "elasticloadbalancing:DescribeTrustStores",
+          "elasticloadbalancing:DescribeListenerAttributes"
         ],
         "Resource" : "*"
       },
@@ -514,7 +515,8 @@ resource "aws_iam_role_policy" "load_balancer_controller" {
           "elasticloadbalancing:DeleteLoadBalancer",
           "elasticloadbalancing:ModifyTargetGroup",
           "elasticloadbalancing:ModifyTargetGroupAttributes",
-          "elasticloadbalancing:DeleteTargetGroup"
+          "elasticloadbalancing:DeleteTargetGroup",
+          "elasticloadbalancing:ModifyListenerAttributes"
         ],
         "Resource" : "*",
         "Condition" : {

terraform/modules/spack_gitlab/dev_projects.tf

@@ -0,0 +1,8 @@
+module "mike_dev_project" {
+  source = "./modules/developer_project"
+
+  deployment_name  = var.deployment_name
+  deployment_stage = var.deployment_stage
+
+  gitlab_repo = "mvandenburgh/spack"
+}

terraform/modules/spack_gitlab/modules/developer_project/binary_mirrors.tf

@@ -0,0 +1,33 @@
+locals {
+  bucket_name_suffix = "-${replace(data.gitlab_project.this.path_with_namespace, "/", "-")}${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}"
+}
+
+data "aws_cloudfront_cache_policy" "min_ttl_zero" {
+  # Same cache policy that is used for production buckets
+  name = "CachingAllowNoCache${"${var.deployment_name != "prod" ? "-${var.deployment_name}" : ""}-${var.deployment_stage}"}"
+}
+
+module "pr_binary_mirror" {
+  source = "../../../binary_mirror"
+
+  bucket_iam_username = "pull-requests-binary-mirror${local.bucket_name_suffix}"
+  bucket_name         = "spack-binaries-prs${local.bucket_name_suffix}"
+
+  enable_logging      = true
+  logging_bucket_name = "spack-logs${local.bucket_name_suffix}"
+
+  cdn_domain      = "binaries-prs${local.bucket_name_suffix}.spack.io"
+  cache_policy_id = data.aws_cloudfront_cache_policy.min_ttl_zero.id
+}
+
+module "protected_binary_mirror" {
+  source = "../../../binary_mirror"
+
+  bucket_iam_username = "protected-binary-mirror${local.bucket_name_suffix}"
+  bucket_name         = "spack-binaries${local.bucket_name_suffix}"
+
+  enable_logging = false
+
+  cdn_domain      = "binaries${local.bucket_name_suffix}.spack.io"
+  cache_policy_id = data.aws_cloudfront_cache_policy.min_ttl_zero.id
+}

terraform/modules/spack_gitlab/modules/developer_project/gitlab.tf

@@ -0,0 +1,15 @@
+data "gitlab_project" "this" {
+  path_with_namespace = var.gitlab_repo
+}
+
+module "gitlab_runner_configuration" {
+  source = "../gitlab_runner_configuration"
+
+  deployment_name  = var.deployment_name
+  deployment_stage = var.deployment_stage
+
+  pr_binary_bucket_arn        = module.pr_binary_mirror.bucket_arn
+  protected_binary_bucket_arn = module.protected_binary_mirror.bucket_arn
+
+  gitlab_repo = data.gitlab_project.this.path_with_namespace
+}

terraform/modules/spack_gitlab/modules/developer_project/variables.tf

@@ -0,0 +1,11 @@
+variable "deployment_name" {
+  type = string
+}
+
+variable "deployment_stage" {
+  type = string
+}
+
+variable "gitlab_repo" {
+  type = string
+}

terraform/modules/spack_gitlab/modules/developer_project/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    gitlab = {
+      source = "gitlabhq/gitlab"
+    }
+  }
+}

terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/gitlab.tf

@@ -0,0 +1,32 @@
+data "gitlab_project" "this" {
+  path_with_namespace = var.gitlab_repo
+}
+
+resource "gitlab_project_variable" "binary_mirror_role_arn" {
+  for_each = resource.aws_iam_role.gitlab_runner
+
+  project = data.gitlab_project.this.id
+  key     = local.mirror_roles[each.key].role_arn_ci_var_name
+  value   = each.value.arn
+}
+
+# pre_build.py needs access to this to request PR prefix scoped permissions
+resource "gitlab_project_variable" "pr_binary_mirror_bucket_arn" {
+  project = data.gitlab_project.this.id
+  key     = "PR_BINARY_MIRROR_BUCKET_ARN"
+  value   = var.pr_binary_bucket_arn
+}
+
+# Configure retries
+resource "gitlab_project_variable" "retries" {
+  for_each = toset([
+    # Enable retries for artifact downloads, source fetching, and cache restoration in CI jobs
+    "ARTIFACT_DOWNLOAD_ATTEMPTS",
+    "GET_SOURCES_ATTEMPTS",
+    "RESTORE_CACHE_ATTEMPTS",
+  ])
+
+  project = data.gitlab_project.this.id
+  key     = each.value
+  value   = "3"
+}

terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/iam.tf

@@ -0,0 +1,115 @@
+locals {
+  gitlab_domain = "gitlab${var.deployment_name == "prod" ? "" : ".${var.deployment_name}"}.spack.io"
+  suffix        = var.deployment_name != "prod" ? "-${var.deployment_name}" : ""
+
+  mirror_roles = {
+    "pr_binary_mirror" = {
+      "role_name_suffix"     = "PRBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
+      "role_arn_ci_var_name" = "PR_BINARY_MIRROR_ROLE_ARN",
+      "conditions"           = ["project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:pr*"],
+    },
+    "protected_binary_mirror" = {
+      "role_name_suffix"     = "ProtectedBinaryMirror${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}-${var.deployment_stage}",
+      "role_arn_ci_var_name" = "PROTECTED_BINARY_MIRROR_ROLE_ARN",
+      "conditions" = [
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:develop",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:branch:ref:releases/v*",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:tag:ref:develop-*",
+        "project_path:${data.gitlab_project.this.path_with_namespace}:ref_type:tag:ref:v*"
+      ],
+    }
+  }
+}
+
+data "aws_caller_identity" "current" {}
+
+data "tls_certificate" "gitlab" {
+  url = "https://${local.gitlab_domain}"
+}
+
+resource "aws_iam_openid_connect_provider" "gitlab" {
+  url            = "https://${local.gitlab_domain}"
+  client_id_list = keys(local.mirror_roles)
+
+  # Only use the last item in the list, since the first certificate is the root CA, and we don't want to use that.
+  thumbprint_list = [data.tls_certificate.gitlab.certificates[length(data.tls_certificate.gitlab.certificates) - 1].sha1_fingerprint]
+}
+
+data "aws_iam_policy_document" "gitlab_oidc_assume_role" {
+  for_each = local.mirror_roles
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+
+    principals {
+      type        = "Federated"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.gitlab_domain}"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "${local.gitlab_domain}:aud"
+      values   = [each.key]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "${local.gitlab_domain}:sub"
+      values   = each.value.conditions
+    }
+  }
+}
+
+resource "aws_iam_role" "gitlab_runner" {
+  for_each = data.aws_iam_policy_document.gitlab_oidc_assume_role
+
+  name                 = "GitLabRunner${local.mirror_roles[each.key].role_name_suffix}"
+  assume_role_policy   = each.value.json
+  max_session_duration = 3600 * 6 # only allow a max of 6 hours for a session to be active
+}
+
+data "aws_iam_policy_document" "gitlab_runner" {
+  for_each = var.deployment_name != "prod" ? local.mirror_roles : {}
+
+  statement {
+    effect  = "Allow"
+    actions = ["s3:PutObject", "s3:DeleteObject"]
+
+    resources = [
+      each.key == "protected_binary_mirror" ? "${var.protected_binary_bucket_arn}/*" : "${var.pr_binary_bucket_arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "gitlab_runner" {
+  for_each = data.aws_iam_policy_document.gitlab_runner
+
+  name        = "WriteBinariesTo${local.mirror_roles[each.key].role_name_suffix}"
+  description = "Managed by Terraform. IAM Policy that provides access to S3 buckets for binary mirrors."
+  policy      = each.value.json
+}
+
+resource "aws_iam_role_policy_attachment" "gitlab_runner" {
+  for_each = aws_iam_policy.gitlab_runner
+
+  role       = aws_iam_role.gitlab_runner[each.key].name
+  policy_arn = each.value.arn
+}
+
+# attachments for the pre-existing hardcoded policies in production
+resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_pr_binary_mirror" {
+  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinariesPRs",
+  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinariesPRs"]) : []
+
+  role       = aws_iam_role.gitlab_runner["pr_binary_mirror"].name
+  policy_arn = each.value
+}
+
+resource "aws_iam_role_policy_attachment" "legacy_gitlab_runner_protected_binary_mirror" {
+  for_each = var.deployment_name == "prod" ? toset(["arn:aws:iam::588562868276:policy/DeleteObjectsFromBucketSpackBinaries",
+  "arn:aws:iam::588562868276:policy/PutObjectsInBucketSpackBinaries"]) : []
+
+  role       = aws_iam_role.gitlab_runner["protected_binary_mirror"].name
+  policy_arn = each.value
+}

terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/variables.tf

@@ -0,0 +1,21 @@
+variable "deployment_name" {
+  type = string
+}
+
+variable "deployment_stage" {
+  type = string
+}
+
+variable "protected_binary_bucket_arn" {
+  description = "The ARN of the S3 bucket that contains protected binaries."
+  type        = string
+}
+
+variable "pr_binary_bucket_arn" {
+  description = "The ARN of the S3 bucket that contains PR binaries."
+  type        = string
+}
+
+variable "gitlab_repo" {
+  type = string
+}

terraform/modules/spack_gitlab/modules/gitlab_runner_configuration/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    gitlab = {
+      source = "gitlabhq/gitlab"
+    }
+  }
+}