finish bootstrap flow

.gitignore

@@ -2,6 +2,7 @@
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 # User-specific stuff
+.idea
 .idea/**/workspace.xml
 .idea/**/tasks.xml
 .idea/**/usage.statistics.xml

infra/clusters/management/.terraform.lock.hcl

@@ -0,0 +1,117 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/aminueza/minio" {
+  version     = "2.5.0"
+  constraints = "~> 2.5.0"
+  hashes = [
+    "h1:RrjfsRy+fBVh7VF3r9u7uCCSjAdR5APa6sqbc9b8GfU=",
+    "zh:066cdb289dbfd1675e22fe58c8b42e2732f24fc1528b1919a78dfe28f80e8b30",
+    "zh:26d5e55106259e69493b95058178ec3d6b2395f03a8fe832af1be0e4d89ef42c",
+    "zh:6247e19de9ec6ef719cfcb174b8f08085c0fd5118b3b0de3fb9bb150702b4ad8",
+    "zh:70c3cbab0ba8edeec0db2e175bcdb47255c92f3153f839c4e8f2b0fe8c1366f4",
+    "zh:713793b4b93ae62070b18983ff525390de6c84547cab4220aa068437149f5035",
+    "zh:72de3e532d4bc7c7a4a872aaf00d7e4dfa09f3730668a738bb881d6734248f02",
+    "zh:9090f9288d7bc9f23043c1e65d8535e91f10413a16699d4a18add811b25fa167",
+    "zh:9847284aecb52718468feccb914d67e8befb8bff8345275cb03c3209b338f68b",
+    "zh:aa09ba1aa6fec278198ff352cc7f2977cfe567d31fd948c54fba5db82b4cd7ec",
+    "zh:ca28efbf60400918b9dadd18ecbf683065bf9329b35cbf3826718d8d50f10263",
+    "zh:cb21b119202ac6a30724beb89aefbb8660762b0e9b7165f1e22d59720dd0f110",
+    "zh:f36b4c9fe4795e892b3be2c80a22461f373541f81d335b51afa963097ab29624",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/helm" {
+  version = "2.15.0"
+  hashes = [
+    "h1:WfjJptfaDzC4XCht262FFizAMX8fvRDZWtqUmuLcg88=",
+    "zh:18b94c7c83c30ad166722a61a412e3de6a67935772960e79aaa24c15f8ea0d0f",
+    "zh:4f07c929a71e8169f7471b7600bfcca36dfb295787e975e82ac0455a3ab68b47",
+    "zh:776b804a14c3c4ae6075b12176f81c1f1987214ee1cae4a542599389591cde11",
+    "zh:7c11e3adbe9bd26e88484dcdbd28c473ce3a5c58950a3e3c4f0a2caee225b845",
+    "zh:821e1a53415df0ae4ed523f098360d367a95d6ce3872ba841f22adfdd2f97664",
+    "zh:94c06e483f75a11c3f139c41b3f64b51a96d1d1485e7d1fd3c0f795e2e750945",
+    "zh:aa2040de0b8150ef40222a965445ec40e3df2997ffde1fb062ab4c226689115e",
+    "zh:ad73eebeffe20228656567963477d034b9ed3d1bd2075c1c81150def4927d810",
+    "zh:b77450a36807f3ad1d3ae736d1d165a94fa26f476504a280e9fb2ccb89f648d0",
+    "zh:d2ebd3c34c50c92106ce2df25d5598f47127dc7c60172b9e2fe56ac73dc863a8",
+    "zh:e565995e2614df5ddde75a743a674129288fb91669596a7b0b2580fa7ed49979",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/kubernetes" {
+  version = "2.32.0"
+  hashes = [
+    "h1:HqeU0sZBh+2loFYqPMFx7jJamNUPEykyqJ9+CkMCYE0=",
+    "zh:0e715d7fb13a8ad569a5fdc937b488590633f6942e986196fdb17cd7b8f7720e",
+    "zh:495fc23acfe508ed981e60af9a3758218b0967993065e10a297fdbc210874974",
+    "zh:4b930a8619910ef528bc90dae739cb4236b9b76ce41367281e3bc3cf586101c7",
+    "zh:5344405fde7b1febf0734052052268ee24e7220818155702907d9ece1c0697c7",
+    "zh:92ee11e8c23bbac3536df7b124456407f35c6c2468bc0dbab15c3fc9f414bd0e",
+    "zh:a45488fe8d5bb59c49380f398da5d109a4ac02ebc10824567dabb87f6102fda8",
+    "zh:a4a0b57cf719a4c91f642436882b7bea24d659c08a5b6f4214ce4fe6a0204caa",
+    "zh:b7a27a6d11ba956a2d7b0f7389a46ec857ebe46ae3aeee537250e66cac15bf03",
+    "zh:bf94ce389028b686bfa70a90f536e81bb776c5c20ab70138bbe5c3d0a04c4253",
+    "zh:d965b2608da0212e26a65a0b3f33c5baae46cbe839196be15d93f70061516908",
+    "zh:f441fc793d03057a17af8bdca8b26d54916645bc5c148f54e22a54ed39089e83",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/null" {
+  version = "3.2.3"
+  hashes = [
+    "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=",
+    "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2",
+    "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d",
+    "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3",
+    "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f",
+    "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301",
+    "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670",
+    "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed",
+    "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65",
+    "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd",
+    "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.6.3"
+  hashes = [
+    "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=",
+    "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451",
+    "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8",
+    "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe",
+    "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1",
+    "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36",
+    "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30",
+    "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615",
+    "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad",
+    "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556",
+    "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/vault" {
+  version = "4.4.0"
+  hashes = [
+    "h1:+OO0KHYslvmN+mgRi+v3B6Yg7CYJUyaFh0GIW2hQcCY=",
+    "zh:12758c5afc4160355c55e808f3d0e960a69ef285ddd57f29c3a775ac63c76135",
+    "zh:190c4fbb620bbc07ff850119e17ffbca9f4d81968e69436024fcfb20c69d177e",
+    "zh:2668d3f37e41a539ddca8507a2f8100711cbe54fd7de6d9e82e191c456999674",
+    "zh:59cf5fe3a5cff561c9d15b1b0748fdaeee8966537a5121a20178a1dd265cc22c",
+    "zh:6bf7107b56132281b05932aa8fce8851cd2351d2f6c7d0de4475b5dabf755d4f",
+    "zh:77ee85a529e9ae519aa63950960bd2c2056dd622ad32b08731cb5237e28a9200",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:c3ff4d9c123cc23f95813800c4cea69d1fab29c65b96de4a5932fc328275f527",
+    "zh:c5a6dd8530f720757861da38c16da6577f30cc00423471f8bcc5682c1852027f",
+    "zh:c5acd773d7d24a6116866bf0e4449a3be6849cd6bd2f87141289d47983a0f777",
+    "zh:ede501e9979a586be279b63e0cf5ce69fa89780e37791fe87d8b4283e3716c96",
+    "zh:f5f6ae50a23a184d126832d688380e22311fa1b0192723507a790e57917c3e78",
+  ]
+}

infra/clusters/management/bootstrap/stage2-harden/local.tf

@@ -10,4 +10,10 @@ locals {
   minio_root_password = data.terraform_remote_state.stage1.outputs.minio_root_password
   vault_uri = data.terraform_remote_state.stage1.outputs.vault_uri
   vault_root_token = data.terraform_remote_state.stage1.outputs.vault_root_token
+
+  node_ip = data.terraform_remote_state.stage1.outputs.rancher_node_ip
+
+  id_rsa = file("../stage1-create/id_rsa")
+  id_rsa_pub = file("../stage1-create/id_rsa.pub")
+  k8s_yaml = file("../stage1-create/kube_config_server.yaml")
 }

infra/clusters/management/bootstrap/stage2-harden/main.tf

@@ -10,16 +10,70 @@ resource "minio_s3_bucket" "management" {
   acl = "private"
 }
 
+# TODO: Enable encryption and versioning on the bucket
 # resource "minio_s3_bucket_server_side_encryption" "management" {
 #   bucket = minio_s3_bucket.management.bucket
 #   encryption_type = "aws:kms"
 #   kms_key_id = var.aws_kms_key_id
 # }
 
+resource "minio_s3_object" "id_rsa" {
+    depends_on = [minio_s3_bucket.management]
+    bucket_name = minio_s3_bucket.management.bucket
+    object_name = "id_rsa"
+    content = local.id_rsa
+    content_type = "text/plain"
+}
+
+resource "minio_s3_object" "id_rsa_pub" {
+    depends_on = [minio_s3_bucket.management]
+    bucket_name = minio_s3_bucket.management.bucket
+    object_name = "id_rsa.pub"
+    content = local.id_rsa_pub
+    content_type = "text/plain"
+}
+
+resource "minio_s3_object" "k8s_yaml" {
+    depends_on = [minio_s3_bucket.management]
+    bucket_name = minio_s3_bucket.management.bucket
+    object_name = "kube_config_server.yaml"
+    content = local.k8s_yaml
+    content_type = "text/plain"
+}
+
 resource "minio_iam_user" "management" {
   name = "management"
 }
 
+resource "minio_iam_policy" "management" {
+  name = minio_s3_bucket.management.bucket
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:ListBucket"]
+        Resource = ["arn:aws:s3:::management"]
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject"
+        ]
+        Resource = ["arn:aws:s3:::management/*"]
+      }
+    ]
+  })
+}
+
+
+resource "minio_iam_user_policy_attachment" "management" {
+  user_name   = minio_iam_user.management.id
+  policy_name = minio_iam_policy.management.id
+}
+
 resource "minio_iam_service_account" "management_service_account" {
   target_user = minio_iam_user.management.name
   policy = jsonencode({
@@ -32,7 +86,11 @@ resource "minio_iam_service_account" "management_service_account" {
       },
       {
         Effect = "Allow"
-        Action = ["s3:GetObject", "s3:PutObject"]
+        Action = [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject"
+        ]
         Resource = ["arn:aws:s3:::management/*"]
       }
     ]
@@ -78,6 +136,6 @@ EOT
 resource "vault_token" "management" {
   policies  = [vault_policy.management.name]
   renewable = true
-  ttl       = "720h"  # 30 days
-  period    = "720h"  # Will be renewed every 30 days
+  ttl       = "1h"
+  period    = "15m"
 }

infra/clusters/management/bootstrap/stage2-harden/output.tf

@@ -6,3 +6,15 @@ output "vault_token" {
   value = vault_token.management.client_token
   sensitive = true
 }
+
+output "minio_uri" {
+  value = local.minio_uri
+}
+
+output "minio_server" {
+  value = local.minio_server
+}
+
+output "node_ip" {
+  value = local.node_ip
+}

infra/clusters/management/helm.tf

@@ -1,5 +1,15 @@
+provider "kubernetes" {
+  host                   = yamldecode(data.minio_s3_object.k8s_yaml.content).clusters[0].cluster.server
+  client_certificate     = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).users[0].user.client-certificate-data)
+  client_key             = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).users[0].user.client-key-data)
+  cluster_ca_certificate = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).clusters[0].cluster.certificate-authority-data)
+}
+
 provider "helm" {
   kubernetes {
-    config_path = "./bootstrap/stage1-initial-infra/kube_config_server.yaml"
+    host                   = yamldecode(data.minio_s3_object.k8s_yaml.content).clusters[0].cluster.server
+    client_certificate     = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).users[0].user.client-certificate-data)
+    client_key             = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).users[0].user.client-key-data)
+    cluster_ca_certificate = base64decode(yamldecode(data.minio_s3_object.k8s_yaml.content).clusters[0].cluster.certificate-authority-data)
   }
 }

infra/clusters/management/main.tf

@@ -1,12 +1,5 @@
 terraform {
   backend "s3" {
-     endpoints = {
-       s3 = "https://storage.168.119.61.8.sslip.io"
-     }
-     bucket     = "management"
-     key        = "terraform.tfstate"
-     region     = "eu-central-1"
-
      #encrypt = false
      skip_region_validation = true
      skip_metadata_api_check = true

infra/clusters/management/minio.tf

@@ -0,0 +1,26 @@
+terraform {
+  required_providers {
+    minio = {
+      source = "aminueza/minio"
+      version = "~> 2.5.0"
+    }
+  }
+}
+
+provider "minio" {
+  minio_server   = var.minio_server
+  minio_region   = var.region
+  minio_user     = var.access_key
+  minio_password = var.secret_key
+  minio_ssl      = true
+}
+
+data "minio_s3_object" "k8s_yaml" {
+  bucket_name = var.bucket
+  object_name = "kube_config_server.yaml"
+}
+
+data "minio_s3_object" "id_rsa" {
+  bucket_name = var.bucket
+  object_name = "id_rsa"
+}

infra/clusters/management/output.tf

@@ -0,0 +1,5 @@
+output "rancher_bootstrap_password" {
+  value = vault_kv_secret_v2.rancher_creds.data["admin_password"]
+  sensitive = true
+}
+

infra/clusters/management/rancher.tf

@@ -12,23 +12,40 @@ resource "vault_kv_secret_v2" "rancher_creds" {
   })
 }
 
-resource "helm_release" "rancher" {
-  name             = "rancher"
+resource "kubernetes_secret" "bootstrap_secret" {
+  metadata {
+    name      = "bootstrap-secret"
     namespace = "cattle-system"
-  chart            = "https://releases.rancher.com/server-charts/latest/rancher-2.9.1.tgz"
-
-  reuse_values     = true
-  recreate_pods    = false
-
-  set_sensitive {
-    name  = "adminPassword"
-    value = vault_kv_secret_v2.rancher_creds.data["admin_password"]
+    annotations = {
+      "field.cattle.io/projectId" = "local:p-q7vbv"
+      "helm.sh/hook"              = "pre-install,pre-upgrade"
+      "helm.sh/hook-weight"       = "-5"
+      "helm.sh/resource-policy"   = "keep"
+    }
   }
 
-  lifecycle {
-    ignore_changes = [
-      set,
-      set_sensitive,
-    ]
+  data = {
+    bootstrapPassword = vault_kv_secret_v2.rancher_creds.data["admin_password"]
   }
+
+  type = "Opaque"
+}
+
+# Force a rollout of the Rancher deployment to pick up the new secret
+resource "null_resource" "rancher_rollout" {
+  triggers = {
+    password_change = kubernetes_secret.bootstrap_secret.data["bootstrapPassword"]
+  }
+
+  provisioner "remote-exec" {
+    inline = ["kubectl rollout restart deployment rancher -n cattle-system"]
+    connection {
+      type        = "ssh"
+      host        = var.node_ip
+      user        = var.node_username
+      private_key = data.minio_s3_object.id_rsa.content
+    }
+  }
+
+  depends_on = [kubernetes_secret.bootstrap_secret]
 }

infra/clusters/management/scripts/generate-tf-backend-config.sh

@@ -1,11 +1,24 @@
 #!/bin/bash
-MINIO_ACCESS_KEY=$(vault kv get -mount="management" -field="access_key" "minio")
-MINIO_SECRET_KEY=$(vault kv get -mount="management" -field="secret_key" "minio")
+
+VAULT_TOKEN=$(cd bootstrap/stage2-harden && terraform output -raw vault_token)
+VAULT_ADDR=$(cd bootstrap/stage2-harden && terraform output -raw vault_uri)
+MINIO_ADDR=$(cd bootstrap/stage2-harden && terraform output -raw minio_uri)
+MINIO_SERVER=$(cd bootstrap/stage2-harden && terraform output -raw minio_server)
+NODE_IP=$(cd bootstrap/stage2-harden && terraform output -raw node_ip)
+
+MINIO_ACCESS_KEY=$(VAULT_TOKEN="$VAULT_TOKEN" VAULT_ADDR="$VAULT_ADDR" vault kv get -mount="management" -field="access_key" "minio")
+MINIO_SECRET_KEY=$(VAULT_TOKEN="$VAULT_TOKEN" VAULT_ADDR="$VAULT_ADDR" vault kv get -mount="management" -field="secret_key" "minio")
 
 cat << EOF > backend.tfvars
+endpoints       = { s3 = "${MINIO_ADDR}" }
 access_key      = "${MINIO_ACCESS_KEY}"
 secret_key      = "${MINIO_SECRET_KEY}"
-bucket     = "${BUCKET}"
+bucket          = "management"
 key             = "terraform.tfstate"
 region          = "eu-central-1"
+minio_server    = "${MINIO_SERVER}"
+vault_token     = "${VAULT_TOKEN}"
+vault_addr      = "${VAULT_ADDR}"
+node_ip         = "${NODE_IP}"
+node_username   = "root"
 EOF

infra/clusters/management/scripts/vault-to-env.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd bootstrap/stage2-harden/
+terraform output vault_token | read -s VAULT_TOKEN
+terraform output vault_uri | read -s VAULT_ADDRESS
+cd - > /dev/null

infra/clusters/management/state.tf

@@ -1,7 +0,0 @@
-data "terraform_remote_state" "stage2" {
-  backend = "local"
-
-  config = {
-    path = "./bootstrap/stage2-harden/terraform.tfstate"
-  }
-}

infra/clusters/management/variables.tf

@@ -0,0 +1,47 @@
+variable "endpoints" {
+  type = map(string)
+}
+
+variable "access_key" {
+  type = string
+  sensitive = true
+}
+
+variable "secret_key" {
+  type = string
+  sensitive = true
+}
+
+variable "bucket" {
+  type = string
+}
+
+variable "key" {
+  type = string
+}
+
+variable "region" {
+  type = string
+}
+
+variable "minio_server" {
+  type = string
+}
+
+variable "vault_addr" {
+  type = string
+}
+
+variable "vault_token" {
+  type = string
+  sensitive = true
+}
+
+variable "node_ip" {
+  type = string
+}
+
+variable "node_username" {
+  type = string
+  default = "root"
+}

infra/clusters/management/vault.tf

@@ -1,4 +1,4 @@
 provider "vault" {
-  address = "http://127.0.0.1:8200"
-  token = data.terraform_remote_state.stage4.outputs.vault_token
+  address = var.vault_addr
+  token = var.vault_token
 }

infra/modules/minio/main.tf

@@ -30,7 +30,7 @@ resource "helm_release" "minio" {
   values = [
     <<-EOT
     ingress:
-      enabled: false
+      enabled: true
       ingressClassName: traefik
       tls: true
       hostname: ${local.hostname_admin}