From 4abf8bc6a9aa901b5375bfb8a48bdbaa700a4a47 Mon Sep 17 00:00:00 2001 From: Thomas Rijpstra Date: Fri, 7 Mar 2025 19:12:54 +0100 Subject: [PATCH] wip --- infra/modules/mongodb/main.tf | 10 +- infra/modules/mongodb/values.yaml | 12 +- infra/modules/postgresql/main.tf | 18 +- infra/modules/zitadel/api-m2m-swagger/main.tf | 9 +- .../project/application/user-agent/main.tf | 6 +- infra/tenants/365zon/zitadel/main.tf | 21 + .../argocd/zitadel/groupsClaim.action.tftpl | 1 + shuttles/.gitignore | 2 + shuttles/kubeconfig | 19 - shuttles/setup-cluster.ts | 476 +++++++++++++----- shuttles/terraform/.terraform.lock.hcl | 143 ------ shuttles/terraform/zitadel-admin-sa.json | 2 +- 12 files changed, 420 insertions(+), 299 deletions(-) create mode 100644 shuttles/.gitignore delete mode 100644 shuttles/kubeconfig delete mode 100644 shuttles/terraform/.terraform.lock.hcl diff --git a/infra/modules/mongodb/main.tf b/infra/modules/mongodb/main.tf index eb5b271..7110384 100644 --- a/infra/modules/mongodb/main.tf +++ b/infra/modules/mongodb/main.tf @@ -58,6 +58,14 @@ output "installed" { } output "connection_string" { - value = format("mongodb://%s:%s@%s:%s/%s?", "root", random_password.mongodb_root_password.result, "mongodb-headless.mongodb.svc.cluster.local", "27017", "admin") + value = format( + "mongodb://%s:%s@%s/%s?replicaSet=rs0&authSource=admin", + "root", + random_password.mongodb_root_password.result, + join(",", [ + for i in range(var.replicas) :format("mongodb-%d.mongodb-headless.mongodb.svc.cluster.local:27017", i) + ]), + "admin" + ) sensitive = true } diff --git a/infra/modules/mongodb/values.yaml b/infra/modules/mongodb/values.yaml index 86acfeb..b0a4614 100644 --- a/infra/modules/mongodb/values.yaml +++ b/infra/modules/mongodb/values.yaml @@ -16,14 +16,14 @@ mongodb: readinessProbe: initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 5 + timeoutSeconds: 15 failureThreshold: 3 successThreshold: 1 livenessProbe: initialDelaySeconds: 60 periodSeconds: 20 - timeoutSeconds: 5 + timeoutSeconds: 15 failureThreshold: 6 # Proper shutdown handling @@ -55,3 +55,11 @@ auth: - ${ database } %{ endfor ~} %{ endif } + +resources: + limits: + cpu: 1000m + memory: 1.5Gi + requests: + cpu: 500m + memory: 1Gi diff --git a/infra/modules/postgresql/main.tf b/infra/modules/postgresql/main.tf index cbc4c0f..5de514b 100644 --- a/infra/modules/postgresql/main.tf +++ b/infra/modules/postgresql/main.tf @@ -11,13 +11,23 @@ resource "kubernetes_namespace" "postgresql" { } resource "random_password" "postgresql_user_password" { - length = 40 - special = true + length = 40 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" + min_special = 2 + min_upper = 2 + min_lower = 2 + min_numeric = 2 } resource "random_password" "postgresql_root_password" { - length = 40 - special = true + length = 40 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" + min_special = 2 + min_upper = 2 + min_lower = 2 + min_numeric = 2 } resource "kubernetes_secret" "postgresql_auth" { diff --git a/infra/modules/zitadel/api-m2m-swagger/main.tf b/infra/modules/zitadel/api-m2m-swagger/main.tf index 325cb2d..d16c8a3 100644 --- a/infra/modules/zitadel/api-m2m-swagger/main.tf +++ b/infra/modules/zitadel/api-m2m-swagger/main.tf @@ -51,7 +51,7 @@ resource "kubernetes_secret" "user-agent" { data = { "authority" = local.authority - "audience" = "urn:zitadel:iam:org:project:id:${ var.project_id }:aud" + "audience" = var.project_id "client_id" = module.zitadel_project_application_ua.client_id } } @@ -110,8 +110,13 @@ resource "kubernetes_secret" "service-account" { data = { "authority" = local.authority - "audience" = "urn:zitadel:iam:org:project:id:${ var.project_id }:aud" + "audience" = var.project_id "client_id" = module.zitadel_service_account[count.index].client_id "client_secret" = module.zitadel_service_account[count.index].client_secret } } + +output "installed" { + value = true + depends_on = [kubernetes_secret.service-account] +} diff --git a/infra/modules/zitadel/project/application/user-agent/main.tf b/infra/modules/zitadel/project/application/user-agent/main.tf index 923bef5..e2d917c 100644 --- a/infra/modules/zitadel/project/application/user-agent/main.tf +++ b/infra/modules/zitadel/project/application/user-agent/main.tf @@ -20,11 +20,11 @@ resource "zitadel_application_oidc" "default" { response_types = ["OIDC_RESPONSE_TYPE_CODE"] # // If selected, the requested roles of the authenticated user are added to the access token. - #access_token_type = "OIDC_TOKEN_TYPE_JWT" - #access_token_role_assertion = true + access_token_type = "OIDC_TOKEN_TYPE_JWT" + access_token_role_assertion = true # BEARER uses an Opaque token, which needs the introspection endpoint and `urn:zitadel:iam:org:project:id::aud` scope - access_token_type = "OIDC_TOKEN_TYPE_BEARER" + #access_token_type = "OIDC_TOKEN_TYPE_BEARER" # // If you want to add additional Origins to your app which is not used as a redirect you can do that here. #additional_origins = [] diff --git a/infra/tenants/365zon/zitadel/main.tf b/infra/tenants/365zon/zitadel/main.tf index 082d492..395263e 100644 --- a/infra/tenants/365zon/zitadel/main.tf +++ b/infra/tenants/365zon/zitadel/main.tf @@ -18,6 +18,7 @@ module "zitadel_project" { module "zitadel_project_operator_roles" { source = "../../../modules/zitadel/project/roles" + wait_on = [module.zitadel_project.installed] org_id = var.org_id project_id = module.zitadel_project.project_id group = "Operator" @@ -29,6 +30,7 @@ module "zitadel_project_operator_roles" { module "zitadel_project_configurator_roles" { source = "../../../modules/zitadel/project/roles" + wait_on = [module.zitadel_project_operator_roles.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -40,6 +42,7 @@ module "zitadel_project_configurator_roles" { module "zitadel_project_developer_roles" { source = "../../../modules/zitadel/project/roles" + wait_on = [module.zitadel_project_configurator_roles.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -49,12 +52,22 @@ module "zitadel_project_developer_roles" { ] } +module "zitadel_project_user_grant" { + source = "../../../modules/zitadel/project/user-grant" + wait_on = [module.zitadel_project_developer_roles.installed] + org_id = var.org_id + project_id = module.zitadel_project.project_id + user_id = var.user_id + roles = concat(module.zitadel_project_developer_roles.roles, module.zitadel_project_configurator_roles.roles, module.zitadel_project_operator_roles.roles) +} + // TODO: Move External (and 365zon Push service account) to own project // TODO: Add grant for external project // TODO: Add read roles module "zitadel_project_application_core" { source = "../../../modules/zitadel/api-m2m-swagger" + wait_on = [module.zitadel_project_user_grant.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -72,6 +85,7 @@ module "zitadel_project_application_core" { module "zitadel_project_application_salesforce" { source = "../../../modules/zitadel/api-m2m-swagger" + wait_on = [module.zitadel_project_application_core.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -88,6 +102,7 @@ module "zitadel_project_application_salesforce" { module "zitadel_project_application_external" { source = "../../../modules/zitadel/api-m2m-swagger" + wait_on = [module.zitadel_project_application_salesforce.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -104,6 +119,7 @@ module "zitadel_project_application_external" { module "zitadel_project_application_module_internal" { source = "../../../modules/zitadel/api-m2m-swagger" + wait_on = [module.zitadel_project_application_external.installed] org_id = var.org_id project_id = module.zitadel_project.project_id @@ -130,3 +146,8 @@ output "org_id" { output "project_id" { value = module.zitadel_project.project_id } + +output "installed" { + value = true + depends_on = [module.zitadel_project_application_external.installed] +} diff --git a/infra/tenants/argocd/zitadel/groupsClaim.action.tftpl b/infra/tenants/argocd/zitadel/groupsClaim.action.tftpl index 308dc51..5aa9094 100644 --- a/infra/tenants/argocd/zitadel/groupsClaim.action.tftpl +++ b/infra/tenants/argocd/zitadel/groupsClaim.action.tftpl @@ -24,4 +24,5 @@ function groupsClaim(ctx, api) { }); api.v1.claims.setClaim("groups", grants); + api.v1.claims.setClaim("scope", grants); } diff --git a/shuttles/.gitignore b/shuttles/.gitignore new file mode 100644 index 0000000..2171f23 --- /dev/null +++ b/shuttles/.gitignore @@ -0,0 +1,2 @@ +kubeconfig +*.lock.hcl diff --git a/shuttles/kubeconfig b/shuttles/kubeconfig deleted file mode 100644 index 342348c..0000000 --- a/shuttles/kubeconfig +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTXprM09ESTROelF3SGhjTk1qVXdNakUzTURrd01URTBXaGNOTXpVd01qRTFNRGt3TVRFMApXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTXprM09ESTROelF3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUWVNEV1Jwbmd6TE5ySGphTmhqdmM1SU82a2dibVpwaER4WVROTG11MjAKaWxaQnZLRlZRdW5kV3ZEQ1VrcGJNRjNsOTRuSmxaYVByK3lDSnJpVVh0UjZvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVQ5bVZxTGcvSFBCUS91L3MzbHAwCjhJQ0RDc013Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQUpjMkJkMjd0SzNZTFpwa01yOFNMSEIvbngzd1E1MU0KRnRaYnBNVzJudVNXQWlFQTMyUmcyVHZNQW9LYll5bnhySkk3U3g5eWszZHFsSWd5TW15d2M5d1JicmM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://10.110.36.47:6443 - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -kind: Config -preferences: {} -users: -- name: default - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJZFh2OWlXRHR6SE13Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOek01TnpneU9EYzBNQjRYRFRJMU1ESXhOekE1TURFeE5Gb1hEVEkyTURJeApOekE1TURFeE5Gb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKNlNVZm5ESVJndVRDMjkKaWFjVTdTM3VPWkw1RERGZjJPQi9IakdTWEErQlRGaE5VOGtMSHBxZlZYeWVKbHNkd09mR1QvL2JQbENsWFYvdQowc0wyTW5halNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCUXdoZkJDTWRocVpXMW96WlEzZG84d1VYOEpCREFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQXczSFpKY1cwaGI3ZUwxSktvcTJ2cExFaFVxVncxRG1oTGJtcUNQTVdmcEFDSVFDRkhXcDhoTTNMdTROTgpGUnYxc2pkYS93VjdmSVpUcUsyZHVNOUNPQVc5emc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZHpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFM016azNPREk0TnpRd0hoY05NalV3TWpFM01Ea3dNVEUwV2hjTk16VXdNakUxTURrd01URTAKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFM016azNPREk0TnpRd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBUjJCcXE5cVhESmZGeVQ1VVpEY3Z6SHVPdDg2TEZ5WTlDb1oxL0xxeldGClZMdHVQYUFXc3BUdUtZckJieTRZRlBQQlQ1M0RkS1F5cjhhWG5HUDRWenlxbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVNSVh3UWpIWWFtVnRhTTJVTjNhUApNRkYvQ1FRd0NnWUlLb1pJemowRUF3SURTQUF3UlFJZ1lmS01YQ3lFelBmM05wN3paLzVYTnFxeTdjTDBpMXBWCkpjZzNzYmtMbXB3Q0lRRDlzYVpmekswRlUrNWljWFpLZmUyVFg0WW5sNS96aFVGR2FHb2RTb1ovUXc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUtlQVpqUzhNM1ZBd2l6cWo0UDN6RURuQmNaYldrcDJPekt2VlNpUSs0azRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFbnBKUitjTWhHQzVNTGIySnB4VHRMZTQ1a3ZrTU1WL1k0SDhlTVpKY0Q0Rk1XRTFUeVFzZQptcDlWZko0bVd4M0E1OFpQLzlzK1VLVmRYKzdTd3ZZeWRnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/shuttles/setup-cluster.ts b/shuttles/setup-cluster.ts index 28a2fef..59a372f 100755 --- a/shuttles/setup-cluster.ts +++ b/shuttles/setup-cluster.ts @@ -1,149 +1,377 @@ #!/usr/bin/env -S deno run --allow-run --allow-read --allow-write +// Note: TypeScript errors related to Deno imports and namespace can be safely ignored +// These are only relevant when running the script with the Deno runtime import { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts"; import { delay } from "https://deno.land/std/async/mod.ts"; +import { exists } from "https://deno.land/std/fs/mod.ts"; -const alpineImage = "alpine/edge/cloud" -const alpineConfig = ['--profile', 'cloud-init-alpine'] -const archImage = "archlinux/current/cloud" -const archConfig = ['--profile', 'cloud-init-arch'] +// Configuration constants +const alpineImage = "alpine/edge/cloud"; +const alpineConfig = ['--profile', 'cloud-init-alpine']; +const archImage = "archlinux/current/cloud"; +const archConfig = ['--profile', 'cloud-init-arch']; +const getIp = (i: number) => `10.110.36.${109 + i}`; -const image = archImage -const config = archConfig +const image = archImage; +const config = archConfig; -const findIP4 = (name: string, nodeList: any) => { - const ip4 = nodeList?.find((n) => n.name === name)?.state?.network?.eth0?.addresses?.find((n) => n.family === 'inet')?.address; - return ip4; +// Enhanced logging function with timestamps and log levels +const log = { + debug: (message: string) => console.log(`[${new Date().toISOString()}] [DEBUG] ${message}`), + info: (message: string) => console.log(`[${new Date().toISOString()}] [INFO] ${message}`), + success: (message: string) => console.log(`[${new Date().toISOString()}] [SUCCESS] ✅ ${message}`), + warning: (message: string) => console.log(`[${new Date().toISOString()}] [WARNING] ⚠️ ${message}`), + error: (message: string) => console.error(`[${new Date().toISOString()}] [ERROR] ❌ ${message}`), + skip: (message: string) => console.log(`[${new Date().toISOString()}] [SKIP] ⏩ ${message}`), +}; + +// Helper function to execute commands with proper error handling +async function executeCommand( + cmdArray: string[], + description: string, + options: { + stdout?: "piped" | "inherit" | "null", + stderr?: "piped" | "inherit" | "null", + throwOnError?: boolean + } = {} +): Promise<{ success: boolean; output?: string; error?: string }> { + const { stdout = "piped", stderr = "piped", throwOnError = true } = options; + + log.debug(`Executing: ${cmdArray.join(" ")}`); + + try { + // Use Deno.Command API which is the modern replacement for Deno.run + const command = new Deno.Command(cmdArray[0], { + args: cmdArray.slice(1), + stdout: stdout === "piped" ? "piped" : stdout === "inherit" ? "inherit" : "null", + stderr: stderr === "piped" ? "piped" : stderr === "inherit" ? "inherit" : "null", + }); + + const { code, stdout: stdoutOutput, stderr: stderrOutput } = await command.output(); + + const stdoutText = stdout === "piped" ? new TextDecoder().decode(stdoutOutput).trim() : ""; + const stderrText = stderr === "piped" ? new TextDecoder().decode(stderrOutput).trim() : ""; + + if (code !== 0) { + log.error(`Failed to ${description}: ${stderrText || "Unknown error"}`); + if (throwOnError) { + throw new Error(`Command failed: ${cmdArray.join(" ")}\n${stderrText}`); + } + } + + return { + success: code === 0, + output: stdoutText, + error: stderrText + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + log.error(`Exception while ${description}: ${errorMessage}`); + if (throwOnError) { + throw error; + } + return { success: false, error: errorMessage }; + } } -const setupCluster = async (numMasters: number) => { - const hostname = await Deno.run({ - cmd: ["hostnamectl", "hostname"], - stdout: "piped", - }).output().then((output) => new TextDecoder().decode(output).trim()); - - const user = await Deno.run({ - cmd: ["whoami"], - stdout: "piped", - }).output().then((output) => new TextDecoder().decode(output).trim()); - - const sshKeyPubFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}.pub`; - const sshKeyPrivateFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}`; - - // Step 1: Create Low-Resource Profile (if not exists) - const profileExists = await Deno.run({ - cmd: ["incus", "profile", "show", "low-resource"], - stdout: "null", - stderr: "null", - }).status().then((status) => status.success); - - if (!profileExists) { - await Deno.run({ - cmd: ["incus", "profile", "create", "low-resource"], - }).status(); - await Deno.run({ - cmd: ["incus", "profile", "set", "low-resource", "limits.cpu=1", "limits.memory=512MB"], - }).status(); - await Deno.run({ - cmd: ["incus", "profile", "device", "add", "low-resource", "root", "disk", "pool=default", "path=/"], - }).status(); - await Deno.run({ - cmd: ["incus", "profile", "device", "add", "low-resource", "eth-0", "nic", "network=incusbr0"], - }).status(); - console.log("✅ Low-resource profile created."); - } else { - console.log("⏩ Low-resource profile already exists."); +// Check if VM is ready for SSH connections +async function isVmReadyForSsh(ip: string, user: string, maxAttempts = 30): Promise { + log.info(`Checking if VM at ${ip} is ready for SSH connections...`); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + log.debug(`SSH readiness check attempt ${attempt}/${maxAttempts}`); + + const { success } = await executeCommand( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=5", `${user}@${ip}`, "echo", "ready"], + `check SSH connectivity to ${ip}`, + { throwOnError: false, stderr: "null" } + ); + + if (success) { + log.success(`VM at ${ip} is ready for SSH connections`); + return true; + } + + log.debug(`VM at ${ip} not ready yet, waiting...`); + await delay(2000); // Wait 2 seconds between attempts } + + log.error(`VM at ${ip} is not ready for SSH connections after ${maxAttempts} attempts`); + return false; +} +// Check if VM is running +async function isVmRunning(vmName: string): Promise { + const { success, output } = await executeCommand( + ["incus", "list", vmName, "--format", "json"], + `check if VM ${vmName} is running`, + { throwOnError: false } + ); + + if (!success || !output) { + return false; + } + + try { + const vmInfo = JSON.parse(output); + return vmInfo.length > 0 && vmInfo[0].status === "Running"; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + log.error(`Failed to parse VM status: ${errorMessage}`); + return false; + } +} - const sshKey = await Deno.readTextFile(sshKeyPubFileName); - - // Step 3: Launch VMs (if not already running) - for (let i = 1; i <= numMasters; i++) { - const vmName = `k3s-master${i}`; - const vmExists = await Deno.run({ - cmd: ["incus", "list", vmName, "--format", "csv"], - stdout: "piped", - }).output().then((output) => new TextDecoder().decode(output).trim() !== ""); - - if (!vmExists) { - await Deno.run({ - cmd: ["incus", "launch", `images:${image}`, vmName, "--profile", "low-resource", "-c", "user.timezone=\"Europe/Amsterdam\"", "-c", `user.ssh_key=\"${sshKey}\"`, ...config], - }).status(); - console.log(`✅ VM ${vmName} launched.`); - } else { - console.log(`⏩ VM ${vmName} already exists.`); +// Cleanup function to handle failures +async function cleanup(vmNames: string[], shouldRemove = false): Promise { + log.info("Starting cleanup process..."); + + for (const vmName of vmNames) { + // Check if VM exists + const { success, output } = await executeCommand( + ["incus", "list", vmName, "--format", "csv"], + `check if VM ${vmName} exists`, + { throwOnError: false } + ); + + if (success && output) { + // Stop VM if it's running + const isRunning = await isVmRunning(vmName); + if (isRunning) { + log.info(`Stopping VM ${vmName}...`); + await executeCommand( + ["incus", "stop", vmName, "--force"], + `stop VM ${vmName}`, + { throwOnError: false } + ); + } + + // Remove VM if requested + if (shouldRemove) { + log.info(`Removing VM ${vmName}...`); + await executeCommand( + ["incus", "delete", vmName], + `remove VM ${vmName}`, + { throwOnError: false } + ); + } } } + + log.success("Cleanup completed"); +} - // Step 4: Install k3sup (if not installed) - const k3supInstalled = await Deno.run({ - cmd: ["which", "k3sup"], - stdout: "null", - stderr: "null", - }).status().then((status) => status.success); - - if (!k3supInstalled) { - await Deno.run({ - cmd: ["sh", "-c", "curl -sLS https://get.k3sup.dev | sh"], - }).status(); - console.log("✅ k3sup installed."); - } else { - console.log("⏩ k3sup already installed."); - } - - // Step 5: Bootstrap First Master Node (if not already bootstrapped) - let firstMasterIP; - let nodes; - while (firstMasterIP === undefined) { - nodes = await Deno.run({ - cmd: ["incus", "list", "--format", "json"], - stdout: "piped", - }).output().then((output) => JSON.parse(new TextDecoder().decode(output))); - firstMasterIP = findIP4('k3s-master1', nodes) - await delay(1000) - } - - const kubeconfigExists = await Deno.stat("./kubeconfig").then(() => true).catch(() => false); - - if (!kubeconfigExists) { - await Deno.run({ - cmd: ["k3sup", "install", "--ip", firstMasterIP, "--user", "picard", "--cluster", "--ssh-key", sshKeyPrivateFileName], - }).status(); - console.log("✅ First master node bootstrapped."); - } else { - console.log("⏩ First master node already bootstrapped."); - } - - // Step 6: Join Additional Master Nodes (if not already joined) - for (let i = 2; i <= numMasters; i++) { - const vmName = `k3s-master${i}`; - const vmIP = findIP4(vmName, nodes) - - const joined = await Deno.run({ - cmd: ["kubectl", "get", "nodes", vmName], - stdout: "null", - stderr: "null", - }).status().then((status) => status.success); - - if (!joined) { - await Deno.run({ - cmd: ["k3sup", "join", "--server", "--ip", vmIP, "--server-ip", firstMasterIP, "--user", "picard", "--ssh-key", sshKeyPrivateFileName], - }).status(); - console.log(`✅ VM ${vmName} joined the cluster.`); - } else { - console.log(`⏩ VM ${vmName} already joined the cluster.`); +const setupCluster = async (numMasters: number, forceCleanup = false) => { + log.info(`Starting setup of k3s cluster with ${numMasters} master nodes`); + + const createdVMs: string[] = []; + + try { + // Get hostname and user + const { output: hostname } = await executeCommand( + ["hostnamectl", "hostname"], + "get hostname" + ); + + const { output: user } = await executeCommand( + ["whoami"], + "get current user" + ); + + const sshKeyPubFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}.pub`; + const sshKeyPrivateFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}`; + + // Check if SSH keys exist + if (!await exists(sshKeyPubFileName) || !await exists(sshKeyPrivateFileName)) { + log.error(`Required SSH keys not found: ${sshKeyPubFileName} or ${sshKeyPrivateFileName}`); + throw new Error("SSH keys not found"); } + + // Step 1: Create Low-Resource Profile (if not exists) + const { success: profileExists } = await executeCommand( + ["incus", "profile", "show", "low-resource"], + "check if low-resource profile exists", + { stdout: "null", stderr: "null", throwOnError: false } + ); + + if (!profileExists) { + log.info("Creating low-resource profile..."); + await executeCommand( + ["incus", "profile", "create", "low-resource"], + "create low-resource profile" + ); + await executeCommand( + ["incus", "profile", "set", "low-resource", "limits.cpu=1", "limits.memory=512MB"], + "set low-resource profile limits" + ); + await executeCommand( + ["incus", "profile", "device", "add", "low-resource", "root", "disk", "pool=default", "path=/"], + "add root disk to low-resource profile" + ); + // await executeCommand( + // ["incus", "profile", "device", "add", "low-resource", "eth-0", "nic", "network=incusbr0"], + // "add network interface to low-resource profile" + // ); + log.success("Low-resource profile created"); + } else { + log.skip("Low-resource profile already exists"); + } + + // Read SSH key + const sshKey = await Deno.readTextFile(sshKeyPubFileName); + + // Step 3: Launch VMs (if not already running) + for (let i = 1; i <= numMasters; i++) { + const vmName = `k3s-master${i}`; + + const { success: vmExists, output: vmOutput } = await executeCommand( + ["incus", "list", vmName, "--format", "csv"], + `check if VM ${vmName} exists`, + { throwOnError: false } + ); + + if (!vmExists || !vmOutput) { + log.info(`Creating VM ${vmName}...`); + await executeCommand( + ["incus", "init", `images:${image}`, vmName, "--profile", "low-resource", "-c", "user.timezone=\"Europe/Amsterdam\"", "-c", `user.ssh_key=\"${sshKey}\"`, ...config], + `initialize VM ${vmName}` + ); + + await executeCommand( + ["incus", "config", 'device', 'add', vmName, 'eth0', 'nic', 'nictype=bridged', 'parent=incusbr0', `ipv4.address=${getIp(i)}`], + `configure network for VM ${vmName}` + ); + + await executeCommand( + ["incus", "start", vmName], + `start VM ${vmName}` + ); + + createdVMs.push(vmName); + log.success(`VM ${vmName} started`); + } else { + // Check if VM is running, if not, start it + const isRunning = await isVmRunning(vmName); + if (!isRunning) { + log.info(`Starting existing VM ${vmName}...`); + await executeCommand( + ["incus", "start", vmName], + `start VM ${vmName}` + ); + } + log.skip(`VM ${vmName} already exists`); + } + } + + // Step 4: Install k3sup (if not installed) + const { success: k3supInstalled } = await executeCommand( + ["which", "k3sup"], + "check if k3sup is installed", + { stdout: "null", stderr: "null", throwOnError: false } + ); + + if (!k3supInstalled) { + log.info("Installing k3sup..."); + await executeCommand( + ["sh", "-c", "curl -sLS https://get.k3sup.dev | sh"], + "install k3sup" + ); + log.success("k3sup installed"); + } else { + log.skip("k3sup already installed"); + } + + // Step 5: Wait for VMs to be ready + const firstMasterIP = getIp(1); + log.info(`Waiting for first master node (${firstMasterIP}) to be ready...`); + + const vmReady = await isVmReadyForSsh(firstMasterIP, "picard"); + if (!vmReady) { + throw new Error(`First master node at ${firstMasterIP} is not ready for SSH connections`); + } + + // Check if kubeconfig exists + const kubeconfigExists = await exists("./kubeconfig"); + + if (!kubeconfigExists) { + log.info("Bootstrapping first master node..."); + await executeCommand( + ["k3sup", "install", "--ip", firstMasterIP, "--user", "picard", "--cluster", "--ssh-key", sshKeyPrivateFileName], + "bootstrap first master node" + ); + log.success("First master node bootstrapped"); + } else { + log.skip("First master node already bootstrapped"); + } + + // Step 6: Join Additional Master Nodes (if not already joined) + for (let i = 2; i <= numMasters; i++) { + const vmName = `k3s-master${i}`; + const vmIP = getIp(i); + + // Wait for VM to be ready + log.info(`Waiting for ${vmName} (${vmIP}) to be ready...`); + const nodeReady = await isVmReadyForSsh(vmIP, "picard"); + if (!nodeReady) { + log.warning(`VM ${vmName} is not ready for SSH connections, skipping join operation`); + continue; + } + + const { success: joined } = await executeCommand( + ["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", vmName], + `check if ${vmName} has joined the cluster`, + { stdout: "null", stderr: "null", throwOnError: false } + ); + + if (!joined) { + log.info(`Joining ${vmName} to the cluster...`); + await executeCommand( + ["k3sup", "join", "--server", "--ip", vmIP, "--server-ip", firstMasterIP, "--user", "picard", "--ssh-key", sshKeyPrivateFileName], + `join ${vmName} to the cluster` + ); + log.success(`VM ${vmName} joined the cluster`); + } else { + log.skip(`VM ${vmName} already joined the cluster`); + } + } + + log.success("HA k3s cluster setup complete! 🚀"); + + // Verify cluster status + log.info("Verifying cluster status..."); + const { success: clusterVerified, output: nodesOutput } = await executeCommand( + ["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", "-o", "wide"], + "verify cluster nodes", + { throwOnError: false } + ); + + if (clusterVerified) { + log.info("Cluster nodes:"); + console.log(nodesOutput); + } else { + log.warning("Could not verify cluster status"); + } + + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + log.error(`Failed to set up cluster: ${errorMessage}`); + + if (createdVMs.length > 0) { + log.warning("An error occurred during setup. Cleaning up created resources..."); + await cleanup(createdVMs, forceCleanup); + } + + Deno.exit(1); } - - console.log("🚀 HA k3s cluster setup complete!"); }; await new Command() .name("setup-k3s-cluster") .version("0.1.0") .description("Automate the setup of an HA k3s cluster using incus and k3sup") - .option("-m, --masters ", "Number of master nodes", {default: 3}) - .action(({masters}) => setupCluster(masters)) + .option("-m, --masters ", "Number of master nodes", { default: 3 }) + .option("-c, --cleanup", "Force cleanup of VMs if setup fails", { default: false }) + .action(({ masters, cleanup }) => setupCluster(masters, cleanup)) .parse(Deno.args); diff --git a/shuttles/terraform/.terraform.lock.hcl b/shuttles/terraform/.terraform.lock.hcl deleted file mode 100644 index 4b5692f..0000000 --- a/shuttles/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,143 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/argoproj-labs/argocd" { - version = "7.0.2" - constraints = "7.0.2" - hashes = [ - "h1:4lbS20EczuzhSNSOjp1mJoe2YbcXniBTzxmJHd+rjIE=", - "zh:083686eaeaa7b51ebaac42c3c7b01a15f020a735dc8dbe50aa6a6bff16888943", - "zh:16b1b813f33874844fadc747c57ae99cf8f119c119b3776a105c154fc4a54488", - "zh:25ed8dca5da5faa52392c7938c61dd9a83bc6388ad771062cecfc15c44bc3d8e", - "zh:3907351bbcb6a0c1c1abeb33dac5d70f798b0ecc05559f2ede40ae84b9079983", - "zh:3a737237f03b9b28de26b1fe9d20bcfa53f580489fc28d774396e5de38906fd3", - "zh:64421961cc342cec8280899352444a96ad1b09144fa933dc3a0dfb9bbae809a9", - "zh:9702119789cc42b98dc9d1a8d7666b608a964cf1355e3cf500b82bed1898f2fd", - "zh:9cc9ad41a6ce25aac40b9dd2291fc4d90a223add197155decdca7d2d82fc60f1", - "zh:a239381a36bf6041d6520c8db83fb281fd2417f4540c895e07db052dd108a72f", - "zh:ecca66064fff07719eec2ef35cd62d1cb65cf4a11f9ce96f3a9b9b7c78d614a5", - ] -} - -provider "registry.terraform.io/hashicorp/helm" { - version = "2.17.0" - hashes = [ - "h1:K5FEjxvDnxb1JF1kG1xr8J3pNGxoaR3Z0IBG9Csm/Is=", - "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4", - "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7", - "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3", - "zh:3956187ec239f4045975b35e8c30741f701aa494c386aaa04ebabffe7749f81c", - "zh:66a9686d92a6b3ec43de3ca3fde60ef3d89fb76259ed3313ca4eb9bb8c13b7dd", - "zh:88644260090aa621e7e8083585c468c8dd5e09a3c01a432fb05da5c4623af940", - "zh:a248f650d174a883b32c5b94f9e725f4057e623b00f171936dcdcc840fad0b3e", - "zh:aa498c1f1ab93be5c8fbf6d48af51dc6ef0f10b2ea88d67bcb9f02d1d80d3930", - "zh:bf01e0f2ec2468c53596e027d376532a2d30feb72b0b5b810334d043109ae32f", - "zh:c46fa84cc8388e5ca87eb575a534ebcf68819c5a5724142998b487cb11246654", - "zh:d0c0f15ffc115c0965cbfe5c81f18c2e114113e7a1e6829f6bfd879ce5744fbb", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.31.0" - constraints = "2.31.0" - hashes = [ - "h1:wGHbATbv/pBVTST1MtEn0zyVhZbzZJD2NYq2EddASHY=", - "zh:0d16b861edb2c021b3e9d759b8911ce4cf6d531320e5dc9457e2ea64d8c54ecd", - "zh:1bad69ed535a5f32dec70561eb481c432273b81045d788eb8b37f2e4a322cc40", - "zh:43c58e3912fcd5bb346b5cb89f31061508a9be3ca7dd4cd8169c066203bcdfb3", - "zh:4778123da9206918a92dfa73cc711475d2b9a8275ff25c13a30513c523ac9660", - "zh:8bfa67d2db03b3bfae62beebe6fb961aee8d91b7a766efdfe4d337b33dfd23dd", - "zh:9020bb5729db59a520ade5e24984b737e65f8b81751fbbd343926f6d44d22176", - "zh:90431dbfc5b92498bfbce38f0b989978c84421a6c33245b97788a46b563fbd6e", - "zh:b71a061dda1244f6a52500e703a9524b851e7b11bbf238c17bbd282f27d51cb2", - "zh:d6232a7651b834b89591b94bf4446050119dcde740247e6083a4d55a2cefd28a", - "zh:d89fba43e699e28e2b5e92fff2f75fc03dbc8de0df9dacefe1a8836f8f430753", - "zh:ef85c0b744f5ba1b10dadc3c11e331ba4225c45bb733e024d7218c24b02b0512", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.5.2" - hashes = [ - "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=", - "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", - "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", - "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", - "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", - "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", - "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", - "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", - "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", - "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", - "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.3" - hashes = [ - "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", - "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", - "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", - "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", - "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", - "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", - "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", - "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", - "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", - "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", - "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", - ] -} - -provider "registry.terraform.io/public-cloud-wl/slugify" { - version = "0.1.1" - constraints = "0.1.1" - hashes = [ - "h1:iOJEMYX1bLfUnKjSxluQkKijr5NgWSqb2lU9Ag2Q12w=", - "zh:13f77dedcc74256053ac51512372510d722116bf58e119fac203fe599d667720", - "zh:2223be634f684f76e265efdaafdf95a948ba9e44f09f8a89540bdb564eff17f1", - "zh:73e8b763c796d57186756cf0bab75323e2d92c873f1df8eccd8a7e336a2e3e81", - "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", - "zh:9f83adcf17de03afb5c27111cb26c580dc5296dffd40fca4571e81ad0bad3bad", - "zh:a5414ade8cbae9aea10dee79e43da247ceecb7e4a54e76d39906ee60b7365a7d", - "zh:bd118ead731e129c92c0dfe3c9a2ebbd8fa25ba6508deaaaccb9ac3a7f70af2d", - "zh:c8ce48ad921956edcee0643cb6184442f3deb438e5605a53794dfd6e8f89a559", - "zh:d96da8a32ef2b807ed3bd943294c6e1d0bd5fc3a793deb762f74d0c54aeff335", - "zh:e30a218b474afe082e005faf51c323ed8747d46845bfacab4cd3adc0c51704ec", - "zh:e3cd265c38da6e65974ac1b9b6be608ba0534178f16f059ad13672de6846e32e", - "zh:f2ded7f8c771a603ad3e2df84986b5f175c38049b7a9ab4a3cd384abafb33dff", - "zh:f2ece1996cf686583afd19384041204a32e08389dc6f4f105501584e653e797d", - "zh:fa2418b74cea55d29dad24f5095aaf30d6253d63ebac3c0c47949b3de8087c88", - "zh:fdc8d3fbca6a19db203802e7a7337075e39b9ffb7a3887a7583e379be61bde17", - ] -} - -provider "registry.terraform.io/zitadel/zitadel" { - version = "2.0.2" - constraints = "2.0.2" - hashes = [ - "h1:iymeaNBrZ4smcr7eHrxO4gbXQ6bx/enKyj3RQ6xZRYA=", - "zh:01e16af0dda9372696b5e1d43ec709aed79829b49ee69a4f9606a248752f672d", - "zh:048c4e726fb846cfe9ab0a0a1f86d3f8922442154b086e2bd8e389b32f69f2f0", - "zh:3a3f6bea621c9d480f1f288cffebace8620979b9260cfeae8f9af5d9a25ed490", - "zh:4d349e584786589bc2037cee691ff1678296f5351e6491aa34dcb08ecbe1dcb7", - "zh:80741c78179788be8d7e33e471e1311197cd4e1067803d438463d0a8ac871a60", - "zh:89178d30f5ec49551e6a6ebc5eb589ab6631012dcec0d03ea7130b1029890e51", - "zh:94cd3b1fe3d1d39bcb3b70208b044bde4c5ce5152e12b29f0fa0ff1085e12863", - "zh:97299c172ada852705f8ca9fa91eeee12c6259263baae3ca53cf41e3130b1731", - "zh:a33d53acc640dc93b81352ba633cf392bc8c7614a72d320d59d3dcdb22d73fc4", - "zh:a95c15960baf8157f79a6490361455767d48e4dd3ce2ef1d0051743f6152733b", - "zh:ae66ad95c7039e6ef844c39389c9077ce7dbb501b6af02afb26a223fd289dbcb", - "zh:b8a9cb3b53653c06d52607368c406112ee1abc6d66dc4aedaedddbb46a66ea8f", - "zh:d48693ecdc985bb4167af0c3164240c13c4ea48167d28f706e7893cbdb20540a", - "zh:f6db1ec30bfbcf4423ab2d29979b775423ba37008fd48a766b5a1cf87a131859", - "zh:fed4e95dc9aaf361c8ff57f819d31fa25152b9e6cb90b7202d8be9ab1446b081", - ] -} diff --git a/shuttles/terraform/zitadel-admin-sa.json b/shuttles/terraform/zitadel-admin-sa.json index 129b06a..a05b37c 100755 --- a/shuttles/terraform/zitadel-admin-sa.json +++ b/shuttles/terraform/zitadel-admin-sa.json @@ -1 +1 @@ -{"type":"serviceaccount","keyId":"308542807332225577","key":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzjNVWZ/vYu2AyrYpDa36tOc7tdRfnTXxUUpUX4Yp8uVnZe2M\nlRJcMl+B4WXiVINhb2cJqKKHp+zzYiUJBCtLACXHWEPBsuNk+mmP+cot5q8lf9VY\nY7jKiUlKSFhuau0bkXTuxsest8Tau9kwt8z0f0N8Rh4Tsqg8uAMjxIfVg+NYOPrV\nVkyCZRowuquF7xyS+qm15HDIhao5y7QcuqSSr/V0DC46iVk/SiSAMSBCKwmgFL5e\naG/dcaT1euzxBxJX/ViAPiOL6rX1v3EjHm9HPhdQd7UuYiueqKPd8nnqBElTG8pV\n3JXcP00Rx8g89IGnYJFlaUrzOkatKBwonzVp0wIDAQABAoIBADr4tWUvd3AsipPu\n6ujNpBDOvOXwufOlNMHR5zV2klK0RxEAkh8kJBgH9oY29DbFaD1LE5kV+oQNIUIl\nY2G82/NL2qoknqYBoKR5QkLnDp+V4bygkGnctJf0zPjJybJs42CDN7LD8mKQOxpU\nMDmwtRAchdMr0OhccxkRVM5lJqxT+ixmwnD9hhhZQsfvrj4j9ALFTDYhdteJpC3t\n4KUMBmlV6PWwXw3gAstlZTXkHlp/BKrY407fOs9GYm5yAz0yoUgoSRawepQWX4Vp\n4qp8l5oI7veGIBE8H0Om9P8O7JoVWzH97iO3nHe0BYekInQgn60C0SshVaDVk8zc\nJ6UsmtkCgYEA7f5H2WI5yG4yO8IxRbx86S0xME+pjkw7GABLl3Ug8N7AKawh3BoU\n2ve5kPo6TA9l3MuUOqYRDAGTzvdhhgEC/V8oaYXKT/73CIcUgoF3cI642Ugo/UIo\n7LqaGy5qaoy2LqwQp+WVEefQ0qImz1c6sVXEBo1kPThxohbaCkYs/x8CgYEA3c1B\nSyX+xTGF8TY8EzyT+DguDmfOMZCb+fxw96J23m2svj7CGQvjM/ctQiGBMyOcdA4Z\nFAOtezuFNczJ6OIc3oiyBJ9/Anc9czZZNjH6ZDuCyIdzuLiWFwvZ+Yk1Gq5w9xJv\nvqIJkfuVsGvFbjY1qJcYTFo/nfPNW8hkIoJVIs0CgYEA0Czy0DXZcXbivd75lpRv\nds+vDSFBoVURA8eOV6d+7vMJh+onnA28XGUAjs4ynEGDyoTQ3hRRKP1gO1OsnLjZ\n0qOgB92dwCe4El+GEzoILg7JplY2dIGgpqH7Fvec4iK/YUflMdfic00hHn30EL2u\ne1wYIdsf6WUsEKqIgyArT+UCgYBMzhS8Fw4f7sQ5ANTQvHtoytt39Y47L54zdK0R\ns1qCL2xP/J9t2OX2SrOLYNrCgKRes7sPaS6bq8K3HJEWaaYhkShD3Y83pV7MFJfT\n4n6YUts44V67KoAevuXeORsAgENx6xpy4t15hasSCl+1iGQQWzH4zo+U/KWyELRM\nFuGwSQKBgFVWScjsWJnWyoZua/+F/r6FvPp8Bknm7N66Wl3Rb6h1q0CoQHnMx3HR\nPvJ2A6caaOt8QiBQDgu+23DaGBjRe8RxDkc4tVTDUWv4RCqub5gELkRW2DYtESdg\nvhMnXkYVubBZj8IRcYPyNddjV4SzlK67RIL/bMrYisAz+PHRYZFU\n-----END RSA PRIVATE KEY-----\n","expirationDate":"2026-01-01T00:00:00Z","userId":"308542807332160041"} +{"type":"serviceaccount","keyId":"310142761184133898","key":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApSaCjkOBVIe33bEIwENq1jGj6MgbN+NqYRN6EVDWVnESM10/\n188hB9UDCvNR5kcBbaz2bD/ymZ/ppWSLqmXONwc3/PtiOluBfkvR1q2pEh+f13wz\n72dbhOVqf+YhL3lndiQ/OupGzaCbmsBNPGag7mgaPxlgoWTWIItPnOomIhwtwXgy\nNyzt9Fmyh/4JsRlIYO90ZO32vKXABRMCGsKxvcY9CR4+LIqddns83YASGFnQ5oBo\nObc8EN2Di7uKWzNwxUJuZtFlHXp06su2mWDGJhKusHYW4KUIs2uwFtjJfAXG/adT\n8qVgi174m1jU2ocSd6o9IqDYf50arCinbgtAdwIDAQABAoIBABwrB1WQefya8Wdk\njKOOXCiQau6HQu0zYq+QDN/rM8OmoX4VR5Bdibq2QECb47otHjdAqv8noQ9G0Ske\njxvPJW8JUilaDxT5CosqD25YTGAE+NReINWSgW+XWaTa8YoRYO4rnIVF9DGaVS/9\n4K6OqqA/LUrZ3ztn4YXHfRq8bSif86GMo1GkwH8xOMJHdaxCs8YzAbpGURL03QtL\nemVNs9VwSWLmnK71FpXkko0aGi14naS7E4jv8uutykLQsc+QE7m9B4OiDkijKCP9\nQwvw/3RZYcrRuWz7uSANyxG4Uc8JhPdUIyvpkvUz8NfRLTDoSAEq1NQuxpyjLYYU\n7uzYcWECgYEAzKZ5wGTJBZafen2I61L8XAMk2df63nnEK+YuZqNZ6yH6IY7cCrlJ\n3LbeNoHNcGMXw1mf9Z9vvAjz7nbec2BYN1KRMR9QOTHcqwQZcOOJnwhdO4uAlsFZ\ngiyoLYCQP8Z6IIC4ht+2hmf8hS3CmWUPAXyLOcg4ok6SRdyNsfWiLwkCgYEAzpbL\n8szYqNY+r5n1DQ9d6zNb2cbkFfzZDxn64BA1xQZtRgxfzNAOvsGl5pPWve7oS/8Y\nmPx+1b08NvCcTuaow7CCw+IDHsI43TRNbvPQBWtINBE6eeBs3laaNvmxTZU5HGog\nt1yRtk0u64hKT7+L7Ku5JP79pxzNOIs1hnImU38CgYAaH84+/x6iNf4Ztti5oZhR\nbp1PqcB+kfC24eVeeM/LskSp8ACq5chGApoPPzaoeB3adCB1TGsJB+OLt2TiOZRJ\nS6L5MFQfWPwgYJ+Wx5UT1g+AwGgj1n7EnUrCtDy1x3Jjn8rufLRiJ/gWUCcdScdG\nm01yjNqd7YXCoUr9Qqv3cQKBgGd2klHZUbDNC7v6SQXvakP/BsM8nsJ8TWEIy+In\nfCZen59zVw9GK/xRE3s1E1kwK1rUOUd1PThie6OwQTgqwN6wqezcZl+jOcNfDGDC\n7q2oGxMohbbANQXtLXLW/nsyftXCOPxb+gXpBdSj/0ONVNCE+EaVBggJnqXw4i+h\nP5yVAoGBAIoXRgX3mSBsC/xgKIXQb4c9WT7W78IOpU43mbX9jC/emfLkOvuxR/Cv\nmJDgTv2zUq7uItbvXmxwmU7JVYlBFaWERsAqzzWUUsdfM3tBFdBbcH9fzoEG0j4u\nkqCwU1if6HTHCmunqt1ZQKN3oP1Uycn/1ZL6NR8ilqIcjCzh4JPQ\n-----END RSA PRIVATE KEY-----\n","expirationDate":"2026-01-01T00:00:00Z","userId":"310142761184068362"}