#!/usr/bin/env -S deno run --allow-run --allow-read --allow-write // Note: TypeScript errors related to Deno imports and namespace can be safely ignored // These are only relevant when running the script with the Deno runtime import { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts"; import { delay } from "https://deno.land/std/async/mod.ts"; import { exists } from "https://deno.land/std/fs/mod.ts"; // Configuration constants const alpineImage = "alpine/edge/cloud"; const alpineConfig = ['--profile', 'cloud-init-alpine']; const archImage = "archlinux/current/cloud"; const archConfig = ['--profile', 'cloud-init-arch']; const getIp = (i: number) => `10.110.36.${109 + i}`; const image = archImage; const config = archConfig; // Enhanced logging function with timestamps and log levels const log = { debug: (message: string) => console.log(`[${new Date().toISOString()}] [DEBUG] ${message}`), info: (message: string) => console.log(`[${new Date().toISOString()}] [INFO] ${message}`), success: (message: string) => console.log(`[${new Date().toISOString()}] [SUCCESS] ✅ ${message}`), warning: (message: string) => console.log(`[${new Date().toISOString()}] [WARNING] ⚠️ ${message}`), error: (message: string) => console.error(`[${new Date().toISOString()}] [ERROR] ❌ ${message}`), skip: (message: string) => console.log(`[${new Date().toISOString()}] [SKIP] ⏩ ${message}`), }; // Helper function to execute commands with proper error handling async function executeCommand( cmdArray: string[], description: string, options: { stdout?: "piped" | "inherit" | "null", stderr?: "piped" | "inherit" | "null", throwOnError?: boolean } = {stdout: 'piped', stderr: 'piped', throwOnError: true} ): Promise<{ success: boolean; output?: string; error?: string }> { const {stdout = "piped", stderr = "piped", throwOnError = true} = options; log.debug(`Executing: ${cmdArray.join(" ")}`); try { // Use Deno.Command API which is the modern replacement for Deno.run const command = new Deno.Command(cmdArray[0], { args: cmdArray.slice(1), stdout: stdout === "piped" ? "piped" : stdout === "inherit" ? "inherit" : "null", stderr: stderr === "piped" ? "piped" : stderr === "inherit" ? "inherit" : "null", }); const {code, stdout: stdoutOutput, stderr: stderrOutput} = await command.output(); const stdoutText = stdout === "piped" ? new TextDecoder().decode(stdoutOutput).trim() : ""; const stderrText = stderr === "piped" ? new TextDecoder().decode(stderrOutput).trim() : ""; if (code !== 0) { log.error(`Failed to ${description}: ${stderrText || "Unknown error"}`); if (throwOnError) { throw new Error(`Command failed: ${cmdArray.join(" ")}\n${stderrText}`); } } return { success: code === 0, output: stdoutText, error: stderrText }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log.error(`Exception while ${description}: ${errorMessage}`); if (throwOnError) { throw error; } return {success: false, error: errorMessage}; } } // Check if VM is ready for SSH connections async function isVmReadyForSsh(ip: string, user: string, sshKeyPath: string, maxAttempts = 30): Promise { log.info(`Checking if VM at ${ip} is ready for SSH connections...`); for (let attempt = 1; attempt <= maxAttempts; attempt++) { log.debug(`SSH readiness check attempt ${attempt}/${maxAttempts}`); const {success} = await executeCommand( ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=5", `${user}@${ip}`, "-i", sshKeyPath, "echo", "ready"], `check SSH connectivity to ${ip}`, {throwOnError: false} ); if (success) { log.success(`VM at ${ip} is ready for SSH connections`); return true; } log.debug(`VM at ${ip} not ready yet, waiting...`); await delay(2000); // Wait 2 seconds between attempts } log.error(`VM at ${ip} is not ready for SSH connections after ${maxAttempts} attempts`); return false; } // Check if VM is running async function isVmRunning(vmName: string): Promise { const {success, output} = await executeCommand( ["incus", "list", vmName, "--format", "json"], `check if VM ${vmName} is running`, {throwOnError: false} ); if (!success || !output) { return false; } try { const vmInfo = JSON.parse(output); return vmInfo.length > 0 && vmInfo[0].status === "Running"; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log.error(`Failed to parse VM status: ${errorMessage}`); return false; } } // Cleanup function to handle failures async function cleanup(vmNames: string[], shouldRemove = false): Promise { log.info("Starting cleanup process..."); return; for (const vmName of vmNames) { // Check if VM exists const {success, output} = await executeCommand( ["incus", "list", vmName, "--format", "csv"], `check if VM ${vmName} exists`, {throwOnError: false} ); if (success && output) { // Stop VM if it's running const isRunning = await isVmRunning(vmName); if (isRunning) { log.info(`Stopping VM ${vmName}...`); await executeCommand( ["incus", "stop", vmName, "--force"], `stop VM ${vmName}`, {throwOnError: false} ); } // Remove VM if requested if (shouldRemove) { log.info(`Removing VM ${vmName}...`); await executeCommand( ["incus", "delete", vmName], `remove VM ${vmName}`, {throwOnError: false} ); } } } log.success("Cleanup completed"); } const setupCluster = async (numMasters: number, forceCleanup = false) => { log.info(`Starting setup of k3s cluster with ${numMasters} master nodes`); const createdVMs: string[] = []; try { // Get hostname and user const {output: hostname} = await executeCommand( ["hostnamectl", "hostname"], "get hostname" ); const {output: user} = await executeCommand( ["whoami"], "get current user" ); const sshKeyPubFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}.pub`; const sshKeyPrivateFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}`; // Check if SSH keys exist if (!await exists(sshKeyPubFileName) || !await exists(sshKeyPrivateFileName)) { log.error(`Required SSH keys not found: ${sshKeyPubFileName} or ${sshKeyPrivateFileName}`); throw new Error("SSH keys not found"); } // Step 1: Create Low-Resource Profile (if not exists) const {success: profileExists} = await executeCommand( ["incus", "profile", "show", "low-resource"], "check if low-resource profile exists", {throwOnError: false} ); if (!profileExists) { log.info("Creating low-resource profile..."); await executeCommand( ["incus", "profile", "create", "low-resource"], "create low-resource profile" ); await executeCommand( ["incus", "profile", "set", "low-resource", "limits.cpu=1", "limits.memory=512MB"], "set low-resource profile limits" ); await executeCommand( ["incus", "profile", "device", "add", "low-resource", "root", "disk", "pool=default", "path=/"], "add root disk to low-resource profile" ); // await executeCommand( // ["incus", "profile", "device", "add", "low-resource", "eth-0", "nic", "network=incusbr0"], // "add network interface to low-resource profile" // ); log.success("Low-resource profile created"); } else { log.skip("Low-resource profile already exists"); } // Read SSH key const sshKey = await Deno.readTextFile(sshKeyPubFileName); // Step 3: Launch VMs (if not already running) for (let i = 1; i <= numMasters; i++) { const vmName = `k3s-master${i}`; const {success: vmExists, output: vmOutput} = await executeCommand( ["incus", "list", vmName, "--format", "csv"], `check if VM ${vmName} exists`, {throwOnError: false} ); if (!vmExists || !vmOutput) { log.info(`Creating VM ${vmName}...`); await executeCommand( ["incus", "init", `images:${image}`, vmName, "--profile", "low-resource", "-c", "user.timezone=\"Europe/Amsterdam\"", "-c", `user.ssh_key=\"${sshKey}\"`, ...config], `initialize VM ${vmName}` ); await executeCommand( ["incus", "config", 'device', 'add', vmName, 'eth0', 'nic', 'nictype=bridged', 'parent=incusbr0', `ipv4.address=${getIp(i)}`], `configure network for VM ${vmName}` ); await executeCommand( ["incus", "start", vmName], `start VM ${vmName}` ); createdVMs.push(vmName); log.success(`VM ${vmName} started`); } else { // Check if VM is running, if not, start it const isRunning = await isVmRunning(vmName); if (!isRunning) { log.info(`Starting existing VM ${vmName}...`); await executeCommand( ["incus", "start", vmName], `start VM ${vmName}` ); } log.skip(`VM ${vmName} already exists`); } } // Step 4: Install k3sup (if not installed) const {success: k3supInstalled} = await executeCommand( ["which", "k3sup"], "check if k3sup is installed", {throwOnError: false} ); if (!k3supInstalled) { log.info("Installing k3sup..."); await executeCommand( ["sh", "-c", "curl -sLS https://get.k3sup.dev | sh"], "install k3sup" ); log.success("k3sup installed"); } else { log.skip("k3sup already installed"); } // Step 5: Wait for VMs to be ready const firstMasterIP = getIp(1); log.info(`Waiting for first master node (${firstMasterIP}) to be ready...`); const vmReady = await isVmReadyForSsh(firstMasterIP, "picard", sshKeyPrivateFileName); if (!vmReady) { throw new Error(`First master node at ${firstMasterIP} is not ready for SSH connections`); } // Check if kubeconfig exists const kubeconfigExists = await exists("./kubeconfig"); if (!kubeconfigExists) { log.info("Bootstrapping first master node..."); await executeCommand( ["k3sup", "install", "--ip", firstMasterIP, "--user", "picard", "--cluster", "--ssh-key", sshKeyPrivateFileName], "bootstrap first master node" ); log.success("First master node bootstrapped"); } else { log.skip("First master node already bootstrapped"); } // Step 6: Join Additional Master Nodes (if not already joined) for (let i = 2; i <= numMasters; i++) { const vmName = `k3s-master${i}`; const vmIP = getIp(i); // Wait for VM to be ready log.info(`Waiting for ${vmName} (${vmIP}) to be ready...`); const nodeReady = await isVmReadyForSsh(vmIP, "picard", sshKeyPrivateFileName); if (!nodeReady) { log.warning(`VM ${vmName} is not ready for SSH connections, skipping join operation`); continue; } const {success: joined} = await executeCommand( ["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", vmName], `check if ${vmName} has joined the cluster`, {throwOnError: false} ); if (!joined) { log.info(`Joining ${vmName} to the cluster...`); await executeCommand( ["k3sup", "join", "--server", "--ip", vmIP, "--server-ip", firstMasterIP, "--user", "picard", "--ssh-key", sshKeyPrivateFileName], `join ${vmName} to the cluster` ); log.success(`VM ${vmName} joined the cluster`); } else { log.skip(`VM ${vmName} already joined the cluster`); } } log.success("HA k3s cluster setup complete! 🚀"); // Verify cluster status log.info("Verifying cluster status..."); const {success: clusterVerified, output: nodesOutput} = await executeCommand( ["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", "-o", "wide"], "verify cluster nodes", {throwOnError: false} ); if (clusterVerified) { log.info("Cluster nodes:"); console.log(nodesOutput); } else { log.warning("Could not verify cluster status"); } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); log.error(`Failed to set up cluster: ${errorMessage}`); if (createdVMs.length > 0) { log.warning("An error occurred during setup. Cleaning up created resources..."); await cleanup(createdVMs, forceCleanup); } Deno.exit(1); } }; await new Command() .name("setup-k3s-cluster") .version("0.1.0") .description("Automate the setup of an HA k3s cluster using incus and k3sup") .option("-m, --masters ", "Number of master nodes", {default: 3}) .option("-c, --cleanup", "Force cleanup of VMs if setup fails", {default: false}) .action(({masters, cleanup}) => setupCluster(masters, cleanup)) .parse(Deno.args);