378 lines
13 KiB
TypeScript
Executable File
378 lines
13 KiB
TypeScript
Executable File
#!/usr/bin/env -S deno run --allow-run --allow-read --allow-write
|
|
|
|
// Note: TypeScript errors related to Deno imports and namespace can be safely ignored
|
|
// These are only relevant when running the script with the Deno runtime
|
|
import { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.4/command/mod.ts";
|
|
import { delay } from "https://deno.land/std/async/mod.ts";
|
|
import { exists } from "https://deno.land/std/fs/mod.ts";
|
|
|
|
// Configuration constants
|
|
const alpineImage = "alpine/edge/cloud";
|
|
const alpineConfig = ['--profile', 'cloud-init-alpine'];
|
|
const archImage = "archlinux/current/cloud";
|
|
const archConfig = ['--profile', 'cloud-init-arch'];
|
|
|
|
const getIp = (i: number) => `10.110.36.${109 + i}`;
|
|
|
|
const image = archImage;
|
|
const config = archConfig;
|
|
|
|
// Enhanced logging function with timestamps and log levels
|
|
const log = {
|
|
debug: (message: string) => console.log(`[${new Date().toISOString()}] [DEBUG] ${message}`),
|
|
info: (message: string) => console.log(`[${new Date().toISOString()}] [INFO] ${message}`),
|
|
success: (message: string) => console.log(`[${new Date().toISOString()}] [SUCCESS] ✅ ${message}`),
|
|
warning: (message: string) => console.log(`[${new Date().toISOString()}] [WARNING] ⚠️ ${message}`),
|
|
error: (message: string) => console.error(`[${new Date().toISOString()}] [ERROR] ❌ ${message}`),
|
|
skip: (message: string) => console.log(`[${new Date().toISOString()}] [SKIP] ⏩ ${message}`),
|
|
};
|
|
|
|
// Helper function to execute commands with proper error handling
|
|
async function executeCommand(
|
|
cmdArray: string[],
|
|
description: string,
|
|
options: {
|
|
stdout?: "piped" | "inherit" | "null",
|
|
stderr?: "piped" | "inherit" | "null",
|
|
throwOnError?: boolean
|
|
} = {}
|
|
): Promise<{ success: boolean; output?: string; error?: string }> {
|
|
const { stdout = "piped", stderr = "piped", throwOnError = true } = options;
|
|
|
|
log.debug(`Executing: ${cmdArray.join(" ")}`);
|
|
|
|
try {
|
|
// Use Deno.Command API which is the modern replacement for Deno.run
|
|
const command = new Deno.Command(cmdArray[0], {
|
|
args: cmdArray.slice(1),
|
|
stdout: stdout === "piped" ? "piped" : stdout === "inherit" ? "inherit" : "null",
|
|
stderr: stderr === "piped" ? "piped" : stderr === "inherit" ? "inherit" : "null",
|
|
});
|
|
|
|
const { code, stdout: stdoutOutput, stderr: stderrOutput } = await command.output();
|
|
|
|
const stdoutText = stdout === "piped" ? new TextDecoder().decode(stdoutOutput).trim() : "";
|
|
const stderrText = stderr === "piped" ? new TextDecoder().decode(stderrOutput).trim() : "";
|
|
|
|
if (code !== 0) {
|
|
log.error(`Failed to ${description}: ${stderrText || "Unknown error"}`);
|
|
if (throwOnError) {
|
|
throw new Error(`Command failed: ${cmdArray.join(" ")}\n${stderrText}`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
success: code === 0,
|
|
output: stdoutText,
|
|
error: stderrText
|
|
};
|
|
} catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
log.error(`Exception while ${description}: ${errorMessage}`);
|
|
if (throwOnError) {
|
|
throw error;
|
|
}
|
|
return { success: false, error: errorMessage };
|
|
}
|
|
}
|
|
|
|
// Check if VM is ready for SSH connections
|
|
async function isVmReadyForSsh(ip: string, user: string, maxAttempts = 30): Promise<boolean> {
|
|
log.info(`Checking if VM at ${ip} is ready for SSH connections...`);
|
|
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
log.debug(`SSH readiness check attempt ${attempt}/${maxAttempts}`);
|
|
|
|
const { success } = await executeCommand(
|
|
["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=5", `${user}@${ip}`, "echo", "ready"],
|
|
`check SSH connectivity to ${ip}`,
|
|
{ throwOnError: false, stderr: "null" }
|
|
);
|
|
|
|
if (success) {
|
|
log.success(`VM at ${ip} is ready for SSH connections`);
|
|
return true;
|
|
}
|
|
|
|
log.debug(`VM at ${ip} not ready yet, waiting...`);
|
|
await delay(2000); // Wait 2 seconds between attempts
|
|
}
|
|
|
|
log.error(`VM at ${ip} is not ready for SSH connections after ${maxAttempts} attempts`);
|
|
return false;
|
|
}
|
|
|
|
// Check if VM is running
|
|
async function isVmRunning(vmName: string): Promise<boolean> {
|
|
const { success, output } = await executeCommand(
|
|
["incus", "list", vmName, "--format", "json"],
|
|
`check if VM ${vmName} is running`,
|
|
{ throwOnError: false }
|
|
);
|
|
|
|
if (!success || !output) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
const vmInfo = JSON.parse(output);
|
|
return vmInfo.length > 0 && vmInfo[0].status === "Running";
|
|
} catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
log.error(`Failed to parse VM status: ${errorMessage}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Cleanup function to handle failures
|
|
async function cleanup(vmNames: string[], shouldRemove = false): Promise<void> {
|
|
log.info("Starting cleanup process...");
|
|
|
|
for (const vmName of vmNames) {
|
|
// Check if VM exists
|
|
const { success, output } = await executeCommand(
|
|
["incus", "list", vmName, "--format", "csv"],
|
|
`check if VM ${vmName} exists`,
|
|
{ throwOnError: false }
|
|
);
|
|
|
|
if (success && output) {
|
|
// Stop VM if it's running
|
|
const isRunning = await isVmRunning(vmName);
|
|
if (isRunning) {
|
|
log.info(`Stopping VM ${vmName}...`);
|
|
await executeCommand(
|
|
["incus", "stop", vmName, "--force"],
|
|
`stop VM ${vmName}`,
|
|
{ throwOnError: false }
|
|
);
|
|
}
|
|
|
|
// Remove VM if requested
|
|
if (shouldRemove) {
|
|
log.info(`Removing VM ${vmName}...`);
|
|
await executeCommand(
|
|
["incus", "delete", vmName],
|
|
`remove VM ${vmName}`,
|
|
{ throwOnError: false }
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
log.success("Cleanup completed");
|
|
}
|
|
|
|
const setupCluster = async (numMasters: number, forceCleanup = false) => {
|
|
log.info(`Starting setup of k3s cluster with ${numMasters} master nodes`);
|
|
|
|
const createdVMs: string[] = [];
|
|
|
|
try {
|
|
// Get hostname and user
|
|
const { output: hostname } = await executeCommand(
|
|
["hostnamectl", "hostname"],
|
|
"get hostname"
|
|
);
|
|
|
|
const { output: user } = await executeCommand(
|
|
["whoami"],
|
|
"get current user"
|
|
);
|
|
|
|
const sshKeyPubFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}.pub`;
|
|
const sshKeyPrivateFileName = `/home/${user}/.ssh/nl.fourlights.${hostname}`;
|
|
|
|
// Check if SSH keys exist
|
|
if (!await exists(sshKeyPubFileName) || !await exists(sshKeyPrivateFileName)) {
|
|
log.error(`Required SSH keys not found: ${sshKeyPubFileName} or ${sshKeyPrivateFileName}`);
|
|
throw new Error("SSH keys not found");
|
|
}
|
|
|
|
// Step 1: Create Low-Resource Profile (if not exists)
|
|
const { success: profileExists } = await executeCommand(
|
|
["incus", "profile", "show", "low-resource"],
|
|
"check if low-resource profile exists",
|
|
{ stdout: "null", stderr: "null", throwOnError: false }
|
|
);
|
|
|
|
if (!profileExists) {
|
|
log.info("Creating low-resource profile...");
|
|
await executeCommand(
|
|
["incus", "profile", "create", "low-resource"],
|
|
"create low-resource profile"
|
|
);
|
|
await executeCommand(
|
|
["incus", "profile", "set", "low-resource", "limits.cpu=1", "limits.memory=512MB"],
|
|
"set low-resource profile limits"
|
|
);
|
|
await executeCommand(
|
|
["incus", "profile", "device", "add", "low-resource", "root", "disk", "pool=default", "path=/"],
|
|
"add root disk to low-resource profile"
|
|
);
|
|
// await executeCommand(
|
|
// ["incus", "profile", "device", "add", "low-resource", "eth-0", "nic", "network=incusbr0"],
|
|
// "add network interface to low-resource profile"
|
|
// );
|
|
log.success("Low-resource profile created");
|
|
} else {
|
|
log.skip("Low-resource profile already exists");
|
|
}
|
|
|
|
// Read SSH key
|
|
const sshKey = await Deno.readTextFile(sshKeyPubFileName);
|
|
|
|
// Step 3: Launch VMs (if not already running)
|
|
for (let i = 1; i <= numMasters; i++) {
|
|
const vmName = `k3s-master${i}`;
|
|
|
|
const { success: vmExists, output: vmOutput } = await executeCommand(
|
|
["incus", "list", vmName, "--format", "csv"],
|
|
`check if VM ${vmName} exists`,
|
|
{ throwOnError: false }
|
|
);
|
|
|
|
if (!vmExists || !vmOutput) {
|
|
log.info(`Creating VM ${vmName}...`);
|
|
await executeCommand(
|
|
["incus", "init", `images:${image}`, vmName, "--profile", "low-resource", "-c", "user.timezone=\"Europe/Amsterdam\"", "-c", `user.ssh_key=\"${sshKey}\"`, ...config],
|
|
`initialize VM ${vmName}`
|
|
);
|
|
|
|
await executeCommand(
|
|
["incus", "config", 'device', 'add', vmName, 'eth0', 'nic', 'nictype=bridged', 'parent=incusbr0', `ipv4.address=${getIp(i)}`],
|
|
`configure network for VM ${vmName}`
|
|
);
|
|
|
|
await executeCommand(
|
|
["incus", "start", vmName],
|
|
`start VM ${vmName}`
|
|
);
|
|
|
|
createdVMs.push(vmName);
|
|
log.success(`VM ${vmName} started`);
|
|
} else {
|
|
// Check if VM is running, if not, start it
|
|
const isRunning = await isVmRunning(vmName);
|
|
if (!isRunning) {
|
|
log.info(`Starting existing VM ${vmName}...`);
|
|
await executeCommand(
|
|
["incus", "start", vmName],
|
|
`start VM ${vmName}`
|
|
);
|
|
}
|
|
log.skip(`VM ${vmName} already exists`);
|
|
}
|
|
}
|
|
|
|
// Step 4: Install k3sup (if not installed)
|
|
const { success: k3supInstalled } = await executeCommand(
|
|
["which", "k3sup"],
|
|
"check if k3sup is installed",
|
|
{ stdout: "null", stderr: "null", throwOnError: false }
|
|
);
|
|
|
|
if (!k3supInstalled) {
|
|
log.info("Installing k3sup...");
|
|
await executeCommand(
|
|
["sh", "-c", "curl -sLS https://get.k3sup.dev | sh"],
|
|
"install k3sup"
|
|
);
|
|
log.success("k3sup installed");
|
|
} else {
|
|
log.skip("k3sup already installed");
|
|
}
|
|
|
|
// Step 5: Wait for VMs to be ready
|
|
const firstMasterIP = getIp(1);
|
|
log.info(`Waiting for first master node (${firstMasterIP}) to be ready...`);
|
|
|
|
const vmReady = await isVmReadyForSsh(firstMasterIP, "picard");
|
|
if (!vmReady) {
|
|
throw new Error(`First master node at ${firstMasterIP} is not ready for SSH connections`);
|
|
}
|
|
|
|
// Check if kubeconfig exists
|
|
const kubeconfigExists = await exists("./kubeconfig");
|
|
|
|
if (!kubeconfigExists) {
|
|
log.info("Bootstrapping first master node...");
|
|
await executeCommand(
|
|
["k3sup", "install", "--ip", firstMasterIP, "--user", "picard", "--cluster", "--ssh-key", sshKeyPrivateFileName],
|
|
"bootstrap first master node"
|
|
);
|
|
log.success("First master node bootstrapped");
|
|
} else {
|
|
log.skip("First master node already bootstrapped");
|
|
}
|
|
|
|
// Step 6: Join Additional Master Nodes (if not already joined)
|
|
for (let i = 2; i <= numMasters; i++) {
|
|
const vmName = `k3s-master${i}`;
|
|
const vmIP = getIp(i);
|
|
|
|
// Wait for VM to be ready
|
|
log.info(`Waiting for ${vmName} (${vmIP}) to be ready...`);
|
|
const nodeReady = await isVmReadyForSsh(vmIP, "picard");
|
|
if (!nodeReady) {
|
|
log.warning(`VM ${vmName} is not ready for SSH connections, skipping join operation`);
|
|
continue;
|
|
}
|
|
|
|
const { success: joined } = await executeCommand(
|
|
["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", vmName],
|
|
`check if ${vmName} has joined the cluster`,
|
|
{ stdout: "null", stderr: "null", throwOnError: false }
|
|
);
|
|
|
|
if (!joined) {
|
|
log.info(`Joining ${vmName} to the cluster...`);
|
|
await executeCommand(
|
|
["k3sup", "join", "--server", "--ip", vmIP, "--server-ip", firstMasterIP, "--user", "picard", "--ssh-key", sshKeyPrivateFileName],
|
|
`join ${vmName} to the cluster`
|
|
);
|
|
log.success(`VM ${vmName} joined the cluster`);
|
|
} else {
|
|
log.skip(`VM ${vmName} already joined the cluster`);
|
|
}
|
|
}
|
|
|
|
log.success("HA k3s cluster setup complete! 🚀");
|
|
|
|
// Verify cluster status
|
|
log.info("Verifying cluster status...");
|
|
const { success: clusterVerified, output: nodesOutput } = await executeCommand(
|
|
["kubectl", "--kubeconfig=./kubeconfig", "get", "nodes", "-o", "wide"],
|
|
"verify cluster nodes",
|
|
{ throwOnError: false }
|
|
);
|
|
|
|
if (clusterVerified) {
|
|
log.info("Cluster nodes:");
|
|
console.log(nodesOutput);
|
|
} else {
|
|
log.warning("Could not verify cluster status");
|
|
}
|
|
|
|
} catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
log.error(`Failed to set up cluster: ${errorMessage}`);
|
|
|
|
if (createdVMs.length > 0) {
|
|
log.warning("An error occurred during setup. Cleaning up created resources...");
|
|
await cleanup(createdVMs, forceCleanup);
|
|
}
|
|
|
|
Deno.exit(1);
|
|
}
|
|
};
|
|
|
|
await new Command()
|
|
.name("setup-k3s-cluster")
|
|
.version("0.1.0")
|
|
.description("Automate the setup of an HA k3s cluster using incus and k3sup")
|
|
.option("-m, --masters <numMasters:number>", "Number of master nodes", { default: 3 })
|
|
.option("-c, --cleanup", "Force cleanup of VMs if setup fails", { default: false })
|
|
.action(({ masters, cleanup }) => setupCluster(masters, cleanup))
|
|
.parse(Deno.args);
|