ships/mars/main.tf

@@ -0,0 +1,288 @@
+variable "hcloud_token" {
+  type = string
+  sensitive = true
+}
+
+variable "hdns_token" {
+  type = string
+  sensitive = true
+}
+
+variable "ssh_public_key_path" {
+  description = "Path to SSH public key"
+  type        = string
+}
+
+variable "ssh_private_key_path" {
+  description = "Path to SSH private key"
+  type        = string
+}
+
+variable "server" {
+    default = "mars"
+}
+
+variable "server_domain" {
+  default = "mars.fourlights.dev"
+}
+
+variable "ghcr_username" {}
+variable "ghcr_token" {}
+
+resource "null_resource" "is_up" {
+  connection {
+    type        = "ssh"
+    host        = var.server
+    user        = "fourlights"
+    timeout     = "10m"
+    agent       = true
+    agent_identity  = var.ssh_private_key_path
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "whoami",
+    ]
+  }
+}
+
+module "containers-network" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/network"
+
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+
+  name = "containers"
+}
+
+module "minio" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/minio"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+  https = true
+}
+
+module "valkey" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/valkey"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "affine" {
+  wait_on = module.postgres.installed
+  source = "../../quadlets/modules/affine"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+  
+  # Pass postgres password for database connection
+  postgres_password = module.postgres.password
+}
+
+module "oci-proxy" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/oci-proxy"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "arcane" {
+  wait_on = module.oci-proxy.installed
+  source = "../../quadlets/modules/arcane"
+  server_domain = var.server_domain
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "qdrant" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/qdrant"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+  server_domain = var.server_domain
+}
+
+module "documentdb" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/documentdb"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+#module "opensign" {
+#  wait_on = module.documentdb.installed
+#  source = "../../quadlets/modules/opensign"
+#  server_ip = var.server
+#  server_domain = var.server_domain
+#  ssh_private_key_path = var.ssh_private_key_path
+#}
+
+module "postgres" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/postgres"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "rabbitmq" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/rabbitmq"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+#module "plane" {
+#  count = 0
+#  wait_on = null_resource.is_up
+#  source = "../../quadlets/modules/plane"
+#  server_ip = var.server
+#  ssh_private_key_path = var.ssh_private_key_path
+#  server_domain = var.server_domain
+#}
+
+#module "airsonic-advanced" {
+#  wait_on = null_resource.is_up
+#  source = "../../quadlets/modules/airsonic-advanced"
+#  server_ip = var.server
+#  ssh_private_key_path = var.ssh_private_key_path
+#  server_domain = var.server_domain
+#}
+
+module "gonic" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/gonic"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "tmail-web" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/tmail-web"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+  server_url = "https://mail.binarysunset.dev"
+}
+
+module "navidrome" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/navidrome"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+  server_domain = var.server_domain
+}
+
+module "mopidy" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/mopidy"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+  server_domain = var.server_domain
+}
+
+module "zot" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/zot"
+  server_ip = var.server
+  ssh_private_key_path = var.ssh_private_key_path
+  server_domain = var.server_domain
+}
+
+# Deploy Plane after all dependencies are ready
+#module "plane" {
+#  wait_on = null_resource.is_up
+#
+#  source = "../../quadlets/modules/plane"
+#  server_ip = var.server
+#  server_domain = var.server_domain
+#  ssh_private_key_path = var.ssh_private_key_path
+#  
+#  # Pass credentials from existing services
+#  postgres_password = module.postgres.password
+#  minio_server = module.minio.server
+#  minio_access_key = module.minio.access_key
+#  minio_secret_key = module.minio.secret_key
+#  rabbitmq_username = module.rabbitmq.username
+#  rabbitmq_password = module.rabbitmq.password
+#}
+
+module "beets" {
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/beets"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+module "deeptutor" {
+  count = 0
+  wait_on = null_resource.is_up
+  source = "../../quadlets/modules/deeptutor"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+}
+
+output "psql_pw" {
+  value = module.postgres.password
+  sensitive = true
+}
+
+module "forgejo" {
+  wait_on = module.postgres.installed
+  source = "../../quadlets/modules/forgejo"
+  server_ip = var.server
+  server_domain = var.server_domain
+  ssh_private_key_path = var.ssh_private_key_path
+  
+  # Pass postgres password for database connection
+  postgres_password = module.postgres.password
+}
+
+output "minio_app_urls" {
+  value = module.minio.app_urls
+}
+
+output "qdrant_api_key" {
+    value = module.qdrant.api_key
+    sensitive = true
+}
+
+#output "plane_app_urls" {
+#  value = module.plane.app_urls
+#}
+#
+#output "plane_credentials" {
+#  value = module.plane.credentials
+#  sensitive = true
+#}
+#
+#output "plane_main_url" {
+#  value = module.plane.main_url
+#}
+
+output "forgejo_app_urls" {
+  value = module.forgejo.app_urls
+}
+
+output "affine_app_urls" {
+  value = module.affine.app_urls
+}
+
+#output "deeptutor_app_urls" {
+#  value = module.deeptutor.app_urls
+#}
+
+output "tmail_web_app_urls" {
+  value = module.tmail-web.app_urls
+}
+
+#output "plane_credentials" {
+#value = module.plane.credentials
+#sensitive = true
+#}

shipyard/cloud-init.yml

@@ -0,0 +1,561 @@
+#cloud-config
+users:
+  - name: fourlights
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users,admin,sudo
+    shell: /bin/bash
+    lock_passwd: false
+    ssh_authorized_keys:
+      - ${ssh_public_key}
+
+packages:
+  - podman
+  - haproxy
+  - python3
+  - python3-requests
+  - curl
+  - wget
+  - jq
+  - socat
+  - nmap
+
+package_update: true
+package_upgrade: true
+
+write_files:
+  - path: /etc/sudoers.d/fourlights-haproxy
+    permissions: '0440'
+    content: |
+      fourlights ALL=(root) NOPASSWD: /bin/systemctl reload haproxy
+      fourlights ALL=(root) NOPASSWD: /bin/systemctl restart haproxy
+      fourlights ALL=(root) NOPASSWD: /bin/systemctl stop haproxy
+      fourlights ALL=(root) NOPASSWD: /bin/systemctl start haproxy
+      fourlights ALL=(root) NOPASSWD: /bin/chown -R haproxy\:haproxy /etc/ssl/haproxy/*
+      fourlights ALL=(root) NOPASSWD: /bin/chmod 600 /etc/ssl/haproxy/*
+  # HAProxy main configuration
+  - path: /etc/haproxy/haproxy.cfg
+    content: |
+      global
+          daemon
+          stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+          stats timeout 30s
+          user haproxy
+          group haproxy
+          log stdout local0 info
+
+      defaults
+          mode http
+          timeout connect 5000ms
+          timeout client 50000ms
+          timeout server 50000ms
+          option httplog
+          log global
+
+      # Stats interface
+      frontend stats
+          bind *:8404
+          http-request use-service prometheus-exporter if { path /metrics }
+          stats enable
+          stats uri /stats
+          stats refresh 10s
+
+      # HTTP Frontend
+      frontend main
+          bind *:80
+          # ACL to detect ACME challenge requests
+          acl is_acme_challenge path_beg /.well-known/acme-challenge/
+          # Route ACME challenges to the acme_challenge backend
+          use_backend acme_challenge if is_acme_challenge
+          default_backend no_match
+
+      # HTTPS Frontend
+      frontend https_main
+          bind *:443
+          default_backend no_match
+
+      # ACME Challenge Backend
+      backend acme_challenge
+          mode http
+          server acme_server 127.0.0.1:8888
+
+      # Default backend
+      backend no_match
+          http-request return status 404 content-type text/plain string "No matching service found"
+
+  - path: /etc/dataplaneapi/dataplaneapi.yml
+    content: |
+      dataplaneapi:
+        host: 0.0.0.0
+        port: 5555
+        user:
+          - insecure: true
+            password: admin
+            username: admin
+
+      haproxy:
+        config_file: /etc/haproxy/haproxy.cfg
+        haproxy_bin: /usr/sbin/haproxy
+        reload:
+          reload_cmd: systemctl reload haproxy
+          restart_cmd: systemctl restart haproxy
+        stats_socket: /var/run/haproxy/admin.sock
+
+  - path: /usr/local/bin/podman-haproxy-acme-sync-wrapper.sh
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+
+      set -e
+
+      MAX_WAIT=60
+      ELAPSED=0
+      
+      # Wait for HAProxy
+      echo "Checking HAProxy status..."
+      while ! systemctl is-active --quiet haproxy; do
+        echo "Waiting for HAProxy to start..."
+        sleep 2
+        ELAPSED=$($ELAPSED + 2)
+        [ $ELAPSED -ge $MAX_WAIT ] && { echo "ERROR: HAProxy timeout"; exit 1; }
+      done
+      echo "HAProxy is active"
+      
+      # Reset and wait for Data Plane API to actually respond
+      ELAPSED=0
+      echo "Checking Data Plane API readiness..."
+      while true; do
+        HTTP_CODE=$(curl -s -w "%%{http_code}" -o /dev/null \
+          --connect-timeout 5 \
+          --max-time 10 \
+          -u :admin \
+          http://localhost:5555/v3/services/haproxy/configuration/version 2>/dev/null || echo "000")
+      
+        [ "$HTTP_CODE" = "200" ] && { echo "Data Plane API ready"; break; }
+      
+        echo "Waiting for Data Plane API... (HTTP $HTTP_CODE)"
+        sleep 2
+        ELAPSED=$((ELAPSED + 2))
+      
+        if [ $ELAPSED -ge $MAX_WAIT ]; then
+          echo "ERROR: Data Plane API not ready within $MAX_WAITs (HTTP $HTTP_CODE)"
+          journalctl -u dataplaneapi -n 50 --no-pager
+          exit 1
+        fi
+      done
+      
+      sleep 2
+      exec /usr/local/bin/podman-haproxy-acme-sync.py
+
+  # Podman HAProxy ACME Sync Script
+  - path: /usr/local/bin/podman-haproxy-acme-sync.py
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env python3
+
+      import json
+      import subprocess
+      import requests
+      import time
+      import os
+      import sys
+
+      HAPROXY_API_BASE = "http://:admin@127.0.0.1:5555/v3"
+      CERT_DIR = "/home/fourlights/.acme.sh"
+      ACME_SCRIPT = "/usr/local/bin/acme.sh"
+
+      class PodmanHAProxyACMESync:
+          def __init__(self):
+              self.ssl_services = set()
+              self.session = requests.Session()
+              self.session.headers.update({'Content-Type': 'application/json'})
+
+          def get_next_index(self, path):
+              response = self.session.get(f"{HAPROXY_API_BASE}/services/haproxy/configuration/{path}")
+              return len(response.json()) if response.status_code == 200 else None
+
+          def get_dataplaneapi_version(self):
+              response = self.session.get(f"{HAPROXY_API_BASE}/services/haproxy/configuration/version")
+              return response.json() if response.status_code == 200 else None
+
+          def get_container_labels(self, container_id):
+              try:
+                  result = subprocess.run(['podman', 'inspect', container_id], 
+                                        capture_output=True, text=True)
+                  if result.returncode == 0:
+                      data = json.loads(result.stdout)
+                      return data[0]['Config']['Labels'] or {}
+              except Exception as e:
+                  print(f"Error getting labels for {container_id}: {e}")
+              return {}
+
+          def request_certificate(self, domain):
+              print(f"[CERT-REQUEST] About to request certificate for {domain}")
+              sys.stdout.flush()
+
+              try:
+                  cmd = [
+                      ACME_SCRIPT, 
+                      "--issue", 
+                      "-d", domain,
+                      "--standalone",
+                      "--httpport", "8888",
+                      "--server", "letsencrypt",
+                      "--listen-v4",
+                      "--debug", "2"
+                  ]
+
+                  # Log the command being executed
+                  print(f"[CERT-REQUEST] Executing: {' '.join(cmd)}")
+                  sys.stdout.flush()
+
+                  result = subprocess.run(cmd, capture_output=True, text=True)
+
+                  # Log both stdout and stderr for complete debugging
+                  if result.stdout:
+                      print(f"[CERT-STDOUT] {result.stdout}")
+                      sys.stdout.flush()
+                  if result.stderr:
+                      print(f"[CERT-STDERR] {result.stderr}")
+                      sys.stderr.flush()
+
+                  if result.returncode == 0:
+                      print(f"[CERT-SUCCESS] Certificate obtained for {domain}")
+                      sys.stdout.flush()
+                      self.install_certificate(domain)
+                      return True
+                  else:
+                      print(f"[CERT-FAILED] Failed to obtain certificate for {domain}")
+                      print(f"[CERT-FAILED] Return code: {result.returncode}")
+                      sys.stdout.flush()
+                      return False
+
+              except Exception as e:
+                  print(f"[CERT-ERROR] Error requesting certificate: {e}")
+                  sys.stdout.flush()
+                  return False
+
+          def install_certificate(self, domain):
+              cert_file = f"{CERT_DIR}/{domain}.pem"
+
+              try:
+                  acme_cert_dir = f"/home/fourlights/.acme.sh/{domain}_ecc"
+
+                  with open(cert_file, 'w') as outfile:
+                      with open(f"{acme_cert_dir}/fullchain.cer") as cert:
+                          outfile.write(cert.read())
+                      with open(f"{acme_cert_dir}/{domain}.key") as key:
+                          outfile.write(key.read())
+                      try:
+                          with open(f"{acme_cert_dir}/ca.cer") as ca:
+                              outfile.write(ca.read())
+                      except FileNotFoundError:
+                          pass
+
+                  os.chmod(cert_file, 0o600)
+                  print(f"Certificate installed at {cert_file}")
+
+                  self.update_haproxy_ssl_bind(domain)
+
+              except Exception as e:
+                  print(f"Error installing certificate for {domain}: {e}")
+
+          def update_haproxy_ssl_bind(self, domain):
+              print(f"Updating ssl bind for {domain}")
+              try:
+                  ssl_bind_data = {
+                      "address": "*",
+                      "port": 443,
+                      "ssl": True,
+                      "ssl_certificate": f"{CERT_DIR}/{domain}.pem",
+                  }
+
+                  response = self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/https_main/binds?version={self.get_dataplaneapi_version()}",
+                                             json=ssl_bind_data)
+                  print(response.json())
+
+                  if response.status_code in [200, 201]:
+                      print(f"Updated HAProxy SSL bind for {domain}")
+
+              except Exception as e:
+                  print(f"Error updating HAProxy SSL bind: {e}")
+
+          def setup_certificate_renewal(self, domain):
+              renewal_script = f"/etc/cron.d/acme-{domain.replace('.', '-')}"
+
+              cron_content = f"""0 0 * * * root {ACME_SCRIPT} --renew -d {domain} --post-hook "systemctl reload haproxy" >/dev/null 2>&1
+      """
+
+              with open(renewal_script, 'w') as f:
+                  f.write(cron_content)
+
+              print(f"Setup automatic renewal for {domain}")
+
+          def update_haproxy_backend(self, service_name, host, port, action='add'):
+              backend_name = f"backend_{service_name}"
+              server_name = f"{service_name}_server"
+
+              if action == 'add':
+                  backend_data = {
+                      "name": backend_name,
+                      "mode": "http",
+                      "balance": {"algorithm": "roundrobin"},
+                  }
+                  backends = self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/backends?version={self.get_dataplaneapi_version()}",
+                                   json=backend_data)
+                  print(backends.json())
+
+                  server_data = {
+                      "name": server_name,
+                      "address": host,
+                      "port": int(port),
+                      "check": "enabled",
+                  }
+                  tweak = self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/backends/{backend_name}/servers?version={self.get_dataplaneapi_version()}",
+                                  json=server_data)
+                  print(tweak.json())
+
+              elif action == 'remove':
+                  self.session.delete(f"{HAPROXY_API_BASE}/services/haproxy/configuration/backends/{backend_name}/servers/{server_name}?version={self.get_dataplaneapi_version()}")
+
+          def update_haproxy_frontend_rule(self, service_name, domain, ssl_enabled=False, action='add'):
+              if action == 'add':
+                  if ssl_enabled and domain and domain not in self.ssl_services:
+                      print(f"Setting up SSL for {domain}")
+                      if self.request_certificate(domain):
+                          self.setup_certificate_renewal(domain)
+                          self.ssl_services.add(domain)
+
+                  acl_data = {
+                      "acl_name": f"is_{service_name}",
+                      "criterion": "hdr(host)",
+                      "value": domain,
+                  }
+                  self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/main/acls/{self.get_next_index('frontends/main/acls')}?version={self.get_dataplaneapi_version()}",
+                                  json=acl_data)
+
+                  if ssl_enabled:
+                      self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/https_main/acls/{self.get_next_index('frontends/https_main/acls')}?version={self.get_dataplaneapi_version()}",
+                                      json=acl_data)
+
+                  rule_data = {
+                      "name": f"backend_{service_name}",
+                      "cond": "if",
+                      "cond_test": f"is_{service_name}",
+                  }
+                  self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/main/backend_switching_rules/{self.get_next_index('frontends/main/backend_switching_rules')}?version={self.get_dataplaneapi_version()}",
+                                  json=rule_data)
+
+                  if ssl_enabled:
+                      self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/https_main/backend_switching_rules/{self.get_next_index('frontends/https_main/backend_switching_rules')}?version={self.get_dataplaneapi_version()}",
+                                      json=rule_data)
+
+                      redirect_rule = {
+                          "type": "redirect",
+                          "redirect_rule": {
+                              "type": "scheme",
+                              "value": "https",
+                              "code": 301
+                          },
+                          "cond": "if",
+                          "cond_test": f"is_{service_name}",
+                      }
+                      self.session.post(f"{HAPROXY_API_BASE}/services/haproxy/configuration/frontends/main/http_request_rules/{self.get_next_index('frontends/main/http_request_rules')}?version={self.get_dataplaneapi_version()}",
+                                      json=redirect_rule)
+
+          def process_container_event(self, event):
+              # DIAGNOSTIC: Log raw event structure
+              print(f"[EVENT-DEBUG] Received event - Type: {event.get('Type', 'MISSING')}, Action: {event.get('Action', 'MISSING')}")
+              sys.stdout.flush()
+              
+              # DIAGNOSTIC: Check for Actor key
+              if 'Actor' not in event:
+                  print(f"[EVENT-SKIP] Skipping event without 'Actor' key - Full event: {json.dumps(event)}")
+                  sys.stdout.flush()
+                  return
+              
+              # DIAGNOSTIC: Check for ID in Actor
+              if 'ID' not in event['Actor']:
+                  print(f"[EVENT-SKIP] Skipping event without 'Actor.ID' - Actor content: {json.dumps(event['Actor'])}")
+                  sys.stdout.flush()
+                  return
+              
+              container_id = event['Actor']['ID'][:12]
+              action = event['Action']
+              
+              print(f"[EVENT-PROCESS] Processing '{action}' event for container {container_id}")
+              sys.stdout.flush()
+
+              labels = self.get_container_labels(container_id)
+              
+              # Dictionary to store discovered services
+              services = {}
+              
+              # First, check for namespaced labels (haproxy.{service_name}.enable)
+              for label_key, label_value in labels.items():
+                  if label_key.startswith('haproxy.') and label_key.endswith('.enable') and label_value.lower() == 'true':
+                      # Extract service name from label key
+                      parts = label_key.split('.')
+                      if len(parts) == 3:  # haproxy.{service_name}.enable
+                          service_name = parts[1]
+                          
+                          # Extract properties for this service namespace
+                          service_config = {
+                              'service_name': service_name,
+                              'host': labels.get(f'haproxy.{service_name}.host', '127.0.0.1'),
+                              'port': labels.get(f'haproxy.{service_name}.port', '8080'),
+                              'domain': labels.get(f'haproxy.{service_name}.domain', None),
+                              'ssl_enabled': labels.get(f'haproxy.{service_name}.tls', 'false').lower() == 'true'
+                          }
+                          services[service_name] = service_config
+              
+              # Backward compatibility: If no namespaced labels found, check for flat labels
+              if not services and 'haproxy.enable' in labels and labels['haproxy.enable'].lower() == 'true':
+                  service_name = labels.get('haproxy.service', container_id)
+                  services[service_name] = {
+                      'service_name': service_name,
+                      'host': labels.get('haproxy.host', '127.0.0.1'),
+                      'port': labels.get('haproxy.port', '8080'),
+                      'domain': labels.get('haproxy.domain', None),
+                      'ssl_enabled': labels.get('haproxy.tls', 'false').lower() == 'true'
+                  }
+              
+              # Process each discovered service
+              for service_name, config in services.items():
+                  if action in ['start', 'restart']:
+                      print(f"Adding service {config['service_name']} to HAProxy (SSL: {config['ssl_enabled']}, Domain: {config['domain']})")
+                      sys.stdout.flush()
+                      self.update_haproxy_backend(config['service_name'], config['host'], config['port'], 'add')
+                      if config['domain']:
+                          self.update_haproxy_frontend_rule(config['service_name'], config['domain'], config['ssl_enabled'], 'add')
+
+                  elif action in ['stop', 'remove', 'died']:
+                      print(f"Removing service {config['service_name']} from HAProxy")
+                      sys.stdout.flush()
+                      self.update_haproxy_backend(config['service_name'], config['host'], config['port'], 'remove')
+
+          def watch_events(self):
+              print("Starting Podman-HAProxy-ACME sync...")
+          
+              # Track last sync time
+              last_full_sync = 0
+              SYNC_INTERVAL = 60  # Re-scan all containers every 60 seconds
+          
+              def do_full_sync():
+                  """Perform a full sync of all running containers"""
+                  print("Performing full container sync...")
+                  try:
+                      result = subprocess.run(['podman', 'ps', '--format', 'json'],
+                                            capture_output=True, text=True)
+                      if result.returncode == 0:
+                          containers = json.loads(result.stdout)
+                          for container in containers:
+                              event = {
+                                  'Type': 'container',
+                                  'Action': 'start',
+                                  'Actor': {'ID': container.get('Id', '')}
+                              }
+                              self.process_container_event(event)
+                          print(f"Synced {len(containers)} containers")
+                  except Exception as e:
+                      print(f"Error during full sync: {e}")
+          
+              # Initial sync
+              do_full_sync()
+              last_full_sync = time.time()
+          
+              print("Watching for container events...")
+          
+              cmd = ['podman', 'events', '--format', 'json']
+              process = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True)
+          
+              # Use select/poll for non-blocking read so we can do periodic syncs
+              import select
+          
+              while True:
+                  # Check if it's time for periodic sync
+                  if time.time() - last_full_sync >= SYNC_INTERVAL:
+                      do_full_sync()
+                      last_full_sync = time.time()
+          
+                  # Check for events with timeout
+                  ready, _, _ = select.select([process.stdout], [], [], 5)
+          
+                  if ready:
+                      line = process.stdout.readline()
+                      if line:
+                          try:
+                              event = json.loads(line.strip())
+                              if event['Type'] == 'container':
+                                  self.process_container_event(event)
+                          except json.JSONDecodeError as e:
+                              print(f"[EVENT-ERROR] JSON decode error: {e} - Line: {line[:100]}")
+                              sys.stdout.flush()
+                          except KeyError as e:
+                              print(f"[EVENT-ERROR] Missing key {e} in event: {json.dumps(event)}")
+                              sys.stdout.flush()
+                          except Exception as e:
+                              print(f"[EVENT-ERROR] Error processing event: {e}")
+                              print(f"[EVENT-ERROR] Event structure: {json.dumps(event)}")
+                              sys.stdout.flush()
+
+      if __name__ == "__main__":
+          os.makedirs(CERT_DIR, exist_ok=True)
+          sync = PodmanHAProxyACMESync()
+          sync.watch_events()
+
+runcmd:
+  # Create necessary directories
+  - mkdir -p /var/run/haproxy /etc/ssl/haproxy /etc/containers/systemd /etc/haproxy/dataplane /etc/dataplaneapi
+  - chown haproxy:haproxy /var/run/haproxy
+
+  # Install Data Plane API
+  - cd /tmp && curl -LO https://github.com/haproxytech/dataplaneapi/releases/download/v3.2.4/dataplaneapi_3.2.4_linux_amd64.deb
+  - env DEBIAN_FRONTEND=noninteractive apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" /tmp/dataplaneapi_3.2.4_linux_amd64.deb
+  - rm /tmp/dataplaneapi_3.2.4_linux_amd64.deb
+
+  - mkdir -p /home/fourlights/.config/containers/systemd
+  - mkdir -p /home/fourlights/.config/systemd/user
+  - |
+    cat > /home/fourlights/.config/systemd/user/podman-haproxy-acme-sync.service << 'EOF'
+    [Unit]
+    Description=Podman HAProxy ACME Sync Service
+    After=network.target
+
+    [Service]
+    Type=simple
+    Environment="XDG_RUNTIME_DIR=/run/user/1000"
+    Environment="DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus"
+    ExecStart=/usr/local/bin/podman-haproxy-acme-sync-wrapper.sh
+    StandardOutput=journal
+    StandardError=journal
+    Restart=always
+    RestartSec=10
+
+    [Install]
+    WantedBy=default.target
+    EOF
+  - chown -R fourlights:fourlights /home/fourlights
+
+  # Install ACME.sh
+  - su - fourlights -c 'curl https://get.acme.sh | sh -s email=${acme_email}'
+  - ln -sf /home/fourlights/.acme.sh/acme.sh /usr/local/bin/acme.sh
+
+  # Setup data directory and mount volume
+  - mkdir -p /opt/storage/data
+  - mkfs.ext4 -F /dev/sdb
+  - mount /dev/sdb /opt/storage/data
+  - echo '/dev/sdb /opt/storage/data ext4 defaults 0 2' >> /etc/fstab
+  - chown -R fourlights:fourlights /opt/storage/data
+
+  # Enable Podman for user services
+  - loginctl enable-linger fourlights
+  - su - fourlights -c 'podman login ghcr.io -u ${ghcr_username} -p ${ghcr_token}'
+
+  # Enable and start services
+  - systemctl daemon-reload
+  - systemctl enable --now haproxy
+  - systemctl enable --now dataplaneapi
+  - su - fourlights -c 'systemctl --user daemon-reload'
+  - su - fourlights -c 'systemctl --user enable --now podman-haproxy-acme-sync'
+
+final_message: "Server setup complete with HAProxy, Podman, and ACME sync configured"