#!/usr/bin/env python3 """Disk-health / node-removal-safety report for a Gallium cluster. The question this answers: **is it safe to remove a node** (permanently, or temporarily for maintenance)? A node is unsafe to remove when some volume's last usable copy of its data lives only on that node -- removing it would take that data offline (permanently, or for the duration of the maintenance). Data model (all read live via `kubectl`): * KubeVirt VirtualMachines / VirtualMachineInstances -- the VMs and whether they're running, and which node each runs on. * The PVCs each VM references (directly, via DataVolume, or hot-plugged). * Longhorn Volumes + Replicas -- for every PVC's underlying volume, where its replicas live (which node) and whether each replica is healthy. * Pods -- to attribute PVCs that aren't owned by a VM to a non-VM workload vs. nothing at all. The report has three parts: 1. A per-node removal-safety summary (the headline -- SAFE / UNSAFE). 2. Per-VM detail: the VM, its running state, and for each of its PVCs a table of (replica, node, health). 3. PVCs not attached to any VM, split into "attached to a non-VM pod" and "completely detached". Self-contained: standard library only (Python 3.6+); the sole dependency is a `kubectl` binary that can reach the cluster -- so it runs unchanged on a cluster node (SLES / Python 3.6) or a workstation. """ import argparse import json import re import subprocess import sys from collections import defaultdict LONGHORN_NS = "longhorn-system" LONGHORN_CSI_DRIVER = "driver.longhorn.io" HEALTHY = "healthy" # --------------------------------------------------------------------------- # # kubectl plumbing # --------------------------------------------------------------------------- # class Kubectl(object): """Thin wrapper that shells out to kubectl and parses JSON.""" def __init__(self, binary, kubeconfig, context): self.base = [binary] if kubeconfig: self.base += ["--kubeconfig", kubeconfig] if context: self.base += ["--context", context] def get(self, resource, namespace=None): """`kubectl get -o json` -> list of items. Returns [] (with a warning) when the resource type isn't present -- e.g. a cluster without KubeVirt or Longhorn installed -- rather than aborting the whole report. """ cmd = list(self.base) + ["get", resource, "-o", "json"] cmd += ["-n", namespace] if namespace else ["-A"] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) if proc.returncode != 0: err = (proc.stderr or "").strip() low = err.lower() if "the server doesn't have a resource type" in low or "could not find" in low: warn("resource %r not found on the cluster; skipping (%s)" % (resource, err)) return [] die("kubectl get %s failed: %s" % (resource, err)) try: return json.loads(proc.stdout).get("items", []) except ValueError as exc: die("could not parse kubectl output for %s: %s" % (resource, exc)) return [] # --------------------------------------------------------------------------- # # colour / output helpers # --------------------------------------------------------------------------- # class C(object): """ANSI colour codes; blanked out when colour is disabled.""" RESET = BOLD = DIM = RED = GREEN = YELLOW = BLUE = CYAN = "" @classmethod def enable(cls): cls.RESET = "\033[0m" cls.BOLD = "\033[1m" cls.DIM = "\033[2m" cls.RED = "\033[31m" cls.GREEN = "\033[32m" cls.YELLOW = "\033[33m" cls.BLUE = "\033[34m" cls.CYAN = "\033[36m" def warn(msg): print("%swarning:%s %s" % (C.YELLOW, C.RESET, msg), file=sys.stderr) def die(msg): print("%serror:%s %s" % (C.RED, C.RESET, msg), file=sys.stderr) sys.exit(1) _ANSI = re.compile(r"\033\[[0-9;]*m") def render_table(headers, rows, indent=" "): """Render a simple fixed-width table. Colour codes are ignored for width.""" def vlen(s): return len(_ANSI.sub("", s)) widths = [vlen(h) for h in headers] for row in rows: for i, cell in enumerate(row): widths[i] = max(widths[i], vlen(cell)) def fmt(row): cells = [] for i, cell in enumerate(row): cells.append(cell + " " * (widths[i] - vlen(cell))) return indent + " ".join(cells).rstrip() sep = indent + " ".join("-" * w for w in widths) out = [fmt(["%s%s%s" % (C.BOLD, h, C.RESET) for h in headers]), sep] out += [fmt(r) for r in rows] return "\n".join(out) # --------------------------------------------------------------------------- # # domain types # --------------------------------------------------------------------------- # class Replica(object): def __init__(self, name, node, health, has_data): self.name = name # replica CR name self.node = node # "" when Longhorn couldn't schedule it self.health = health # display: healthy|stopped|rebuilding|failed|error|unknown self.has_data = has_data # holds a usable copy (healthyAt set, not failed) class Volume(object): """A Longhorn volume backing one PVC (or a non-Longhorn PVC's stand-in).""" def __init__(self, pvc_ns, pvc_name, lh_name, storage_class, robustness, state, desired_replicas, replicas=None, note=""): self.pvc_ns = pvc_ns self.pvc_name = pvc_name self.lh_name = lh_name # Longhorn volume name (== bound PV name) self.storage_class = storage_class self.robustness = robustness # healthy|degraded|faulted|unknown|- self.state = state # attached|detached|- self.desired_replicas = desired_replicas self.replicas = replicas if replicas is not None else [] self.note = note # e.g. non-Longhorn, unbound, missing @property def data_nodes(self): """Nodes that hold a usable copy of this volume's data. Uses data validity (healthyAt && !failedAt), NOT engine-running state: a cleanly-detached volume (stopped VM) still has good copies on its replica nodes, so losing one of those nodes still loses data. """ return set(r.node for r in self.replicas if r.has_data and r.node) @property def all_nodes(self): return set(r.node for r in self.replicas if r.node) class VM(object): def __init__(self, ns, name, status, node, pvcs): self.ns = ns self.name = name self.status = status # Running | Stopped | ... self.node = node # node the VMI runs on (or None) self.pvcs = pvcs # list of (ns, name) referenced by this VM @property def running(self): return self.status == "Running" # --------------------------------------------------------------------------- # # building the model # --------------------------------------------------------------------------- # def classify_replica(rep): """Return (display-health, has_data) for a Longhorn Replica CR. has_data is the data-safety signal: the replica holds a usable copy iff it has been healthy (`healthyAt`) and hasn't failed (`failedAt`) -- true even when the engine is stopped because the volume is detached. """ spec = rep.get("spec", {}) status = rep.get("status", {}) failed = bool(spec.get("failedAt")) healthy_at = bool(spec.get("healthyAt")) cur = status.get("currentState", "") has_data = healthy_at and not failed if failed: disp = "failed" elif cur == "running": disp = HEALTHY if healthy_at else "rebuilding" elif healthy_at: disp = "stopped" # good copy, engine down (volume detached) elif cur in ("stopped", "error"): disp = cur else: disp = cur or "unknown" return disp, has_data def vm_referenced_pvcs(vm_spec_volumes, ns): """PVC (ns, name) pairs a set of KubeVirt `volumes[]` entries reference.""" out = [] for v in vm_spec_volumes or []: pvc = v.get("persistentVolumeClaim") if pvc and pvc.get("claimName"): out.append((ns, pvc["claimName"])) continue # A DataVolume materialises a PVC with the same name in the same ns. dv = v.get("dataVolume") if dv and dv.get("name"): out.append((ns, dv["name"])) return out def is_virt_launcher(pod): labels = pod.get("metadata", {}).get("labels", {}) or {} if labels.get("kubevirt.io") == "virt-launcher": return True for ref in pod.get("metadata", {}).get("ownerReferences", []) or []: if ref.get("kind") == "VirtualMachineInstance": return True return False def virt_launcher_vm_name(pod): """The VM/VMI name a virt-launcher pod belongs to (VMI name == VM name).""" for ref in pod.get("metadata", {}).get("ownerReferences", []) or []: if ref.get("kind") == "VirtualMachineInstance" and ref.get("name"): return ref["name"] return (pod.get("metadata", {}).get("labels", {}) or {}).get("vm.kubevirt.io/name", "") def build_volume(pvc_ns, pvc_name, pvc_index, pv_index, lh_vol_index, replicas_by_vol): pvc = pvc_index.get((pvc_ns, pvc_name)) if pvc is None: return Volume(pvc_ns, pvc_name, None, "-", "-", "-", None, note="PVC not found") sc = pvc.get("spec", {}).get("storageClassName", "") or "-" pv_name = pvc.get("spec", {}).get("volumeName") or "" phase = pvc.get("status", {}).get("phase", "") if not pv_name: return Volume(pvc_ns, pvc_name, None, sc, "-", "-", None, note="unbound PVC (phase=%s)" % phase) pv = pv_index.get(pv_name) driver = "" if pv: driver = pv.get("spec", {}).get("csi", {}).get("driver", "") if pv and driver and driver != LONGHORN_CSI_DRIVER: # Non-Longhorn PV: its data lives wherever the PV binds (often one node). node = _pv_pinned_node(pv) vol = Volume(pvc_ns, pvc_name, pv_name, sc, "-", "-", None, note="non-Longhorn (%s)" % driver) if node: vol.replicas.append(Replica(pv_name, node, HEALTHY, True)) return vol # Longhorn: the volume CR name equals the bound PV name. lh = lh_vol_index.get(pv_name) if lh is None: return Volume(pvc_ns, pvc_name, pv_name, sc, "-", "-", None, note="no Longhorn volume CR") robustness = lh.get("status", {}).get("robustness", "unknown") or "unknown" state = lh.get("status", {}).get("state", "unknown") or "unknown" desired = lh.get("spec", {}).get("numberOfReplicas") vol = Volume(pvc_ns, pvc_name, pv_name, sc, robustness, state, desired) for rep in replicas_by_vol.get(pv_name, []): # An empty nodeID means Longhorn couldn't schedule this replica (e.g. a # 3rd replica on a 2-node cluster). Keep it "" so it never counts as a # node that holds data, and render it as "(unscheduled)". node = rep.get("spec", {}).get("nodeID", "") or "" disp, has_data = classify_replica(rep) vol.replicas.append(Replica( rep.get("metadata", {}).get("name", "?"), node, disp, has_data)) vol.replicas.sort(key=lambda r: (r.node, r.name)) return vol def _pv_pinned_node(pv): """Best-effort node a non-Longhorn PV is pinned to (nodeAffinity).""" terms = (pv.get("spec", {}).get("nodeAffinity", {}) .get("required", {}).get("nodeSelectorTerms", [])) for term in terms: for expr in term.get("matchExpressions", []): if expr.get("key") == "kubernetes.io/hostname" and expr.get("values"): return expr["values"][0] return None # --------------------------------------------------------------------------- # # health formatting # --------------------------------------------------------------------------- # def colour_health(health): good = (HEALTHY, "attached") warnish = ("rebuilding", "degraded", "stopped", "detached") bad = ("failed", "faulted", "error", "unknown") if health in good: return "%s%s%s" % (C.GREEN, health, C.RESET) if health in warnish: return "%s%s%s" % (C.YELLOW, health, C.RESET) if health in bad: return "%s%s%s" % (C.RED, health, C.RESET) return health def colour_status(status): if status == "Running": return "%s%s%s" % (C.GREEN, status, C.RESET) if status in ("Stopped", "Paused"): return "%s%s%s" % (C.YELLOW, status, C.RESET) return "%s%s%s" % (C.CYAN, status, C.RESET) # --------------------------------------------------------------------------- # # main report # --------------------------------------------------------------------------- # def main(): ap = argparse.ArgumentParser( description="Report Longhorn disk health and assess node-removal safety.") ap.add_argument("--kubectl", default="kubectl", help="kubectl binary to use (default: kubectl on PATH)") ap.add_argument("--kubeconfig", help="path to kubeconfig (else KUBECONFIG/default)") ap.add_argument("--context", help="kube context to use") ap.add_argument("--json", action="store_true", dest="as_json", help="emit the structured model as JSON instead of a report") ap.add_argument("--no-color", action="store_true", help="disable ANSI colour") args = ap.parse_args() if not args.no_color and sys.stdout.isatty(): C.enable() kc = Kubectl(args.kubectl, args.kubeconfig, args.context) # --- fetch everything up front ---------------------------------------- # vms_raw = kc.get("virtualmachines.kubevirt.io") vmis_raw = kc.get("virtualmachineinstances.kubevirt.io") pvcs_raw = kc.get("persistentvolumeclaims") pvs_raw = kc.get("persistentvolumes") pods_raw = kc.get("pods") lh_vols_raw = kc.get("volumes.longhorn.io", LONGHORN_NS) lh_reps_raw = kc.get("replicas.longhorn.io", LONGHORN_NS) pvc_index = dict(((p["metadata"]["namespace"], p["metadata"]["name"]), p) for p in pvcs_raw) pv_index = dict((p["metadata"]["name"], p) for p in pvs_raw) lh_vol_index = dict((v["metadata"]["name"], v) for v in lh_vols_raw) replicas_by_vol = defaultdict(list) for rep in lh_reps_raw: vol_name = rep.get("spec", {}).get("volumeName") if vol_name: replicas_by_vol[vol_name].append(rep) vmi_index = dict(((v["metadata"]["namespace"], v["metadata"]["name"]), v) for v in vmis_raw) # --- assemble VMs ------------------------------------------------------ # vms = [] for raw in vms_raw: ns = raw["metadata"]["namespace"] name = raw["metadata"]["name"] status = raw.get("status", {}).get("printableStatus", "") or "Unknown" spec_vols = (raw.get("spec", {}).get("template", {}) .get("spec", {}).get("volumes", [])) pvcs = vm_referenced_pvcs(spec_vols, ns) node = None vmi = vmi_index.get((ns, name)) if vmi: node = vmi.get("status", {}).get("nodeName") # Include hot-plugged / VMI-level volumes too. for extra in vm_referenced_pvcs(vmi.get("spec", {}).get("volumes", []), ns): if extra not in pvcs: pvcs.append(extra) vms.append(VM(ns, name, status, node, pvcs)) vms.sort(key=lambda v: (v.ns, v.name)) vm_by_key = dict(((vm.ns, vm.name), vm) for vm in vms) # --- pods that mount PVCs --------------------------------------------- # # A running VM's virt-launcher pod mounts every volume the VM actually uses # -- including ones KubeVirt manages outside spec.template.spec.volumes (the # persistent-state/EFI-vars PVC, hot-plugged disks). Attribute those to the # owning VM so they show under the VM, not as "detached". Other pods are # real non-VM consumers. pods_by_pvc = defaultdict(list) for pod in pods_raw: pns = pod["metadata"]["namespace"] pnode = pod.get("spec", {}).get("nodeName", "") or "?" pname = pod["metadata"]["name"] claims = [v["persistentVolumeClaim"]["claimName"] for v in pod.get("spec", {}).get("volumes", []) or [] if v.get("persistentVolumeClaim", {}).get("claimName")] if is_virt_launcher(pod): vm = vm_by_key.get((pns, virt_launcher_vm_name(pod))) if vm is not None: for c in claims: if (pns, c) not in vm.pvcs: vm.pvcs.append((pns, c)) continue # a VM's own pod; its PVCs belong to the VM section for c in claims: pods_by_pvc[(pns, c)].append((pname, pnode)) # Set of PVCs owned by a VM (after folding in virt-launcher-mounted ones). vm_pvc_set = set(p for vm in vms for p in vm.pvcs) # --- build volumes for every PVC -------------------------------------- # def vol_for(ns_name): return build_volume(ns_name[0], ns_name[1], pvc_index, pv_index, lh_vol_index, replicas_by_vol) all_pvc_keys = set(pvc_index.keys()) | vm_pvc_set volumes = dict((k, vol_for(k)) for k in all_pvc_keys) # --- node inventory ---------------------------------------------------- # nodes = sorted(set(r.node for v in volumes.values() for r in v.replicas if r.node)) if args.as_json: emit_json(vms, volumes, pods_by_pvc, vm_pvc_set, nodes) return print_node_safety(nodes, vms, volumes) print_vm_detail(vms, volumes) print_unattached(volumes, vm_pvc_set, pods_by_pvc) def _risk_line(vol): lh = vol.lh_name or "-" hn = ", ".join(sorted(vol.data_nodes)) or "(none)" return ("%s/%s [%s] robustness=%s usable-copies-on: %s" % (vol.pvc_ns, vol.pvc_name, lh, vol.robustness, hn)) def print_node_safety(nodes, vms, volumes): print("%s=== NODE REMOVAL SAFETY ===%s" % (C.BOLD, C.RESET)) if not nodes: print(" no replica-bearing nodes found (no Longhorn volumes?).\n") return # Which VM (running?) owns each PVC, for severity. running_pvcs = set(p for vm in vms if vm.running for p in vm.pvcs) # Volumes already unavailable (no usable copy on any node) -- independent of # which node you remove. Reported once, globally, not per node. broken = [vol for vol in volumes.values() if vol.replicas and not vol.data_nodes] for node in nodes: crit, warns = [], [] for key, vol in volumes.items(): if not vol.replicas or not vol.data_nodes: continue if vol.data_nodes == set([node]): # Removing this node drops the last usable copy. (crit if key in running_pvcs else warns).append(vol) if not crit and not warns: print(" %sSAFE%s %s%s%s: every volume keeps a usable copy on another node." % (C.GREEN, C.RESET, C.BOLD, node, C.RESET)) continue verdict = ("%sUNSAFE%s" % (C.RED, C.RESET)) if crit else ("%sCAUTION%s" % (C.YELLOW, C.RESET)) print(" %s %s%s%s:" % (verdict, C.BOLD, node, C.RESET)) if crit: print(" %srunning-VM data lost if removed (%d):%s" % (C.RED, len(crit), C.RESET)) for vol in crit: print(" - %s" % _risk_line(vol)) if warns: print(" %sidle/non-VM data lost if removed (%d):%s" % (C.YELLOW, len(warns), C.RESET)) for vol in warns: print(" - %s" % _risk_line(vol)) if broken: print("\n %sAlready unavailable%s (no usable copy on ANY node -- fix before draining anything):" % (C.RED, C.RESET)) for vol in broken: print(" - %s" % _risk_line(vol)) print() def _pvc_block(vol): head = (" PVC %s%s/%s%s vol=%s sc=%s" % (C.CYAN, vol.pvc_ns, vol.pvc_name, C.RESET, vol.lh_name or "-", vol.storage_class)) meta = [] if vol.robustness != "-": meta.append("robustness=%s" % colour_health(vol.robustness)) if vol.state != "-": meta.append("state=%s" % colour_health(vol.state)) if vol.desired_replicas is not None: meta.append("desired-replicas=%s" % vol.desired_replicas) if meta: head += " (" + " ".join(meta) + ")" if vol.note: head += " %s[%s]%s" % (C.DIM, vol.note, C.RESET) lines = [head] if vol.replicas: rows = [[r.name, r.node or "(unscheduled)", colour_health(r.health)] for r in vol.replicas] lines.append(render_table(["REPLICA", "NODE", "HEALTH"], rows, indent=" ")) elif not vol.note: lines.append(" %s(no replicas reported)%s" % (C.DIM, C.RESET)) return "\n".join(lines) def print_vm_detail(vms, volumes): print("%s=== VIRTUAL MACHINES ===%s" % (C.BOLD, C.RESET)) if not vms: print(" no VirtualMachines found.\n") return for vm in vms: node = (" node=%s" % vm.node) if vm.node else "" print("\n%sVM %s/%s%s [%s]%s" % (C.BOLD, vm.ns, vm.name, C.RESET, colour_status(vm.status), node)) if not vm.pvcs: print(" %s(no PVC-backed volumes)%s" % (C.DIM, C.RESET)) continue for key in vm.pvcs: print(_pvc_block(volumes[key])) print() def print_unattached(volumes, vm_pvc_set, pods_by_pvc): non_vm = dict((k, v) for k, v in volumes.items() if k not in vm_pvc_set) pod_attached = dict((k, v) for k, v in non_vm.items() if pods_by_pvc.get(k)) detached = dict((k, v) for k, v in non_vm.items() if not pods_by_pvc.get(k)) print("%s=== PVCs NOT ATTACHED TO A VM ===%s" % (C.BOLD, C.RESET)) if not non_vm: print(" none: every PVC is attached to a VM.\n") return if pod_attached: print("\n %sAttached to non-VM pod(s):%s" % (C.BOLD, C.RESET)) for key in sorted(pod_attached): consumers = ", ".join("%s (node=%s)" % (n, nd) for n, nd in pods_by_pvc[key]) print(" used by: %s" % consumers) print(_pvc_block(pod_attached[key])) if detached: print("\n %sCompletely detached (no consumer):%s" % (C.BOLD, C.RESET)) for key in sorted(detached): print(_pvc_block(detached[key])) print() def emit_json(vms, volumes, pods_by_pvc, vm_pvc_set, nodes): def vol_dict(v): return { "pvc_namespace": v.pvc_ns, "pvc_name": v.pvc_name, "longhorn_volume": v.lh_name, "storage_class": v.storage_class, "robustness": v.robustness, "state": v.state, "desired_replicas": v.desired_replicas, "note": v.note or None, "replicas": [ {"name": r.name, "node": r.node, "health": r.health, "has_data": r.has_data} for r in v.replicas ], "usable_copy_nodes": sorted(v.data_nodes), } running_pvcs = set(p for vm in vms if vm.running for p in vm.pvcs) node_safety = {} for node in nodes: crit, warns = [], [] for key, vol in volumes.items(): if vol.replicas and vol.data_nodes == set([node]): (crit if key in running_pvcs else warns).append( "%s/%s" % (vol.pvc_ns, vol.pvc_name)) node_safety[node] = { "safe": not crit and not warns, "running_vm_data_only_here": crit, "other_data_only_here": warns, } out = { "nodes": nodes, "node_safety": node_safety, "vms": [ { "namespace": vm.ns, "name": vm.name, "status": vm.status, "running": vm.running, "node": vm.node, "pvcs": ["%s/%s" % (ns, n) for ns, n in vm.pvcs], } for vm in vms ], "volumes": [vol_dict(v) for v in volumes.values()], "unattached": dict( ("%s/%s" % (k[0], k[1]), {"pods": pods_by_pvc.get(k, []), "detached": not pods_by_pvc.get(k)}) for k in volumes if k not in vm_pvc_set ), } print(json.dumps(out, indent=2)) if __name__ == "__main__": main()