614 lines
24 KiB
Python
Executable File
614 lines
24 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Disk-health / node-removal-safety report for a Gallium cluster.
|
|
|
|
The question this answers: **is it safe to remove a node** (permanently, or
|
|
temporarily for maintenance)? A node is unsafe to remove when some volume's
|
|
last usable copy of its data lives only on that node -- removing it would take
|
|
that data offline (permanently, or for the duration of the maintenance).
|
|
|
|
Data model (all read live via `kubectl`):
|
|
|
|
* KubeVirt VirtualMachines / VirtualMachineInstances -- the VMs and whether
|
|
they're running, and which node each runs on.
|
|
* The PVCs each VM references (directly, via DataVolume, or hot-plugged).
|
|
* Longhorn Volumes + Replicas -- for every PVC's underlying volume, where its
|
|
replicas live (which node) and whether each replica is healthy.
|
|
* Pods -- to attribute PVCs that aren't owned by a VM to a non-VM workload
|
|
vs. nothing at all.
|
|
|
|
The report has three parts:
|
|
|
|
1. A per-node removal-safety summary (the headline -- SAFE / UNSAFE).
|
|
2. Per-VM detail: the VM, its running state, and for each of its PVCs a
|
|
table of (replica, node, health).
|
|
3. PVCs not attached to any VM, split into "attached to a non-VM pod" and
|
|
"completely detached".
|
|
|
|
Self-contained: standard library only (Python 3.6+); the sole dependency is a
|
|
`kubectl` binary that can reach the cluster -- so it runs unchanged on a cluster
|
|
node (SLES / Python 3.6) or a workstation.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
LONGHORN_NS = "longhorn-system"
|
|
LONGHORN_CSI_DRIVER = "driver.longhorn.io"
|
|
HEALTHY = "healthy"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# kubectl plumbing
|
|
# --------------------------------------------------------------------------- #
|
|
class Kubectl(object):
|
|
"""Thin wrapper that shells out to kubectl and parses JSON."""
|
|
|
|
def __init__(self, binary, kubeconfig, context):
|
|
self.base = [binary]
|
|
if kubeconfig:
|
|
self.base += ["--kubeconfig", kubeconfig]
|
|
if context:
|
|
self.base += ["--context", context]
|
|
|
|
def get(self, resource, namespace=None):
|
|
"""`kubectl get <resource> -o json` -> list of items.
|
|
|
|
Returns [] (with a warning) when the resource type isn't present -- e.g.
|
|
a cluster without KubeVirt or Longhorn installed -- rather than aborting
|
|
the whole report.
|
|
"""
|
|
cmd = list(self.base) + ["get", resource, "-o", "json"]
|
|
cmd += ["-n", namespace] if namespace else ["-A"]
|
|
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
if proc.returncode != 0:
|
|
err = (proc.stderr or "").strip()
|
|
low = err.lower()
|
|
if "the server doesn't have a resource type" in low or "could not find" in low:
|
|
warn("resource %r not found on the cluster; skipping (%s)" % (resource, err))
|
|
return []
|
|
die("kubectl get %s failed: %s" % (resource, err))
|
|
try:
|
|
return json.loads(proc.stdout).get("items", [])
|
|
except ValueError as exc:
|
|
die("could not parse kubectl output for %s: %s" % (resource, exc))
|
|
return []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# colour / output helpers
|
|
# --------------------------------------------------------------------------- #
|
|
class C(object):
|
|
"""ANSI colour codes; blanked out when colour is disabled."""
|
|
|
|
RESET = BOLD = DIM = RED = GREEN = YELLOW = BLUE = CYAN = ""
|
|
|
|
@classmethod
|
|
def enable(cls):
|
|
cls.RESET = "\033[0m"
|
|
cls.BOLD = "\033[1m"
|
|
cls.DIM = "\033[2m"
|
|
cls.RED = "\033[31m"
|
|
cls.GREEN = "\033[32m"
|
|
cls.YELLOW = "\033[33m"
|
|
cls.BLUE = "\033[34m"
|
|
cls.CYAN = "\033[36m"
|
|
|
|
|
|
def warn(msg):
|
|
print("%swarning:%s %s" % (C.YELLOW, C.RESET, msg), file=sys.stderr)
|
|
|
|
|
|
def die(msg):
|
|
print("%serror:%s %s" % (C.RED, C.RESET, msg), file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
_ANSI = re.compile(r"\033\[[0-9;]*m")
|
|
|
|
|
|
def render_table(headers, rows, indent=" "):
|
|
"""Render a simple fixed-width table. Colour codes are ignored for width."""
|
|
def vlen(s):
|
|
return len(_ANSI.sub("", s))
|
|
|
|
widths = [vlen(h) for h in headers]
|
|
for row in rows:
|
|
for i, cell in enumerate(row):
|
|
widths[i] = max(widths[i], vlen(cell))
|
|
|
|
def fmt(row):
|
|
cells = []
|
|
for i, cell in enumerate(row):
|
|
cells.append(cell + " " * (widths[i] - vlen(cell)))
|
|
return indent + " ".join(cells).rstrip()
|
|
|
|
sep = indent + " ".join("-" * w for w in widths)
|
|
out = [fmt(["%s%s%s" % (C.BOLD, h, C.RESET) for h in headers]), sep]
|
|
out += [fmt(r) for r in rows]
|
|
return "\n".join(out)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# domain types
|
|
# --------------------------------------------------------------------------- #
|
|
class Replica(object):
|
|
def __init__(self, name, node, health, has_data):
|
|
self.name = name # replica CR name
|
|
self.node = node # "" when Longhorn couldn't schedule it
|
|
self.health = health # display: healthy|stopped|rebuilding|failed|error|unknown
|
|
self.has_data = has_data # holds a usable copy (healthyAt set, not failed)
|
|
|
|
|
|
class Volume(object):
|
|
"""A Longhorn volume backing one PVC (or a non-Longhorn PVC's stand-in)."""
|
|
|
|
def __init__(self, pvc_ns, pvc_name, lh_name, storage_class, robustness,
|
|
state, desired_replicas, replicas=None, note=""):
|
|
self.pvc_ns = pvc_ns
|
|
self.pvc_name = pvc_name
|
|
self.lh_name = lh_name # Longhorn volume name (== bound PV name)
|
|
self.storage_class = storage_class
|
|
self.robustness = robustness # healthy|degraded|faulted|unknown|-
|
|
self.state = state # attached|detached|-
|
|
self.desired_replicas = desired_replicas
|
|
self.replicas = replicas if replicas is not None else []
|
|
self.note = note # e.g. non-Longhorn, unbound, missing
|
|
|
|
@property
|
|
def data_nodes(self):
|
|
"""Nodes that hold a usable copy of this volume's data.
|
|
|
|
Uses data validity (healthyAt && !failedAt), NOT engine-running state:
|
|
a cleanly-detached volume (stopped VM) still has good copies on its
|
|
replica nodes, so losing one of those nodes still loses data.
|
|
"""
|
|
return set(r.node for r in self.replicas if r.has_data and r.node)
|
|
|
|
@property
|
|
def all_nodes(self):
|
|
return set(r.node for r in self.replicas if r.node)
|
|
|
|
|
|
class VM(object):
|
|
def __init__(self, ns, name, status, node, pvcs):
|
|
self.ns = ns
|
|
self.name = name
|
|
self.status = status # Running | Stopped | ...
|
|
self.node = node # node the VMI runs on (or None)
|
|
self.pvcs = pvcs # list of (ns, name) referenced by this VM
|
|
|
|
@property
|
|
def running(self):
|
|
return self.status == "Running"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# building the model
|
|
# --------------------------------------------------------------------------- #
|
|
def classify_replica(rep):
|
|
"""Return (display-health, has_data) for a Longhorn Replica CR.
|
|
|
|
has_data is the data-safety signal: the replica holds a usable copy iff it
|
|
has been healthy (`healthyAt`) and hasn't failed (`failedAt`) -- true even
|
|
when the engine is stopped because the volume is detached.
|
|
"""
|
|
spec = rep.get("spec", {})
|
|
status = rep.get("status", {})
|
|
failed = bool(spec.get("failedAt"))
|
|
healthy_at = bool(spec.get("healthyAt"))
|
|
cur = status.get("currentState", "")
|
|
has_data = healthy_at and not failed
|
|
if failed:
|
|
disp = "failed"
|
|
elif cur == "running":
|
|
disp = HEALTHY if healthy_at else "rebuilding"
|
|
elif healthy_at:
|
|
disp = "stopped" # good copy, engine down (volume detached)
|
|
elif cur in ("stopped", "error"):
|
|
disp = cur
|
|
else:
|
|
disp = cur or "unknown"
|
|
return disp, has_data
|
|
|
|
|
|
def vm_referenced_pvcs(vm_spec_volumes, ns):
|
|
"""PVC (ns, name) pairs a set of KubeVirt `volumes[]` entries reference."""
|
|
out = []
|
|
for v in vm_spec_volumes or []:
|
|
pvc = v.get("persistentVolumeClaim")
|
|
if pvc and pvc.get("claimName"):
|
|
out.append((ns, pvc["claimName"]))
|
|
continue
|
|
# A DataVolume materialises a PVC with the same name in the same ns.
|
|
dv = v.get("dataVolume")
|
|
if dv and dv.get("name"):
|
|
out.append((ns, dv["name"]))
|
|
return out
|
|
|
|
|
|
def is_virt_launcher(pod):
|
|
labels = pod.get("metadata", {}).get("labels", {}) or {}
|
|
if labels.get("kubevirt.io") == "virt-launcher":
|
|
return True
|
|
for ref in pod.get("metadata", {}).get("ownerReferences", []) or []:
|
|
if ref.get("kind") == "VirtualMachineInstance":
|
|
return True
|
|
return False
|
|
|
|
|
|
def virt_launcher_vm_name(pod):
|
|
"""The VM/VMI name a virt-launcher pod belongs to (VMI name == VM name)."""
|
|
for ref in pod.get("metadata", {}).get("ownerReferences", []) or []:
|
|
if ref.get("kind") == "VirtualMachineInstance" and ref.get("name"):
|
|
return ref["name"]
|
|
return (pod.get("metadata", {}).get("labels", {}) or {}).get("vm.kubevirt.io/name", "")
|
|
|
|
|
|
def build_volume(pvc_ns, pvc_name, pvc_index, pv_index, lh_vol_index, replicas_by_vol):
|
|
pvc = pvc_index.get((pvc_ns, pvc_name))
|
|
if pvc is None:
|
|
return Volume(pvc_ns, pvc_name, None, "-", "-", "-", None, note="PVC not found")
|
|
sc = pvc.get("spec", {}).get("storageClassName", "") or "-"
|
|
pv_name = pvc.get("spec", {}).get("volumeName") or ""
|
|
phase = pvc.get("status", {}).get("phase", "")
|
|
|
|
if not pv_name:
|
|
return Volume(pvc_ns, pvc_name, None, sc, "-", "-", None,
|
|
note="unbound PVC (phase=%s)" % phase)
|
|
|
|
pv = pv_index.get(pv_name)
|
|
driver = ""
|
|
if pv:
|
|
driver = pv.get("spec", {}).get("csi", {}).get("driver", "")
|
|
if pv and driver and driver != LONGHORN_CSI_DRIVER:
|
|
# Non-Longhorn PV: its data lives wherever the PV binds (often one node).
|
|
node = _pv_pinned_node(pv)
|
|
vol = Volume(pvc_ns, pvc_name, pv_name, sc, "-", "-", None,
|
|
note="non-Longhorn (%s)" % driver)
|
|
if node:
|
|
vol.replicas.append(Replica(pv_name, node, HEALTHY, True))
|
|
return vol
|
|
|
|
# Longhorn: the volume CR name equals the bound PV name.
|
|
lh = lh_vol_index.get(pv_name)
|
|
if lh is None:
|
|
return Volume(pvc_ns, pvc_name, pv_name, sc, "-", "-", None,
|
|
note="no Longhorn volume CR")
|
|
robustness = lh.get("status", {}).get("robustness", "unknown") or "unknown"
|
|
state = lh.get("status", {}).get("state", "unknown") or "unknown"
|
|
desired = lh.get("spec", {}).get("numberOfReplicas")
|
|
vol = Volume(pvc_ns, pvc_name, pv_name, sc, robustness, state, desired)
|
|
for rep in replicas_by_vol.get(pv_name, []):
|
|
# An empty nodeID means Longhorn couldn't schedule this replica (e.g. a
|
|
# 3rd replica on a 2-node cluster). Keep it "" so it never counts as a
|
|
# node that holds data, and render it as "(unscheduled)".
|
|
node = rep.get("spec", {}).get("nodeID", "") or ""
|
|
disp, has_data = classify_replica(rep)
|
|
vol.replicas.append(Replica(
|
|
rep.get("metadata", {}).get("name", "?"), node, disp, has_data))
|
|
vol.replicas.sort(key=lambda r: (r.node, r.name))
|
|
return vol
|
|
|
|
|
|
def _pv_pinned_node(pv):
|
|
"""Best-effort node a non-Longhorn PV is pinned to (nodeAffinity)."""
|
|
terms = (pv.get("spec", {}).get("nodeAffinity", {})
|
|
.get("required", {}).get("nodeSelectorTerms", []))
|
|
for term in terms:
|
|
for expr in term.get("matchExpressions", []):
|
|
if expr.get("key") == "kubernetes.io/hostname" and expr.get("values"):
|
|
return expr["values"][0]
|
|
return None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# health formatting
|
|
# --------------------------------------------------------------------------- #
|
|
def colour_health(health):
|
|
good = (HEALTHY, "attached")
|
|
warnish = ("rebuilding", "degraded", "stopped", "detached")
|
|
bad = ("failed", "faulted", "error", "unknown")
|
|
if health in good:
|
|
return "%s%s%s" % (C.GREEN, health, C.RESET)
|
|
if health in warnish:
|
|
return "%s%s%s" % (C.YELLOW, health, C.RESET)
|
|
if health in bad:
|
|
return "%s%s%s" % (C.RED, health, C.RESET)
|
|
return health
|
|
|
|
|
|
def colour_status(status):
|
|
if status == "Running":
|
|
return "%s%s%s" % (C.GREEN, status, C.RESET)
|
|
if status in ("Stopped", "Paused"):
|
|
return "%s%s%s" % (C.YELLOW, status, C.RESET)
|
|
return "%s%s%s" % (C.CYAN, status, C.RESET)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# main report
|
|
# --------------------------------------------------------------------------- #
|
|
def main():
|
|
ap = argparse.ArgumentParser(
|
|
description="Report Longhorn disk health and assess node-removal safety.")
|
|
ap.add_argument("--kubectl", default="kubectl",
|
|
help="kubectl binary to use (default: kubectl on PATH)")
|
|
ap.add_argument("--kubeconfig", help="path to kubeconfig (else KUBECONFIG/default)")
|
|
ap.add_argument("--context", help="kube context to use")
|
|
ap.add_argument("--json", action="store_true", dest="as_json",
|
|
help="emit the structured model as JSON instead of a report")
|
|
ap.add_argument("--no-color", action="store_true", help="disable ANSI colour")
|
|
args = ap.parse_args()
|
|
|
|
if not args.no_color and sys.stdout.isatty():
|
|
C.enable()
|
|
|
|
kc = Kubectl(args.kubectl, args.kubeconfig, args.context)
|
|
|
|
# --- fetch everything up front ---------------------------------------- #
|
|
vms_raw = kc.get("virtualmachines.kubevirt.io")
|
|
vmis_raw = kc.get("virtualmachineinstances.kubevirt.io")
|
|
pvcs_raw = kc.get("persistentvolumeclaims")
|
|
pvs_raw = kc.get("persistentvolumes")
|
|
pods_raw = kc.get("pods")
|
|
lh_vols_raw = kc.get("volumes.longhorn.io", LONGHORN_NS)
|
|
lh_reps_raw = kc.get("replicas.longhorn.io", LONGHORN_NS)
|
|
|
|
pvc_index = dict(((p["metadata"]["namespace"], p["metadata"]["name"]), p) for p in pvcs_raw)
|
|
pv_index = dict((p["metadata"]["name"], p) for p in pvs_raw)
|
|
lh_vol_index = dict((v["metadata"]["name"], v) for v in lh_vols_raw)
|
|
replicas_by_vol = defaultdict(list)
|
|
for rep in lh_reps_raw:
|
|
vol_name = rep.get("spec", {}).get("volumeName")
|
|
if vol_name:
|
|
replicas_by_vol[vol_name].append(rep)
|
|
|
|
vmi_index = dict(((v["metadata"]["namespace"], v["metadata"]["name"]), v) for v in vmis_raw)
|
|
|
|
# --- assemble VMs ------------------------------------------------------ #
|
|
vms = []
|
|
for raw in vms_raw:
|
|
ns = raw["metadata"]["namespace"]
|
|
name = raw["metadata"]["name"]
|
|
status = raw.get("status", {}).get("printableStatus", "") or "Unknown"
|
|
spec_vols = (raw.get("spec", {}).get("template", {})
|
|
.get("spec", {}).get("volumes", []))
|
|
pvcs = vm_referenced_pvcs(spec_vols, ns)
|
|
node = None
|
|
vmi = vmi_index.get((ns, name))
|
|
if vmi:
|
|
node = vmi.get("status", {}).get("nodeName")
|
|
# Include hot-plugged / VMI-level volumes too.
|
|
for extra in vm_referenced_pvcs(vmi.get("spec", {}).get("volumes", []), ns):
|
|
if extra not in pvcs:
|
|
pvcs.append(extra)
|
|
vms.append(VM(ns, name, status, node, pvcs))
|
|
vms.sort(key=lambda v: (v.ns, v.name))
|
|
vm_by_key = dict(((vm.ns, vm.name), vm) for vm in vms)
|
|
|
|
# --- pods that mount PVCs --------------------------------------------- #
|
|
# A running VM's virt-launcher pod mounts every volume the VM actually uses
|
|
# -- including ones KubeVirt manages outside spec.template.spec.volumes (the
|
|
# persistent-state/EFI-vars PVC, hot-plugged disks). Attribute those to the
|
|
# owning VM so they show under the VM, not as "detached". Other pods are
|
|
# real non-VM consumers.
|
|
pods_by_pvc = defaultdict(list)
|
|
for pod in pods_raw:
|
|
pns = pod["metadata"]["namespace"]
|
|
pnode = pod.get("spec", {}).get("nodeName", "") or "?"
|
|
pname = pod["metadata"]["name"]
|
|
claims = [v["persistentVolumeClaim"]["claimName"]
|
|
for v in pod.get("spec", {}).get("volumes", []) or []
|
|
if v.get("persistentVolumeClaim", {}).get("claimName")]
|
|
if is_virt_launcher(pod):
|
|
vm = vm_by_key.get((pns, virt_launcher_vm_name(pod)))
|
|
if vm is not None:
|
|
for c in claims:
|
|
if (pns, c) not in vm.pvcs:
|
|
vm.pvcs.append((pns, c))
|
|
continue # a VM's own pod; its PVCs belong to the VM section
|
|
for c in claims:
|
|
pods_by_pvc[(pns, c)].append((pname, pnode))
|
|
|
|
# Set of PVCs owned by a VM (after folding in virt-launcher-mounted ones).
|
|
vm_pvc_set = set(p for vm in vms for p in vm.pvcs)
|
|
|
|
# --- build volumes for every PVC -------------------------------------- #
|
|
def vol_for(ns_name):
|
|
return build_volume(ns_name[0], ns_name[1], pvc_index, pv_index,
|
|
lh_vol_index, replicas_by_vol)
|
|
|
|
all_pvc_keys = set(pvc_index.keys()) | vm_pvc_set
|
|
volumes = dict((k, vol_for(k)) for k in all_pvc_keys)
|
|
|
|
# --- node inventory ---------------------------------------------------- #
|
|
nodes = sorted(set(r.node for v in volumes.values() for r in v.replicas if r.node))
|
|
|
|
if args.as_json:
|
|
emit_json(vms, volumes, pods_by_pvc, vm_pvc_set, nodes)
|
|
return
|
|
|
|
print_node_safety(nodes, vms, volumes)
|
|
print_vm_detail(vms, volumes)
|
|
print_unattached(volumes, vm_pvc_set, pods_by_pvc)
|
|
|
|
|
|
def _risk_line(vol):
|
|
lh = vol.lh_name or "-"
|
|
hn = ", ".join(sorted(vol.data_nodes)) or "(none)"
|
|
return ("%s/%s [%s] robustness=%s usable-copies-on: %s"
|
|
% (vol.pvc_ns, vol.pvc_name, lh, vol.robustness, hn))
|
|
|
|
|
|
def print_node_safety(nodes, vms, volumes):
|
|
print("%s=== NODE REMOVAL SAFETY ===%s" % (C.BOLD, C.RESET))
|
|
if not nodes:
|
|
print(" no replica-bearing nodes found (no Longhorn volumes?).\n")
|
|
return
|
|
|
|
# Which VM (running?) owns each PVC, for severity.
|
|
running_pvcs = set(p for vm in vms if vm.running for p in vm.pvcs)
|
|
|
|
# Volumes already unavailable (no usable copy on any node) -- independent of
|
|
# which node you remove. Reported once, globally, not per node.
|
|
broken = [vol for vol in volumes.values() if vol.replicas and not vol.data_nodes]
|
|
|
|
for node in nodes:
|
|
crit, warns = [], []
|
|
for key, vol in volumes.items():
|
|
if not vol.replicas or not vol.data_nodes:
|
|
continue
|
|
if vol.data_nodes == set([node]):
|
|
# Removing this node drops the last usable copy.
|
|
(crit if key in running_pvcs else warns).append(vol)
|
|
|
|
if not crit and not warns:
|
|
print(" %sSAFE%s %s%s%s: every volume keeps a usable copy on another node."
|
|
% (C.GREEN, C.RESET, C.BOLD, node, C.RESET))
|
|
continue
|
|
|
|
verdict = ("%sUNSAFE%s" % (C.RED, C.RESET)) if crit else ("%sCAUTION%s" % (C.YELLOW, C.RESET))
|
|
print(" %s %s%s%s:" % (verdict, C.BOLD, node, C.RESET))
|
|
if crit:
|
|
print(" %srunning-VM data lost if removed (%d):%s" % (C.RED, len(crit), C.RESET))
|
|
for vol in crit:
|
|
print(" - %s" % _risk_line(vol))
|
|
if warns:
|
|
print(" %sidle/non-VM data lost if removed (%d):%s" % (C.YELLOW, len(warns), C.RESET))
|
|
for vol in warns:
|
|
print(" - %s" % _risk_line(vol))
|
|
|
|
if broken:
|
|
print("\n %sAlready unavailable%s (no usable copy on ANY node -- fix before draining anything):"
|
|
% (C.RED, C.RESET))
|
|
for vol in broken:
|
|
print(" - %s" % _risk_line(vol))
|
|
print()
|
|
|
|
|
|
def _pvc_block(vol):
|
|
head = (" PVC %s%s/%s%s vol=%s sc=%s"
|
|
% (C.CYAN, vol.pvc_ns, vol.pvc_name, C.RESET, vol.lh_name or "-", vol.storage_class))
|
|
meta = []
|
|
if vol.robustness != "-":
|
|
meta.append("robustness=%s" % colour_health(vol.robustness))
|
|
if vol.state != "-":
|
|
meta.append("state=%s" % colour_health(vol.state))
|
|
if vol.desired_replicas is not None:
|
|
meta.append("desired-replicas=%s" % vol.desired_replicas)
|
|
if meta:
|
|
head += " (" + " ".join(meta) + ")"
|
|
if vol.note:
|
|
head += " %s[%s]%s" % (C.DIM, vol.note, C.RESET)
|
|
lines = [head]
|
|
if vol.replicas:
|
|
rows = [[r.name, r.node or "(unscheduled)", colour_health(r.health)]
|
|
for r in vol.replicas]
|
|
lines.append(render_table(["REPLICA", "NODE", "HEALTH"], rows, indent=" "))
|
|
elif not vol.note:
|
|
lines.append(" %s(no replicas reported)%s" % (C.DIM, C.RESET))
|
|
return "\n".join(lines)
|
|
|
|
|
|
def print_vm_detail(vms, volumes):
|
|
print("%s=== VIRTUAL MACHINES ===%s" % (C.BOLD, C.RESET))
|
|
if not vms:
|
|
print(" no VirtualMachines found.\n")
|
|
return
|
|
for vm in vms:
|
|
node = (" node=%s" % vm.node) if vm.node else ""
|
|
print("\n%sVM %s/%s%s [%s]%s"
|
|
% (C.BOLD, vm.ns, vm.name, C.RESET, colour_status(vm.status), node))
|
|
if not vm.pvcs:
|
|
print(" %s(no PVC-backed volumes)%s" % (C.DIM, C.RESET))
|
|
continue
|
|
for key in vm.pvcs:
|
|
print(_pvc_block(volumes[key]))
|
|
print()
|
|
|
|
|
|
def print_unattached(volumes, vm_pvc_set, pods_by_pvc):
|
|
non_vm = dict((k, v) for k, v in volumes.items() if k not in vm_pvc_set)
|
|
pod_attached = dict((k, v) for k, v in non_vm.items() if pods_by_pvc.get(k))
|
|
detached = dict((k, v) for k, v in non_vm.items() if not pods_by_pvc.get(k))
|
|
|
|
print("%s=== PVCs NOT ATTACHED TO A VM ===%s" % (C.BOLD, C.RESET))
|
|
if not non_vm:
|
|
print(" none: every PVC is attached to a VM.\n")
|
|
return
|
|
|
|
if pod_attached:
|
|
print("\n %sAttached to non-VM pod(s):%s" % (C.BOLD, C.RESET))
|
|
for key in sorted(pod_attached):
|
|
consumers = ", ".join("%s (node=%s)" % (n, nd) for n, nd in pods_by_pvc[key])
|
|
print(" used by: %s" % consumers)
|
|
print(_pvc_block(pod_attached[key]))
|
|
|
|
if detached:
|
|
print("\n %sCompletely detached (no consumer):%s" % (C.BOLD, C.RESET))
|
|
for key in sorted(detached):
|
|
print(_pvc_block(detached[key]))
|
|
print()
|
|
|
|
|
|
def emit_json(vms, volumes, pods_by_pvc, vm_pvc_set, nodes):
|
|
def vol_dict(v):
|
|
return {
|
|
"pvc_namespace": v.pvc_ns,
|
|
"pvc_name": v.pvc_name,
|
|
"longhorn_volume": v.lh_name,
|
|
"storage_class": v.storage_class,
|
|
"robustness": v.robustness,
|
|
"state": v.state,
|
|
"desired_replicas": v.desired_replicas,
|
|
"note": v.note or None,
|
|
"replicas": [
|
|
{"name": r.name, "node": r.node, "health": r.health, "has_data": r.has_data}
|
|
for r in v.replicas
|
|
],
|
|
"usable_copy_nodes": sorted(v.data_nodes),
|
|
}
|
|
|
|
running_pvcs = set(p for vm in vms if vm.running for p in vm.pvcs)
|
|
node_safety = {}
|
|
for node in nodes:
|
|
crit, warns = [], []
|
|
for key, vol in volumes.items():
|
|
if vol.replicas and vol.data_nodes == set([node]):
|
|
(crit if key in running_pvcs else warns).append(
|
|
"%s/%s" % (vol.pvc_ns, vol.pvc_name))
|
|
node_safety[node] = {
|
|
"safe": not crit and not warns,
|
|
"running_vm_data_only_here": crit,
|
|
"other_data_only_here": warns,
|
|
}
|
|
|
|
out = {
|
|
"nodes": nodes,
|
|
"node_safety": node_safety,
|
|
"vms": [
|
|
{
|
|
"namespace": vm.ns, "name": vm.name, "status": vm.status,
|
|
"running": vm.running, "node": vm.node,
|
|
"pvcs": ["%s/%s" % (ns, n) for ns, n in vm.pvcs],
|
|
}
|
|
for vm in vms
|
|
],
|
|
"volumes": [vol_dict(v) for v in volumes.values()],
|
|
"unattached": dict(
|
|
("%s/%s" % (k[0], k[1]),
|
|
{"pods": pods_by_pvc.get(k, []), "detached": not pods_by_pvc.get(k)})
|
|
for k in volumes if k not in vm_pvc_set
|
|
),
|
|
}
|
|
print(json.dumps(out, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|