add throttle scanner

cgroup_throttle_scan.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+Scan the cgroup tree for CPU (CFS bandwidth) throttling.
+
+Pass 1: every cgroup that has *ever* been throttled (throttled_time > 0).
+Pass 2: after an interval, every cgroup that throttled *during* the window
+        (delta throttled_time > 0).
+
+Targets cgroups v1 (throttled_time, nanoseconds) but also understands the
+v2 field name (throttled_usec) so it degrades gracefully on unified hierarchies.
+"""
+
+import argparse
+import os
+import sys
+import time
+
+DEFAULT_ROOT = "/sys/fs/cgroup"
+
+
+def parse_cpu_stat(path):
+    """Parse a cpu.stat file into a dict of {key: int}, or None if unreadable."""
+    stats = {}
+    try:
+        with open(path) as f:
+            for line in f:
+                parts = line.split()
+                if len(parts) == 2:
+                    try:
+                        stats[parts[0]] = int(parts[1])
+                    except ValueError:
+                        pass
+    except OSError:
+        return None
+    return stats or None
+
+
+def throttled_ns(stats):
+    """Return throttled time in nanoseconds regardless of v1/v2 field name."""
+    if "throttled_time" in stats:            # cgroup v1: nanoseconds
+        return stats["throttled_time"]
+    if "throttled_usec" in stats:            # cgroup v2: microseconds
+        return stats["throttled_usec"] * 1000
+    return 0
+
+
+def scan(root):
+    """
+    Walk the cgroup tree, returning {cgroup_path: stats_dict} for every
+    directory that exposes a cpu.stat file.
+    """
+    results = {}
+    for dirpath, _dirnames, filenames in os.walk(root):
+        if "cpu.stat" in filenames:
+            stats = parse_cpu_stat(os.path.join(dirpath, "cpu.stat"))
+            if stats is not None:
+                results[dirpath] = stats
+    return results
+
+
+def human_ns(ns):
+    if ns >= 1_000_000_000:
+        return f"{ns / 1_000_000_000:.2f}s"
+    if ns >= 1_000_000:
+        return f"{ns / 1_000_000:.1f}ms"
+    if ns >= 1_000:
+        return f"{ns / 1_000:.1f}us"
+    return f"{ns}ns"
+
+
+def display_name(path, root):
+    """Strip the mount prefix so output shows the logical cgroup path."""
+    rel = os.path.relpath(path, root)
+    return "/" if rel == "." else "/" + rel
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--root", default=DEFAULT_ROOT,
+                    help=f"cgroup mount root to scan (default: {DEFAULT_ROOT})")
+    ap.add_argument("--interval", type=float, default=5.0,
+                    help="seconds to wait between the two passes (default: 5)")
+    args = ap.parse_args()
+
+    if not os.path.isdir(args.root):
+        sys.exit(f"error: {args.root} is not a directory")
+
+    # ---- Pass 1: lifetime throttling ------------------------------------
+    first = scan(args.root)
+    ever = sorted(
+        ((p, s) for p, s in first.items() if throttled_ns(s) > 0),
+        key=lambda kv: throttled_ns(kv[1]),
+        reverse=True,
+    )
+
+    print(f"=== Cgroups that have EVER throttled ({len(ever)}) ===")
+    if not ever:
+        print("  (none)")
+    else:
+        for path, stats in ever:
+            t = throttled_ns(stats)
+            n = stats.get("nr_throttled", 0)
+            print(f"  {human_ns(t):>9}  nr_throttled={n:<8} {display_name(path, args.root)}")
+
+    # ---- Pass 2: throttling during the interval -------------------------
+    print(f"\nSleeping {args.interval:g}s...\n")
+    time.sleep(args.interval)
+    second = scan(args.root)
+
+    deltas = []
+    for path, stats in second.items():
+        base = throttled_ns(first[path]) if path in first else 0
+        d_ns = throttled_ns(stats) - base
+        if d_ns > 0:
+            base_n = first[path].get("nr_throttled", 0) if path in first else 0
+            d_n = stats.get("nr_throttled", 0) - base_n
+            deltas.append((path, d_ns, d_n))
+
+    deltas.sort(key=lambda x: x[1], reverse=True)
+
+    print(f"=== Cgroups that throttled in the last {args.interval:g}s ({len(deltas)}) ===")
+    if not deltas:
+        print("  (none)")
+    else:
+        for path, d_ns, d_n in deltas:
+            print(f"  +{human_ns(d_ns):>9}  +{d_n:<6} throttled  {display_name(path, args.root)}")
+
+
+if __name__ == "__main__":
+    main()