132 lines
4.2 KiB
Python
132 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Scan the cgroup tree for CPU (CFS bandwidth) throttling.
|
|
|
|
Pass 1: every cgroup that has *ever* been throttled (throttled_time > 0).
|
|
Pass 2: after an interval, every cgroup that throttled *during* the window
|
|
(delta throttled_time > 0).
|
|
|
|
Targets cgroups v1 (throttled_time, nanoseconds) but also understands the
|
|
v2 field name (throttled_usec) so it degrades gracefully on unified hierarchies.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
DEFAULT_ROOT = "/sys/fs/cgroup"
|
|
|
|
|
|
def parse_cpu_stat(path):
|
|
"""Parse a cpu.stat file into a dict of {key: int}, or None if unreadable."""
|
|
stats = {}
|
|
try:
|
|
with open(path) as f:
|
|
for line in f:
|
|
parts = line.split()
|
|
if len(parts) == 2:
|
|
try:
|
|
stats[parts[0]] = int(parts[1])
|
|
except ValueError:
|
|
pass
|
|
except OSError:
|
|
return None
|
|
return stats or None
|
|
|
|
|
|
def throttled_ns(stats):
|
|
"""Return throttled time in nanoseconds regardless of v1/v2 field name."""
|
|
if "throttled_time" in stats: # cgroup v1: nanoseconds
|
|
return stats["throttled_time"]
|
|
if "throttled_usec" in stats: # cgroup v2: microseconds
|
|
return stats["throttled_usec"] * 1000
|
|
return 0
|
|
|
|
|
|
def scan(root):
|
|
"""
|
|
Walk the cgroup tree, returning {cgroup_path: stats_dict} for every
|
|
directory that exposes a cpu.stat file.
|
|
"""
|
|
results = {}
|
|
for dirpath, _dirnames, filenames in os.walk(root):
|
|
if "cpu.stat" in filenames:
|
|
stats = parse_cpu_stat(os.path.join(dirpath, "cpu.stat"))
|
|
if stats is not None:
|
|
results[dirpath] = stats
|
|
return results
|
|
|
|
|
|
def human_ns(ns):
|
|
if ns >= 1_000_000_000:
|
|
return f"{ns / 1_000_000_000:.2f}s"
|
|
if ns >= 1_000_000:
|
|
return f"{ns / 1_000_000:.1f}ms"
|
|
if ns >= 1_000:
|
|
return f"{ns / 1_000:.1f}us"
|
|
return f"{ns}ns"
|
|
|
|
|
|
def display_name(path, root):
|
|
"""Strip the mount prefix so output shows the logical cgroup path."""
|
|
rel = os.path.relpath(path, root)
|
|
return "/" if rel == "." else "/" + rel
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--root", default=DEFAULT_ROOT,
|
|
help=f"cgroup mount root to scan (default: {DEFAULT_ROOT})")
|
|
ap.add_argument("--interval", type=float, default=5.0,
|
|
help="seconds to wait between the two passes (default: 5)")
|
|
args = ap.parse_args()
|
|
|
|
if not os.path.isdir(args.root):
|
|
sys.exit(f"error: {args.root} is not a directory")
|
|
|
|
# ---- Pass 1: lifetime throttling ------------------------------------
|
|
first = scan(args.root)
|
|
ever = sorted(
|
|
((p, s) for p, s in first.items() if throttled_ns(s) > 0),
|
|
key=lambda kv: throttled_ns(kv[1]),
|
|
reverse=True,
|
|
)
|
|
|
|
print(f"=== Cgroups that have EVER throttled ({len(ever)}) ===")
|
|
if not ever:
|
|
print(" (none)")
|
|
else:
|
|
for path, stats in ever:
|
|
t = throttled_ns(stats)
|
|
n = stats.get("nr_throttled", 0)
|
|
print(f" {human_ns(t):>9} nr_throttled={n:<8} {display_name(path, args.root)}")
|
|
|
|
# ---- Pass 2: throttling during the interval -------------------------
|
|
print(f"\nSleeping {args.interval:g}s...\n")
|
|
time.sleep(args.interval)
|
|
second = scan(args.root)
|
|
|
|
deltas = []
|
|
for path, stats in second.items():
|
|
base = throttled_ns(first[path]) if path in first else 0
|
|
d_ns = throttled_ns(stats) - base
|
|
if d_ns > 0:
|
|
base_n = first[path].get("nr_throttled", 0) if path in first else 0
|
|
d_n = stats.get("nr_throttled", 0) - base_n
|
|
deltas.append((path, d_ns, d_n))
|
|
|
|
deltas.sort(key=lambda x: x[1], reverse=True)
|
|
|
|
print(f"=== Cgroups that throttled in the last {args.interval:g}s ({len(deltas)}) ===")
|
|
if not deltas:
|
|
print(" (none)")
|
|
else:
|
|
for path, d_ns, d_n in deltas:
|
|
print(f" +{human_ns(d_ns):>9} +{d_n:<6} throttled {display_name(path, args.root)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|