Resource Utilization
CPU, memory, disk I/O, and network throughput analysis for PQ Devnet clients.
This notebook examines container-level resource usage using cAdvisor metrics:
- CPU usage (cores) per client
- Memory working set and RSS per client
- Disk read/write throughput
- Disk usage over time
- Network receive/transmit throughput
Show code
import json
from pathlib import Path
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from IPython.display import HTML, display
# Set default renderer for static HTML output
import plotly.io as pio
pio.renderers.default = "notebook"
Show code
# Resolve devnet_id
DATA_DIR = Path("../data")
if devnet_id is None:
# Use latest devnet from manifest
devnets_path = DATA_DIR / "devnets.json"
if devnets_path.exists():
with open(devnets_path) as f:
devnets = json.load(f).get("devnets", [])
if devnets:
devnet_id = devnets[-1]["id"] # Latest
print(f"Using latest devnet: {devnet_id}")
else:
raise ValueError("No devnets.json found. Run 'just detect-devnets' first.")
DEVNET_DIR = DATA_DIR / devnet_id
print(f"Loading data from: {DEVNET_DIR}")
Show code
# Load devnet metadata
with open(DATA_DIR / "devnets.json") as f:
devnets_data = json.load(f)
devnet_info = next((d for d in devnets_data["devnets"] if d["id"] == devnet_id), None)
if devnet_info:
print(f"Devnet: {devnet_info['id']}")
print(f"Duration: {devnet_info['duration_hours']:.1f} hours")
print(f"Time: {devnet_info['start_time']} to {devnet_info['end_time']}")
print(f"Slots: {devnet_info['start_slot']} \u2192 {devnet_info['end_slot']}")
print(f"Clients: {', '.join(devnet_info['clients'])}")
Show code
def format_bytes(val: float) -> str:
"""Format bytes to human-readable units."""
for unit in ["B", "KB", "MB", "GB", "TB"]:
if abs(val) < 1024:
return f"{val:.1f} {unit}"
val /= 1024
return f"{val:.1f} PB"
def format_bytes_per_sec(val: float) -> str:
"""Format bytes/s to human-readable units."""
return format_bytes(val) + "/s"
Load Data¶
Show code
# Load container resource data
data_files = {
"cpu": "container_cpu.parquet",
"memory": "container_memory.parquet",
"disk_io": "container_disk_io.parquet",
"network": "container_network.parquet",
}
# Infrastructure containers irrelevant to devnet client analysis
EXCLUDED_CONTAINERS = {"unknown", "cadvisor", "prometheus", "promtail", "node-exporter", "node_exporter", "grafana"}
# Aggregation strategy per data type:
# - cpu/memory: max (gauge-like, take the active container's value)
# - disk_io/network: sum (per-device/interface rates should be summed)
AGG_STRATEGY = {"cpu": "max", "memory": "max", "disk_io": "sum", "network": "sum"}
# Group-by columns per data type (all have container+timestamp, some have metric)
GROUP_COLS = {
"cpu": ["container", "timestamp"],
"memory": ["container", "metric", "timestamp"],
"disk_io": ["container", "metric", "timestamp"],
"network": ["container", "metric", "timestamp"],
}
dfs = {}
for key, filename in data_files.items():
path = DEVNET_DIR / filename
if path.exists():
df = pd.read_parquet(path)
df = df[~df["container"].isin(EXCLUDED_CONTAINERS)]
# Deduplicate: multiple Prometheus series (interfaces, devices, container
# IDs after restarts) can produce duplicate rows per container+timestamp.
df = df.groupby(GROUP_COLS[key], as_index=False)["value"].agg(AGG_STRATEGY[key])
dfs[key] = df
print(f"{key}: {len(df)} records, containers: {df['container'].nunique()}")
else:
dfs[key] = pd.DataFrame()
print(f"{key}: no data (file not found)")
# Unified container list: use devnet_info["clients"] which already contains
# full names with instance suffixes (e.g., "ream_0", "ream_1").
all_containers = sorted(devnet_info["clients"])
n_cols = min(len(all_containers), 2)
n_rows = -(-len(all_containers) // n_cols)
print(f"\nAll containers ({len(all_containers)}): {all_containers}")
CPU Usage¶
CPU cores used per container over time, derived from rate(container_cpu_usage_seconds_total[5m]).
Show code
cpu_df = dfs["cpu"]
if cpu_df.empty:
print("No CPU data available")
else:
fig = make_subplots(
rows=n_rows, cols=n_cols,
subplot_titles=all_containers,
vertical_spacing=0.12 / max(n_rows - 1, 1) * 2,
horizontal_spacing=0.08,
)
for i, container in enumerate(all_containers):
row = i // n_cols + 1
col = i % n_cols + 1
cdf = cpu_df[cpu_df["container"] == container].sort_values("timestamp")
if not cdf.empty:
fig.add_trace(
go.Scatter(
x=cdf["timestamp"], y=cdf["value"],
name=container, showlegend=False,
line=dict(color="#636EFA"),
),
row=row, col=col,
)
else:
fig.add_trace(
go.Scatter(x=[None], y=[None], showlegend=False, hoverinfo='skip'),
row=row, col=col,
)
_n = (row - 1) * n_cols + col
_s = "" if _n == 1 else str(_n)
fig.add_annotation(
text="No data available",
xref=f"x{_s} domain", yref=f"y{_s} domain",
x=0.5, y=0.5,
showarrow=False,
font=dict(size=12, color="#999"),
)
fig.update_yaxes(title_text="CPU (cores)", row=row, col=col)
fig.update_layout(
title="CPU Usage per Container",
height=270 * n_rows,
)
fig.show()