PQ Devnet 2026-07-15 Latest

Resource Utilization

CPU, memory, disk I/O, and network throughput analysis for PQ Devnet clients.

This notebook examines container-level resource usage using cAdvisor metrics:

  • CPU usage (cores) per client
  • Memory working set and RSS per client
  • Disk read/write throughput
  • Disk usage over time
  • Network receive/transmit throughput
Show code
import json
from pathlib import Path

import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from IPython.display import HTML, display

# Set default renderer for static HTML output
import plotly.io as pio
pio.renderers.default = "notebook"
Show code
# Resolve devnet_id
DATA_DIR = Path("../data")

if devnet_id is None:
    # Use latest devnet from manifest
    devnets_path = DATA_DIR / "devnets.json"
    if devnets_path.exists():
        with open(devnets_path) as f:
            devnets = json.load(f).get("devnets", [])
        if devnets:
            devnet_id = devnets[-1]["id"]  # Latest
            print(f"Using latest devnet: {devnet_id}")
    else:
        raise ValueError("No devnets.json found. Run 'just detect-devnets' first.")

DEVNET_DIR = DATA_DIR / devnet_id
print(f"Loading data from: {DEVNET_DIR}")
Loading data from: ../data/pqdevnet-20260715T0251Z
Show code
# Load devnet metadata
with open(DATA_DIR / "devnets.json") as f:
    devnets_data = json.load(f)
    devnet_info = next((d for d in devnets_data["devnets"] if d["id"] == devnet_id), None)

if devnet_info:
    print(f"Devnet: {devnet_info['id']}")
    print(f"Duration: {devnet_info['duration_hours']:.1f} hours")
    print(f"Time: {devnet_info['start_time']} to {devnet_info['end_time']}")
    print(f"Slots: {devnet_info['start_slot']} \u2192 {devnet_info['end_slot']}")
    print(f"Clients: {', '.join(devnet_info['clients'])}")
Devnet: pqdevnet-20260715T0251Z
Duration: 1.3 hours
Time: 2026-07-15T02:51:46+00:00 to 2026-07-15T04:08:24+00:00
Slots: 5316 → 6716
Clients: buildx_buildkit_multiarch0, ethlambda_0, ethlambda_10, ethlambda_2, ethlambda_4, ethlambda_7, ethlambda_8, gean_0, gean_1, gean_2, gean_3, grandine_0, grandine_2, grandine_3, grandine_4, grandine_5, grandine_6, lantern_0, lantern_1, lantern_2, lantern_3, ream_0, ream_1, ream_2, ream_3, ream_4, ream_6, zeam_0, zeam_1, zeam_2, zeam_4, zeam_5, zeam_6
Show code
def format_bytes(val: float) -> str:
    """Format bytes to human-readable units."""
    for unit in ["B", "KB", "MB", "GB", "TB"]:
        if abs(val) < 1024:
            return f"{val:.1f} {unit}"
        val /= 1024
    return f"{val:.1f} PB"


def format_bytes_per_sec(val: float) -> str:
    """Format bytes/s to human-readable units."""
    return format_bytes(val) + "/s"

Load Data¶

Show code
# Load container resource data
data_files = {
    "cpu": "container_cpu.parquet",
    "memory": "container_memory.parquet",
    "disk_io": "container_disk_io.parquet",
    "network": "container_network.parquet",
}

# Infrastructure containers irrelevant to devnet client analysis
EXCLUDED_CONTAINERS = {"unknown", "cadvisor", "prometheus", "promtail", "node-exporter", "node_exporter", "grafana"}

# Aggregation strategy per data type:
# - cpu/memory: max (gauge-like, take the active container's value)
# - disk_io/network: sum (per-device/interface rates should be summed)
AGG_STRATEGY = {"cpu": "max", "memory": "max", "disk_io": "sum", "network": "sum"}

# Group-by columns per data type (all have container+timestamp, some have metric)
GROUP_COLS = {
    "cpu": ["container", "timestamp"],
    "memory": ["container", "metric", "timestamp"],
    "disk_io": ["container", "metric", "timestamp"],
    "network": ["container", "metric", "timestamp"],
}

dfs = {}
for key, filename in data_files.items():
    path = DEVNET_DIR / filename
    if path.exists():
        df = pd.read_parquet(path)
        df = df[~df["container"].isin(EXCLUDED_CONTAINERS)]
        # Deduplicate: multiple Prometheus series (interfaces, devices, container
        # IDs after restarts) can produce duplicate rows per container+timestamp.
        df = df.groupby(GROUP_COLS[key], as_index=False)["value"].agg(AGG_STRATEGY[key])
        dfs[key] = df
        print(f"{key}: {len(df)} records, containers: {df['container'].nunique()}")
    else:
        dfs[key] = pd.DataFrame()
        print(f"{key}: no data (file not found)")

# Unified container list: use devnet_info["clients"] which already contains
# full names with instance suffixes (e.g., "ream_0", "ream_1").
all_containers = sorted(devnet_info["clients"])
n_cols = min(len(all_containers), 2)
n_rows = -(-len(all_containers) // n_cols)
print(f"\nAll containers ({len(all_containers)}): {all_containers}")
cpu: 528 records, containers: 33
memory: 5082 records, containers: 33
disk_io: 864 records, containers: 27
network: 1056 records, containers: 33

All containers (33): ['buildx_buildkit_multiarch0', 'ethlambda_0', 'ethlambda_10', 'ethlambda_2', 'ethlambda_4', 'ethlambda_7', 'ethlambda_8', 'gean_0', 'gean_1', 'gean_2', 'gean_3', 'grandine_0', 'grandine_2', 'grandine_3', 'grandine_4', 'grandine_5', 'grandine_6', 'lantern_0', 'lantern_1', 'lantern_2', 'lantern_3', 'ream_0', 'ream_1', 'ream_2', 'ream_3', 'ream_4', 'ream_6', 'zeam_0', 'zeam_1', 'zeam_2', 'zeam_4', 'zeam_5', 'zeam_6']

CPU Usage¶

CPU cores used per container over time, derived from rate(container_cpu_usage_seconds_total[5m]).

Show code
cpu_df = dfs["cpu"]

if cpu_df.empty:
    print("No CPU data available")
else:
    fig = make_subplots(
        rows=n_rows, cols=n_cols,
        subplot_titles=all_containers,
        vertical_spacing=0.12 / max(n_rows - 1, 1) * 2,
        horizontal_spacing=0.08,
    )

    for i, container in enumerate(all_containers):
        row = i // n_cols + 1
        col = i % n_cols + 1
        cdf = cpu_df[cpu_df["container"] == container].sort_values("timestamp")
        if not cdf.empty:
            fig.add_trace(
                go.Scatter(
                    x=cdf["timestamp"], y=cdf["value"],
                    name=container, showlegend=False,
                    line=dict(color="#636EFA"),
                ),
                row=row, col=col,
            )
        else:
            fig.add_trace(
                go.Scatter(x=[None], y=[None], showlegend=False, hoverinfo='skip'),
                row=row, col=col,
            )
            _n = (row - 1) * n_cols + col
            _s = "" if _n == 1 else str(_n)
            fig.add_annotation(
                text="No data available",
                xref=f"x{_s} domain", yref=f"y{_s} domain",
                x=0.5, y=0.5,
                showarrow=False,
                font=dict(size=12, color="#999"),
            )
        fig.update_yaxes(title_text="CPU (cores)", row=row, col=col)

    fig.update_layout(
        title="CPU Usage per Container",
        height=270 * n_rows,
    )
    fig.show()