Experiment tracking and reproducibility

TensorBoard integration, logging metrics and sample predictions, seeding, deterministic flags, and comparing runs so the comparison is fair.

Logging to TensorBoard

import torch
from torch.utils.tensorboard import SummaryWriter

writer = SummaryWriter(log_dir="runs/resnet18-lr3e4-seed0")

for epoch in range(epochs):
    model.train()
    for step, (x, y) in enumerate(train_loader):
        loss = train_step(x, y)
        global_step = epoch * len(train_loader) + step
        writer.add_scalar("loss/train_step", loss, global_step)

    val_loss, val_acc = evaluate(model, val_loader)
    writer.add_scalar("loss/val", val_loss, epoch)
    writer.add_scalar("acc/val", val_acc, epoch)
    writer.add_scalar("lr", opt.param_groups[0]["lr"], epoch)

    # weight and gradient distributions
    for name, p in model.named_parameters():
        writer.add_histogram(f"weights/{name}", p.detach().cpu(), epoch)
        if p.grad is not None:
            writer.add_histogram(f"grads/{name}", p.grad.detach().cpu(), epoch)

    # a grid of predictions to see what the model actually got wrong
    model.eval()
    with torch.no_grad():
        images, targets = next(iter(val_loader))
        preds = model(images.cuda()).argmax(dim=1).cpu()
    writer.add_images("val/images", images[:8], epoch)

writer.flush()
writer.close()

• Put the configuration in the run name or a text summary: resnet18-lr3e4-bs64-seed0 is interpretable a month later, run_2026_09_18 is not.
• Log per-batch loss sparsely. Writing every step for a long run produces gigabytes of event files and slows training measurably.
• TensorBoard scalars alone are not tracking. Also write the resolved config, the git commit, and the dataset version so a run can be reproduced.
• The add_images call needs a (N, C, H, W) float tensor with values in [0, 1]. Un-normalise before logging, or every image looks black.

Seeding and determinism

import os
import random
import numpy as np
import torch

def set_seed(seed=0, deterministic=True):
    os.environ["PYTHONHASHSEED"] = str(seed)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    if deterministic:
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        # some CUDA reductions are non-deterministic: opt in to error instead
        os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
        torch.use_deterministic_algorithms(True, warn_only=True)

set_seed(0)

# dataloader workers need their own seeds
def seed_worker(worker_id):
    worker_seed = torch.initial_seed() % 2 ** 32
    np.random.seed(worker_seed)
    random.seed(worker_seed)

g = torch.Generator()
g.manual_seed(0)
loader = torch.utils.data.DataLoader(
    dataset, batch_size=32, shuffle=True,
    worker_init_fn=seed_worker, generator=g, num_workers=4,
)

# save everything needed to resume
torch.save({
    "model": model.state_dict(),
    "optimizer": opt.state_dict(),
    "scheduler": sched.state_dict(),
    "epoch": epoch,
    "seed": 0,
    "config": config,
}, "checkpoint.pt")

Comparing runs honestly

# a small helper that makes comparisons explicit and cheap
import json
from pathlib import Path

def log_run(store="runs/index.jsonl", **fields):
    Path(store).parent.mkdir(parents=True, exist_ok=True)
    with open(store, "a", encoding="utf-8") as fh:
        fh.write(json.dumps(fields, default=str) + "
")

log_run(
    run="resnet18-lr3e4-bs64-seed0",
    base_lr=3e-4, batch_size=64, epochs=20, seed=0,
    val_acc=0.9123, val_loss=0.2714,
    git_commit="a1b2c3d", dataset="imagenet-subset-v3",
    notes="baseline, no fine-tuning",
)

# re-run the same config with three seeds before believing a delta
# python train.py --seed 0 --base-lr 3e-4
# python train.py --seed 1 --base-lr 3e-4
# python train.py --seed 2 --base-lr 3e-4

• Change one variable per run. If you also changed the batch size, the learning rate comparison means nothing.
• Report the spread across seeds, not the best seed. A best-of-three number is an optimistic selection and will not repeat.
• Compare on identical evaluation data and identical metric code. A different augmentation applied to validation can move a score by more than the model change.
• Keep the config that produced the number. Reconstructing it from memory after the fact is how results quietly stop being reproducible.

FAQ

Related

The training loop Debugging PyTorch models

Last refreshed 2026-09-18.