# Core scientific stack. Only Dense and Dropout are permitted as model layers;
# NumPy, pandas, Matplotlib, seaborn, and scikit-learn support data handling,
# metrics, and figures, and Keras callbacks act as training controls.
import os

# Quieten TensorFlow low-level C++ logs before import so the report output stays
# clean; this affects logging only, not computation.
os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")

import ast
import platform
from collections import Counter, defaultdict
import time
import random
import warnings

import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns

import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import (
    f1_score,
    precision_score,
    recall_score,
    accuracy_score,
    balanced_accuracy_score,
    cohen_kappa_score,
    matthews_corrcoef,
    top_k_accuracy_score,
    roc_auc_score,
    log_loss,
    classification_report,
    confusion_matrix,
)
from sklearn.preprocessing import label_binarize

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.callbacks import EarlyStopping

# Silence known-harmless warnings so the report output stays clean: the Keras
# input_shape notice, since the brief requires input_shape on the first Dense
# layer, and scikit-learn notices raised when a bootstrap resample of the
# imbalanced label set omits a rare class. That omission is a real property of
# resampling an imbalanced set, and its effect on the macro F1 interval is
# measured in Section 3 and is not assumed away.
from sklearn.exceptions import UndefinedMetricWarning
warnings.filterwarnings("ignore", category=UndefinedMetricWarning)
warnings.filterwarnings("ignore", message="Do not pass an", category=UserWarning)
warnings.filterwarnings("ignore", message="y_pred contains classes not in y_true")
# Raise the TensorFlow logger to ERROR so tf.function retracing notices during
# the many short predict calls do not clutter the report output.
tf.get_logger().setLevel("ERROR")

# Version and hardware provenance for reproducibility.
versions = pd.DataFrame(
    {
        "component": [
            "Python", "TensorFlow", "Keras", "NumPy", "pandas",
            "scikit-learn", "Matplotlib", "seaborn",
        ],
        "version": [
            platform.python_version(), tf.__version__, keras.__version__,
            np.__version__, pd.__version__, sklearn.__version__,
            matplotlib.__version__, sns.__version__,
        ],
    }
)

gpus = tf.config.list_physical_devices("GPU")
hardware = f"{platform.system()} {platform.machine()}, GPUs visible to TensorFlow: {len(gpus)}"
print(hardware)
versions

Darwin arm64, GPUs visible to TensorFlow: 0

# Global controls fixed for the whole investigation.
RANDOM_STATE = 42          # development and validation split
REUTERS_SEED = 113         # documented shuffle seed for the Keras loader default
VOCAB_MAIN = 10000         # main multi-hot vocabulary
VOCAB_SMALL = 5000         # controlled comparison vocabulary
NUM_CLASSES = 46
BATCH_SIZE = 512
MAX_EPOCHS = 30
SEEDS = [7, 42, 113, 2026, 31415]   # repetition seeds for finalists
TIE_THRESHOLD = 0.005      # macro F1 values within this margin count as tied

EARLY_STOPPING = dict(monitor="val_loss", patience=4, min_delta=0.001,
                      restore_best_weights=True)

# Deterministic operations where supported; exact equality can still depend on
# platform and library versions.
os.environ["PYTHONHASHSEED"] = "0"
try:
    tf.config.experimental.enable_op_determinism()
except Exception:
    pass


def set_all_seeds(seed):
    # Seed Python, NumPy, and TensorFlow before each fit for repeatable training.
    random.seed(seed)
    np.random.seed(seed)
    keras.utils.set_random_seed(seed)


# Restrained, consistent figure style used throughout.
sns.set_theme(style="whitegrid", palette="colorblind")
plt.rcParams.update({
    "figure.dpi": 110, "font.size": 11, "axes.titlesize": 12,
    "axes.labelsize": 11, "legend.fontsize": 10, "figure.autolayout": True,
})
PALETTE = sns.color_palette("colorblind")

# Show result tables in full. The value ledger in Appendix B is longer than
# the pandas row default, and the provenance and metric tables hold entries
# longer than the 50-character column default, so both limits are lifted to
# keep every table readable in the exported HTML.
pd.set_option("display.max_rows", 120)
pd.set_option("display.max_colwidth", None)
print("Global controls set.")

Global controls set.

# Load Reuters with the top VOCAB_MAIN words and a documented shuffle seed.
# The Keras loader applies its own train and test split; that test split is
# reserved for the single final evaluation in Section 3.
(train_data, train_labels), (test_data, test_labels) = keras.datasets.reuters.load_data(
    num_words=VOCAB_MAIN, seed=REUTERS_SEED
)

train_labels = np.asarray(train_labels, dtype="int64")
test_labels = np.asarray(test_labels, dtype="int64")

assert train_labels.min() == 0 and train_labels.max() == NUM_CLASSES - 1
assert set(np.unique(train_labels)) == set(range(NUM_CLASSES))
print(f"Training newswires: {len(train_data)}   Test newswires: {len(test_data)}")

Training newswires: 8982   Test newswires: 2246

# Compact dataset summary. Sequence lengths describe raw newswire lengths before
# multi-hot encoding, which motivates the fixed-length representation in Step 4.
train_lengths = np.array([len(seq) for seq in train_data])
class_counts = pd.Series(train_labels).value_counts().sort_index()

summary = pd.DataFrame(
    {
        "quantity": [
            "Training examples", "Test examples", "Number of classes",
            "Vocabulary limit", "Median newswire length (tokens)",
            "Mean newswire length (tokens)", "Maximum newswire length (tokens)",
            "Largest class share", "Smallest class support",
        ],
        "value": [
            len(train_data), len(test_data), NUM_CLASSES, VOCAB_MAIN,
            int(np.median(train_lengths)), round(float(train_lengths.mean()), 1),
            int(train_lengths.max()),
            f"{class_counts.max() / len(train_labels):.1%} (class {int(class_counts.idxmax())})",
            f"{int(class_counts.min())} (class {int(class_counts.idxmin())})",
        ],
    }
)
summary

# The metric hierarchy is fixed here so that later selection is transparent.
metric_plan = pd.DataFrame(
    {
        "measure": [
            "Categorical cross-entropy", "Macro F1", "Accuracy",
            "Weighted F1", "Generalisation gap", "Parameter count",
            "Balanced accuracy", "Cohen's kappa",
            "Matthews correlation coefficient", "Top-3 and top-5 accuracy",
            "Macro one-vs-rest ROC AUC", "Expected calibration error",
        ],
        "role": [
            "Optimisation objective and epoch selection",
            "Primary model-selection measure",
            "Secondary, interpretable overall measure",
            "Secondary, support-weighted quality",
            "Diagnostic for overfitting",
            "Tie-break and efficiency",
            "Final diagnostic, imbalance-robust recall",
            "Final diagnostic, chance-corrected agreement",
            "Final diagnostic, balanced multiclass correlation",
            "Final diagnostic, ranked-shortlist quality",
            "Final diagnostic, threshold-independent separability",
            "Final diagnostic, confidence calibration",
        ],
    }
)
metric_plan

LABELS = list(range(NUM_CLASSES))


def macro_ovr_auc(y_true, y_proba):
    # One-vs-rest ROC AUC averaged over classes that contain both a positive and
    # a negative test case; classes without both cannot yield a defined AUC.
    binary = label_binarize(y_true, classes=LABELS)
    scores = []
    for k in LABELS:
        column = binary[:, k]
        if 0 < column.sum() < len(column):
            scores.append(roc_auc_score(column, y_proba[:, k]))
    return float(np.mean(scores)), len(scores)


def expected_calibration_error(confidence, correct, n_bins=10):
    # Partition predictions into equal-width confidence bins and accumulate the
    # support-weighted gap between mean confidence and empirical accuracy.
    edges = np.linspace(0.0, 1.0, n_bins + 1)
    total = len(confidence)
    ece = 0.0
    rows = []
    for b in range(n_bins):
        low, high = edges[b], edges[b + 1]
        in_bin = (confidence > low) & (confidence <= high)
        if b == 0:
            in_bin = (confidence >= low) & (confidence <= high)
        count = int(in_bin.sum())
        if count == 0:
            rows.append({"bin_low": low, "bin_high": high, "count": 0,
                         "accuracy": np.nan, "confidence": np.nan})
            continue
        acc = float(correct[in_bin].mean())
        conf = float(confidence[in_bin].mean())
        ece += (count / total) * abs(acc - conf)
        rows.append({"bin_low": low, "bin_high": high, "count": count,
                     "accuracy": acc, "confidence": conf})
    return ece, pd.DataFrame(rows)


def full_metric_suite(y_true, y_pred, y_proba):
    # Scalar diagnostic suite reported at the final evaluation. Macro and
    # weighted variants use zero_division=0 so absent classes score zero.
    confidence = y_proba.max(axis=1)
    correct = (y_pred == y_true).astype(float)
    ece, _ = expected_calibration_error(confidence, correct)
    auc, auc_classes = macro_ovr_auc(y_true, y_proba)
    return {
        "Cross-entropy": log_loss(y_true, y_proba, labels=LABELS),
        "Accuracy": accuracy_score(y_true, y_pred),
        "Balanced accuracy": balanced_accuracy_score(y_true, y_pred),
        "Top-3 accuracy": top_k_accuracy_score(y_true, y_proba, k=3, labels=LABELS),
        "Top-5 accuracy": top_k_accuracy_score(y_true, y_proba, k=5, labels=LABELS),
        "Macro precision": precision_score(y_true, y_pred, average="macro",
                                           labels=LABELS, zero_division=0),
        "Macro recall": recall_score(y_true, y_pred, average="macro",
                                     labels=LABELS, zero_division=0),
        "Macro F1": f1_score(y_true, y_pred, average="macro",
                             labels=LABELS, zero_division=0),
        "Weighted precision": precision_score(y_true, y_pred, average="weighted",
                                              labels=LABELS, zero_division=0),
        "Weighted recall": recall_score(y_true, y_pred, average="weighted",
                                        labels=LABELS, zero_division=0),
        "Weighted F1": f1_score(y_true, y_pred, average="weighted",
                                labels=LABELS, zero_division=0),
        "Cohen's kappa": cohen_kappa_score(y_true, y_pred),
        "Matthews corrcoef": matthews_corrcoef(y_true, y_pred),
        f"Macro OVR ROC AUC ({auc_classes} classes)": auc,
        "Expected calibration error": ece,
    }

# One reproducible stratified 80:20 development and validation split.
# Integer targets are retained for scikit-learn metrics; one-hot targets are
# created in Step 4 for categorical cross-entropy.
dev_idx, val_idx = train_test_split(
    np.arange(len(train_labels)),
    test_size=0.20,
    random_state=RANDOM_STATE,
    stratify=train_labels,
)

# Disjointness and coverage assertions guard against leakage between partitions.
assert set(dev_idx).isdisjoint(set(val_idx))
assert len(dev_idx) + len(val_idx) == len(train_labels)

y_dev_int = train_labels[dev_idx]
y_val_int = train_labels[val_idx]

split_table = pd.DataFrame(
    {
        "partition": ["Development", "Validation", "Test (reserved)"],
        "examples": [len(dev_idx), len(val_idx), len(test_labels)],
        "share of training": [
            f"{len(dev_idx) / len(train_labels):.0%}",
            f"{len(val_idx) / len(train_labels):.0%}",
            "held out",
        ],
    }
)
split_table

def multi_hot_encode(sequences, dimension):
    # Adapted from the vectorisation approach in Chollet (2017), Chapter 3.
    # Word indices at or above the vocabulary bound are dropped, which lets the
    # same raw sequences produce a smaller vocabulary in the Step 7 comparison.
    assert dimension > 0
    result = np.zeros((len(sequences), dimension), dtype="float32")
    for i, sequence in enumerate(sequences):
        valid = [token for token in sequence if 0 <= token < dimension]
        result[i, valid] = 1.0
    return result


def prepare_dev_val(dimension):
    # Build multi-hot features for the development and validation rows at a
    # chosen vocabulary size. The test rows are deliberately not encoded here;
    # the test set is materialised only at the final evaluation in Section 3.
    x_dev = multi_hot_encode([train_data[i] for i in dev_idx], dimension)
    x_val = multi_hot_encode([train_data[i] for i in val_idx], dimension)
    return x_dev, x_val


# Main representation at the full vocabulary.
x_dev, x_val = prepare_dev_val(VOCAB_MAIN)

# One-hot targets for the softmax output; integer targets kept for metrics.
y_dev = keras.utils.to_categorical(y_dev_int, NUM_CLASSES)
y_val = keras.utils.to_categorical(y_val_int, NUM_CLASSES)

# Validity assertions on shapes, dtypes, ranges, and finiteness.
assert x_dev.dtype == np.float32 and x_val.dtype == np.float32
assert x_dev.shape[1] == VOCAB_MAIN and y_dev.shape[1] == NUM_CLASSES
assert set(np.unique(x_dev)).issubset({0.0, 1.0})
assert np.isfinite(x_dev).all()
assert np.isclose(y_dev.sum(axis=1), 1).all()
print(f"x_dev {x_dev.shape} {x_dev.dtype}   y_dev {y_dev.shape}")
print(f"x_val {x_val.shape}   validation targets {y_val.shape}")

x_dev (7185, 10000) float32   y_dev (7185, 46)
x_val (1797, 10000)   validation targets (1797, 46)

# Figure 1: class support in the development partition. This is the first figure
# in the report and appears only after the problem, measures, protocol, and data
# preparation have been explained. It motivates the macro F1 emphasis.
dev_counts = pd.Series(y_dev_int).value_counts().sort_index()

fig, ax = plt.subplots(figsize=(9, 3.2))
ax.bar(dev_counts.index, dev_counts.values, color=PALETTE[0])
ax.set_xlabel("Class identifier")
ax.set_ylabel("Development examples")
ax.set_title("Figure 1. Class support in the development partition")
ax.set_xlim(-1, NUM_CLASSES)
plt.show()
plt.close(fig)

def build_model(input_dim, hidden_units, dropout_rate, num_classes, seed):
    # Sequential model of Dense and Dropout layers only. The first Dense layer
    # receives input_shape, so a separate Input layer is unnecessary. Dropout is
    # placed after every hidden Dense layer and never after the output.
    set_all_seeds(seed)
    model = keras.Sequential(name="reuters_mlp")
    for position, units in enumerate(hidden_units):
        if position == 0:
            model.add(layers.Dense(units, activation="relu", input_shape=(input_dim,)))
        else:
            model.add(layers.Dense(units, activation="relu"))
        if dropout_rate > 0.0:
            model.add(layers.Dropout(dropout_rate))
    model.add(layers.Dense(num_classes, activation="softmax"))

    # Permitted-layer audit: every layer must be Dense or Dropout.
    for layer in model.layers:
        assert isinstance(layer, (layers.Dense, layers.Dropout)), type(layer)

    model.compile(optimizer="rmsprop", loss="categorical_crossentropy",
                  metrics=["accuracy"])
    return model

def run_experiment(name, hidden_units, dropout_rate, vocab_size, seed,
                   features, use_early_stopping=True,
                   max_epochs=MAX_EPOCHS, keep_history=False):
    # Train one configuration and return a compact record. The feature bundle
    # carries the development and validation arrays so the same runner serves
    # every vocabulary and data subset.
    x_tr, y_tr, x_va, y_va, y_va_int = features
    keras.backend.clear_session()
    set_all_seeds(seed)

    model = build_model(x_tr.shape[1], hidden_units, dropout_rate,
                        NUM_CLASSES, seed)
    callbacks = [EarlyStopping(**EARLY_STOPPING)] if use_early_stopping else []

    start = time.perf_counter()
    history = model.fit(x_tr, y_tr, validation_data=(x_va, y_va),
                        epochs=max_epochs, batch_size=BATCH_SIZE,
                        callbacks=callbacks, verbose=0)
    elapsed = time.perf_counter() - start

    val_loss_curve = history.history["val_loss"]
    best_epoch = int(np.argmin(val_loss_curve)) + 1

    # With restore_best_weights the model holds the minimum-val-loss weights, so
    # evaluation and prediction reflect the selected epoch.
    val_loss, val_acc = model.evaluate(x_va, y_va, batch_size=BATCH_SIZE, verbose=0)
    proba = model.predict(x_va, batch_size=BATCH_SIZE, verbose=0)
    preds = proba.argmax(axis=1)
    macro = f1_score(y_va_int, preds, average="macro",
                     labels=list(range(NUM_CLASSES)), zero_division=0)
    weighted = f1_score(y_va_int, preds, average="weighted",
                        labels=list(range(NUM_CLASSES)), zero_division=0)

    train_acc = history.history["accuracy"][best_epoch - 1]
    record = {
        "name": name,
        "hidden_units": str(list(hidden_units)),
        "dropout": dropout_rate,
        "vocab": vocab_size,
        "seed": seed,
        "params": model.count_params(),
        "epochs_run": len(val_loss_curve),
        "best_epoch": best_epoch,
        "train_loss": round(history.history["loss"][best_epoch - 1], 4),
        "val_loss": round(val_loss, 4),
        "train_acc": round(train_acc, 4),
        "val_acc": round(val_acc, 4),
        "val_macro_f1": round(macro, 4),
        "val_weighted_f1": round(weighted, 4),
        "acc_gap": round(train_acc - val_acc, 4),
        "seconds": round(elapsed, 1),
    }
    hist = history.history if keep_history else None
    keras.backend.clear_session()
    return record, hist


def parse_units(units_field):
    # run_experiment stores hidden_units as a string so that result frames
    # can be grouped and de-duplicated; literal_eval recovers the list.
    return ast.literal_eval(units_field)


# A running count of executed model fits, reported in the abstract.
FIT_COUNTER = {"n": 0}


def counted_run(*args, **kwargs):
    record, hist = run_experiment(*args, **kwargs)
    FIT_COUNTER["n"] += 1
    return record, hist


# Feature bundle for the main vocabulary, reused across phases.
features_main = (x_dev, y_dev, x_val, y_val, y_val_int)
print("Model factory and experiment runner ready.")

Model factory and experiment runner ready.

# Baselines on the validation partition.
uniform_acc = 1.0 / NUM_CLASSES
majority_class = int(pd.Series(y_dev_int).value_counts().idxmax())
majority_pred = np.full_like(y_val_int, majority_class)
majority_acc = accuracy_score(y_val_int, majority_pred)
majority_macro = f1_score(y_val_int, majority_pred, average="macro",
                          labels=list(range(NUM_CLASSES)), zero_division=0)

baseline_table = pd.DataFrame(
    {
        "baseline": ["Uniform chance", "Majority class"],
        "val_accuracy": [round(uniform_acc, 4), round(majority_acc, 4)],
        "val_macro_f1": ["n/a", round(majority_macro, 4)],
    }
)
baseline_table

# Small densely connected neural baseline: one hidden layer of 16 units. This is
# also the small_16 point in the Step 7 capacity phase, so its record is reused.
baseline_record, baseline_hist = counted_run(
    "small_16", [16], 0.0, VOCAB_MAIN, RANDOM_STATE,
    features_main, use_early_stopping=True, keep_history=True,
)
pd.DataFrame([baseline_record])[
    ["name", "params", "best_epoch", "val_loss", "val_acc", "val_macro_f1"]
]

# High-capacity probe: three hidden layers of 256 units, no Dropout, no early
# stopping, so the full divergence is visible in the curves.
probe_record, probe_hist = counted_run(
    "probe_256_256_256", [256, 256, 256], 0.0, VOCAB_MAIN, RANDOM_STATE,
    features_main, use_early_stopping=False, max_epochs=30, keep_history=True,
)
probe_best_epoch = probe_record["best_epoch"]
pd.DataFrame([probe_record])[
    ["name", "params", "epochs_run", "best_epoch", "train_loss", "val_loss",
     "train_acc", "val_acc"]
]

# Figure 2: training and validation curves for the high-capacity probe.
epochs_axis = range(1, len(probe_hist["loss"]) + 1)
fig, axes = plt.subplots(1, 2, figsize=(11, 3.8))

axes[0].plot(epochs_axis, probe_hist["loss"], color=PALETTE[0], label="Training")
axes[0].plot(epochs_axis, probe_hist["val_loss"], color=PALETTE[3], label="Validation")
axes[0].axvline(probe_best_epoch, color="grey", linestyle="--", linewidth=1,
                label=f"Min val loss (epoch {probe_best_epoch})")
axes[0].set_xlabel("Epoch"); axes[0].set_ylabel("Categorical cross-entropy")
axes[0].set_title("Loss"); axes[0].legend()

axes[1].plot(epochs_axis, probe_hist["accuracy"], color=PALETTE[0], label="Training")
axes[1].plot(epochs_axis, probe_hist["val_accuracy"], color=PALETTE[3], label="Validation")
axes[1].axvline(probe_best_epoch, color="grey", linestyle="--", linewidth=1)
axes[1].set_xlabel("Epoch"); axes[1].set_ylabel("Accuracy")
axes[1].set_title("Accuracy"); axes[1].legend()

fig.suptitle("Figure 2. High-capacity probe training and validation curves", y=1.03)
plt.show()
plt.close(fig)

# Phase 1 capacity configurations. The small_16 baseline record is reused.
phase1_configs = [
    ("small_16", [16]),
    ("reference_64_64", [64, 64]),
    ("wide_128_128", [128, 128]),
    ("deep_64_64_64", [64, 64, 64]),
    ("bottleneck_64_4", [64, 4]),
]

phase1_records = [baseline_record]
phase1_histories = {"small_16": baseline_hist}
for name, units in phase1_configs:
    if name == "small_16":
        continue
    rec, hist = counted_run(name, units, 0.0, VOCAB_MAIN, RANDOM_STATE,
                            features_main, use_early_stopping=True, keep_history=True)
    phase1_records.append(rec)
    phase1_histories[name] = hist

phase1_df = pd.DataFrame(phase1_records)
phase1_view = phase1_df[["name", "hidden_units", "params", "best_epoch",
                         "val_loss", "val_acc", "val_macro_f1", "acc_gap",
                         "seconds"]]
phase1_view

def select_best(df):
    # Apply the declared rule: highest macro F1, ties within TIE_THRESHOLD
    # resolved by lower validation loss then fewer parameters.
    top = df.sort_values("val_macro_f1", ascending=False).iloc[0]
    contenders = df[df["val_macro_f1"] >= top["val_macro_f1"] - TIE_THRESHOLD]
    contenders = contenders.sort_values(["val_loss", "params"],
                                        ascending=[True, True])
    return contenders.iloc[0]


phase1_best = select_best(phase1_df)
best_units = parse_units(phase1_best["hidden_units"])
top_two = phase1_df["val_macro_f1"].nlargest(2)
phase1_margin = round(float(top_two.iloc[0] - top_two.iloc[1]), 4)
print(f"Phase 1 selection: {phase1_best['name']} "
      f"(macro F1 {phase1_best['val_macro_f1']}, params {phase1_best['params']})")
print(f"Margin over the runner-up: {phase1_margin}")

Phase 1 selection: wide_128_128 (macro F1 0.5298, params 1302574)
Margin over the runner-up: 0.0431

# Phase 2 varies Dropout on the Phase 1 winner, all else fixed.
dropout_rates = [0.0, 0.2, 0.35, 0.5]
phase2_records, phase2_histories = [], {}
for rate in dropout_rates:
    name = f"drop_{rate}"
    rec, hist = counted_run(name, best_units, rate, VOCAB_MAIN, RANDOM_STATE,
                            features_main, use_early_stopping=True, keep_history=True)
    phase2_records.append(rec)
    phase2_histories[rate] = hist

phase2_df = pd.DataFrame(phase2_records)
phase2_view = phase2_df[["name", "dropout", "params", "best_epoch", "val_loss",
                         "val_acc", "val_macro_f1", "acc_gap", "seconds"]]
phase2_view

phase2_best = select_best(phase2_df)
best_dropout = float(phase2_best["dropout"])
print(f"Phase 2 selection: dropout {best_dropout} "
      f"(macro F1 {phase2_best['val_macro_f1']}, gap {phase2_best['acc_gap']})")

Phase 2 selection: dropout 0.0 (macro F1 0.5298, gap 0.1245)

# Figure 3: Dropout effect on validation macro F1 and the accuracy gap.
fig, ax1 = plt.subplots(figsize=(7, 3.6))
ax1.plot(phase2_df["dropout"], phase2_df["val_macro_f1"], marker="o",
         color=PALETTE[0], label="Validation macro F1")
ax1.set_xlabel("Dropout rate"); ax1.set_ylabel("Validation macro F1", color=PALETTE[0])
ax1.tick_params(axis="y", labelcolor=PALETTE[0])

ax2 = ax1.twinx()
ax2.plot(phase2_df["dropout"], phase2_df["acc_gap"], marker="s",
         color=PALETTE[3], label="Accuracy gap")
ax2.set_ylabel("Train minus validation accuracy", color=PALETTE[3])
ax2.tick_params(axis="y", labelcolor=PALETTE[3])
ax2.grid(False)

fig.suptitle("Figure 3. Effect of Dropout on validation macro F1 and generalisation gap")
plt.show()
plt.close(fig)

# Phase 3 varies vocabulary size only. Re-encode features at 5000 words.
x_dev_s, x_val_s = prepare_dev_val(VOCAB_SMALL)
features_small = (x_dev_s, y_dev, x_val_s, y_val, y_val_int)

phase3_records = []
for vocab, feats in [(VOCAB_SMALL, features_small), (VOCAB_MAIN, features_main)]:
    name = f"vocab_{vocab}"
    rec, _ = counted_run(name, best_units, best_dropout, vocab, RANDOM_STATE,
                         feats, use_early_stopping=True, keep_history=False)
    phase3_records.append(rec)

phase3_df = pd.DataFrame(phase3_records)
phase3_view = phase3_df[["name", "vocab", "params", "best_epoch", "val_loss",
                         "val_acc", "val_macro_f1", "acc_gap", "seconds"]]
phase3_view

phase3_best = select_best(phase3_df)
best_vocab = int(phase3_best["vocab"])
print(f"Phase 3 selection: vocabulary {best_vocab} "
      f"(macro F1 {phase3_best['val_macro_f1']})")

Phase 3 selection: vocabulary 10000 (macro F1 0.5298)

# Assemble a pool of complete configurations observed across the phases and pick
# the two leading distinct configurations by single-seed validation macro F1.
pool = pd.concat([phase1_df, phase2_df, phase3_df], ignore_index=True)
pool["config_key"] = list(zip(pool["hidden_units"], pool["dropout"], pool["vocab"]))
pool_unique = (pool.sort_values("val_macro_f1", ascending=False)
                   .drop_duplicates("config_key")
                   .reset_index(drop=True))
finalists = pool_unique.head(2)
finalists[["name", "hidden_units", "dropout", "vocab", "val_macro_f1", "val_loss"]]

def feature_bundle_for_vocab(vocab):
    # Return the precomputed feature bundle for a given vocabulary size.
    if vocab == VOCAB_MAIN:
        return features_main
    if vocab == VOCAB_SMALL:
        return features_small
    raise ValueError(f"Unsupported vocabulary size: {vocab}")


# Five-seed repetition for each finalist. The split is fixed, so the training
# seed is the only factor that varies here.
stability_rows = []
for _, cfg in finalists.iterrows():
    units = parse_units(cfg["hidden_units"])
    rate = float(cfg["dropout"])
    vocab = int(cfg["vocab"])
    feats = feature_bundle_for_vocab(vocab)
    for seed in SEEDS:
        rec, _ = counted_run(cfg["name"], units, rate, vocab, seed, feats,
                             use_early_stopping=True, keep_history=False)
        stability_rows.append(rec)

stability_df = pd.DataFrame(stability_rows)
stability_df[["name", "seed", "best_epoch", "val_loss", "val_acc",
              "val_macro_f1"]]

# Summarise each finalist across seeds: mean and standard deviation.
def summarise(group):
    return pd.Series({
        "hidden_units": group["hidden_units"].iloc[0],
        "dropout": group["dropout"].iloc[0],
        "vocab": group["vocab"].iloc[0],
        "params": int(group["params"].iloc[0]),
        "macro_f1_mean": round(group["val_macro_f1"].mean(), 4),
        "macro_f1_std": round(group["val_macro_f1"].std(ddof=1), 4),
        "acc_mean": round(group["val_acc"].mean(), 4),
        "acc_std": round(group["val_acc"].std(ddof=1), 4),
        "loss_mean": round(group["val_loss"].mean(), 4),
        "loss_std": round(group["val_loss"].std(ddof=1), 4),
        "median_best_epoch": int(np.median(group["best_epoch"])),
    })


stability_summary = (stability_df.groupby("name", sort=False)
                                 .apply(summarise, include_groups=False)
                                 .reset_index())
stability_summary

# Final selection by mean validation macro F1 with the declared tie-break on
# mean validation loss then parameter count.
ranked = stability_summary.sort_values("macro_f1_mean", ascending=False)
lead = ranked.iloc[0]
close = ranked[ranked["macro_f1_mean"] >= lead["macro_f1_mean"] - 0.005]
close = close.sort_values(["loss_mean", "params"], ascending=[True, True])
final_row = close.iloc[0]

final_units = parse_units(final_row["hidden_units"])
final_dropout = float(final_row["dropout"])
final_vocab = int(final_row["vocab"])
final_epochs = int(round(final_row["median_best_epoch"]))
print(f"Selected configuration: units={final_units}, dropout={final_dropout}, "
      f"vocab={final_vocab}, epochs={final_epochs}")

Selected configuration: units=[128, 128], dropout=0.0, vocab=10000, epochs=9

# Repeat both finalists across five independent stratified splits of the
# supplied training data. Features are encoded once per vocabulary and then
# indexed, which avoids re-encoding the corpus for every split, and the large
# arrays are released as soon as a vocabulary is finished.
SPLIT_SEEDS = SEEDS


def split_feature_bundle(x_all, seed):
    # Draw a fresh stratified 80:20 partition and assemble a feature bundle in
    # the layout that run_experiment expects.
    d_idx, v_idx = train_test_split(
        np.arange(len(train_labels)), test_size=0.20,
        random_state=seed, stratify=train_labels,
    )
    assert set(d_idx).isdisjoint(set(v_idx))
    assert len(d_idx) + len(v_idx) == len(train_labels)
    return (
        x_all[d_idx],
        keras.utils.to_categorical(train_labels[d_idx], NUM_CLASSES),
        x_all[v_idx],
        keras.utils.to_categorical(train_labels[v_idx], NUM_CLASSES),
        train_labels[v_idx],
    )


split_rows = []
for vocab in sorted({int(v) for v in finalists["vocab"]}):
    x_all_vocab = multi_hot_encode(train_data, vocab)
    for split_seed in SPLIT_SEEDS:
        bundle = split_feature_bundle(x_all_vocab, split_seed)
        for _, cfg in finalists[finalists["vocab"] == vocab].iterrows():
            rec, _ = counted_run(cfg["name"], parse_units(cfg["hidden_units"]),
                                 float(cfg["dropout"]), vocab, RANDOM_STATE,
                                 bundle, use_early_stopping=True,
                                 keep_history=False)
            rec["split_seed"] = split_seed
            split_rows.append(rec)
        del bundle
    del x_all_vocab

split_df = pd.DataFrame(split_rows)
split_wide = split_df.pivot(index="split_seed", columns="name",
                            values="val_macro_f1").round(4)
split_wide

# Marginal means across splits, then the paired per-split difference. The paired
# view removes the split effect that both configurations share, so it answers
# the ranking question more directly than the marginal means.
split_summary = (split_df.groupby("name")
                 .agg(macro_f1_mean=("val_macro_f1", "mean"),
                      macro_f1_std=("val_macro_f1", "std"),
                      acc_mean=("val_acc", "mean"),
                      acc_std=("val_acc", "std"),
                      loss_mean=("val_loss", "mean"),
                      loss_std=("val_loss", "std"))
                 .round(4).reset_index())

lead_name = stability_summary.sort_values("macro_f1_mean",
                                          ascending=False)["name"].iloc[0]
other_name = [c for c in split_wide.columns if c != lead_name][0]
paired_diff = split_wide[lead_name] - split_wide[other_name]
split_range = float(split_wide[lead_name].max() - split_wide[lead_name].min())
seed_range = float(stability_df.loc[stability_df["name"] == lead_name,
                                    "val_macro_f1"].max()
                   - stability_df.loc[stability_df["name"] == lead_name,
                                      "val_macro_f1"].min())

print(f"Range of validation macro F1 for {lead_name}: "
      f"{split_range:.4f} across splits, {seed_range:.4f} across training seeds")
print(f"Paired difference ({lead_name} minus {other_name}): "
      f"mean {paired_diff.mean():.4f}, standard deviation {paired_diff.std():.4f}, "
      f"favourable on {int((paired_diff > 0).sum())} of {len(paired_diff)} splits")

# Margin between the finalists on the fixed split, for comparison with the
# paired difference measured across splits.
fixed_split_margin = float(stability_summary["macro_f1_mean"].max()
                           - stability_summary["macro_f1_mean"].min())
print(f"Margin between finalists on the fixed split of Section 2.7.4: "
      f"{fixed_split_margin:.4f}")

# The declared selection rule, applied to the repeated-split means.
if abs(float(paired_diff.mean())) < TIE_THRESHOLD:
    split_choice = split_summary.sort_values("loss_mean")["name"].iloc[0]
    print(f"The paired difference lies inside the declared tie threshold of "
          f"{TIE_THRESHOLD}, so the two configurations are practically tied "
          f"across splits; the tie-break on lower mean validation loss "
          f"resolves to {split_choice}")
else:
    split_choice = split_summary.sort_values("macro_f1_mean",
                                             ascending=False)["name"].iloc[0]
    print(f"The paired difference exceeds the declared tie threshold of "
          f"{TIE_THRESHOLD}; the higher mean across splits belongs to "
          f"{split_choice}")
split_summary

Range of validation macro F1 for wide_128_128: 0.0716 across splits, 0.1120 across training seeds
Paired difference (wide_128_128 minus vocab_5000): mean 0.0018, standard deviation 0.0330, favourable on 2 of 5 splits
Margin between finalists on the fixed split of Section 2.7.4: 0.0285
The paired difference lies inside the declared tie threshold of 0.005, so the two configurations are practically tied across splits; the tie-break on lower mean validation loss resolves to wide_128_128

# Nested stratified subsets of the development partition. Each larger fraction
# contains the smaller ones, so the curve reflects the effect of added data.
# The validation partition is fixed throughout. Repeating every fraction over
# the declared seeds gives each point a spread, not a single value.
def nested_fraction_indices(labels_int, fractions, seed):
    order = np.arange(len(labels_int))
    rng = np.random.default_rng(seed)
    per_class = {c: rng.permutation(order[labels_int == c])
                 for c in range(NUM_CLASSES)}
    subsets = {}
    for frac in fractions:
        chosen = []
        for c in range(NUM_CLASSES):
            take = max(1, int(round(len(per_class[c]) * frac)))
            chosen.extend(per_class[c][:take])
        subsets[frac] = np.array(sorted(chosen))
    return subsets


fractions = [0.25, 0.5, 0.75, 1.0]
final_feats = feature_bundle_for_vocab(final_vocab)
x_dev_final, y_dev_final = final_feats[0], final_feats[1]

learning_rows = []
for seed in SEEDS:
    subset_idx = nested_fraction_indices(y_dev_int, fractions, seed)
    for frac in fractions:
        idx = subset_idx[frac]
        feats = (x_dev_final[idx], y_dev_final[idx],
                 final_feats[2], final_feats[3], final_feats[4])
        rec, _ = counted_run(f"frac_{frac}", final_units, final_dropout,
                             final_vocab, seed, feats, use_early_stopping=True)
        rec["fraction"] = frac
        rec["dev_examples"] = len(idx)
        learning_rows.append(rec)

learning_df = pd.DataFrame(learning_rows)
learning_summary = (learning_df.groupby(["fraction", "dev_examples"])
                    .agg(macro_f1_mean=("val_macro_f1", "mean"),
                         macro_f1_std=("val_macro_f1", "std"),
                         acc_mean=("val_acc", "mean"),
                         acc_std=("val_acc", "std"),
                         loss_mean=("val_loss", "mean"))
                    .round(4).reset_index())

# The change between neighbouring fractions is the quantity the curve is read
# for, so it is tabulated directly.
learning_summary["macro_f1_step"] = learning_summary["macro_f1_mean"].diff().round(4)
learning_summary["acc_step"] = learning_summary["acc_mean"].diff().round(4)

# The per-seed values are shown as well, because the spread at the larger
# fractions is what determines whether the trend can be read from one run.
learning_wide = learning_df.pivot(index="seed", columns="fraction",
                                  values="val_macro_f1").round(4)
print("Validation macro F1 by training seed and fraction:")
print(learning_wide.to_string())

# The seeds pair across fractions, so the final step can be examined as a
# paired difference. This is the quantity RQ5 turns on.
final_step = learning_wide[1.0] - learning_wide[0.75]
step_se = final_step.std(ddof=1) / np.sqrt(len(final_step))
print(f"\nFinal step from 0.75 to 1.00 by seed: {final_step.round(4).tolist()}")
print(f"  mean {final_step.mean():.4f}, standard deviation "
      f"{final_step.std(ddof=1):.4f}, positive for "
      f"{int((final_step > 0).sum())} of {len(final_step)} seeds, "
      f"mean is {final_step.mean() / step_se:.2f} times its standard error")
learning_summary

Validation macro F1 by training seed and fraction:
fraction    0.25    0.50    0.75    1.00
seed                                    
7         0.3626  0.4100  0.4677  0.4888
42        0.3278  0.4510  0.5340  0.5298
113       0.3305  0.4194  0.4570  0.5999
2026      0.3698  0.4295  0.4907  0.6008
31415     0.3410  0.4382  0.4631  0.5261

Final step from 0.75 to 1.00 by seed: [0.0211, -0.0042, 0.1429, 0.1101, 0.063]
  mean 0.0666, standard deviation 0.0608, positive for 4 of 5 seeds, mean is 2.45 times its standard error

# Figure 4: learning curve of validation macro F1 against development size. The
# band is plus or minus one standard deviation over the five seeds, so the
# change between neighbouring fractions can be judged against seed variation.
fig, ax = plt.subplots(figsize=(7, 3.8))
sizes = learning_summary["dev_examples"]
mean_f1 = learning_summary["macro_f1_mean"]
sd_f1 = learning_summary["macro_f1_std"]
ax.plot(sizes, mean_f1, marker="o", color=PALETTE[0], label="Mean over five seeds")
ax.fill_between(sizes, mean_f1 - sd_f1, mean_f1 + sd_f1, color=PALETTE[0],
                alpha=0.20, label="Plus or minus one standard deviation")
ax.set_xlabel("Development examples used")
ax.set_ylabel("Validation macro F1")
ax.set_title("Figure 4. Learning curve for the selected configuration")
ax.legend(loc="lower right")
plt.show()
plt.close(fig)

# Final architecture summary before test access.
final_arch = pd.DataFrame(
    {
        "property": ["Hidden units", "Dropout rate", "Vocabulary size",
                     "Output layer", "Loss", "Optimizer", "Training epochs",
                     "Epoch rule"],
        "value": [
            str(final_units), final_dropout, final_vocab,
            "Dense(46, softmax)", "categorical_crossentropy", "rmsprop",
            final_epochs, "Rounded median best epoch over five seeds",
        ],
    }
)
final_arch

# Encode the full training set and the test set at the selected vocabulary, then
# train once for the fixed epoch count. The former validation examples are now
# included in training.
x_train_full = multi_hot_encode(train_data, final_vocab)
x_test_full = multi_hot_encode(test_data, final_vocab)
y_train_full = keras.utils.to_categorical(train_labels, NUM_CLASSES)

assert x_train_full.shape[1] == final_vocab
assert np.isfinite(x_train_full).all()

keras.backend.clear_session()
set_all_seeds(RANDOM_STATE)
final_model = build_model(final_vocab, final_units, final_dropout,
                          NUM_CLASSES, RANDOM_STATE)
final_model.fit(x_train_full, y_train_full, epochs=final_epochs,
                batch_size=BATCH_SIZE, verbose=0)
FIT_COUNTER["n"] += 1
print(f"Final model trained for {final_epochs} epochs on {len(x_train_full)} examples.")

Final model trained for 9 epochs on 8982 examples.

# Single test evaluation. Probability rows are checked to sum to one before use.
test_proba = final_model.predict(x_test_full, batch_size=BATCH_SIZE, verbose=0)
assert np.allclose(test_proba.sum(axis=1), 1.0, atol=1e-4)
test_pred = test_proba.argmax(axis=1)

# Full diagnostic suite on the untouched test split.
suite = full_metric_suite(test_labels, test_pred, test_proba)

# The four required headline measures, retained by name for the abstract.
test_loss = suite["Cross-entropy"]
test_acc = suite["Accuracy"]
test_macro = suite["Macro F1"]
test_weighted = suite["Weighted F1"]
test_bal_acc = suite["Balanced accuracy"]
test_kappa = suite["Cohen's kappa"]
test_mcc = suite["Matthews corrcoef"]
test_top3 = suite["Top-3 accuracy"]
test_top5 = suite["Top-5 accuracy"]
test_ece = suite["Expected calibration error"]
test_auc = [v for k, v in suite.items() if k.startswith("Macro OVR")][0]

test_metrics = pd.DataFrame(
    {"metric": list(suite.keys()),
     "value": [round(v, 4) for v in suite.values()]}
)
test_metrics

# Seeded non-parametric bootstrap for 95 percent percentile intervals on
# accuracy and macro F1. Predictions are fixed; test labels are resampled in
# pairs with replacement over 2000 resamples.
def bootstrap_intervals(y_true, y_pred, n_resamples=2000, seed=RANDOM_STATE):
    rng = np.random.default_rng(seed)
    n = len(y_true)
    accs = np.empty(n_resamples)
    macros = np.empty(n_resamples)
    bals = np.empty(n_resamples)
    labels = list(range(NUM_CLASSES))
    for b in range(n_resamples):
        sample = rng.integers(0, n, n)
        yt = y_true[sample]
        yp = y_pred[sample]
        accs[b] = accuracy_score(yt, yp)
        macros[b] = f1_score(yt, yp, average="macro", labels=labels, zero_division=0)
        bals[b] = balanced_accuracy_score(yt, yp)
    ci = lambda a: (round(np.percentile(a, 2.5), 4), round(np.percentile(a, 97.5), 4))
    return ci(accs), ci(macros), ci(bals)


acc_ci, macro_ci, bal_ci = bootstrap_intervals(test_labels, test_pred)
ci_table = pd.DataFrame(
    {
        "metric": ["Accuracy", "Macro F1", "Balanced accuracy"],
        "point_estimate": [round(test_acc, 4), round(test_macro, 4),
                           round(test_bal_acc, 4)],
        "ci_lower_95": [acc_ci[0], macro_ci[0], bal_ci[0]],
        "ci_upper_95": [acc_ci[1], macro_ci[1], bal_ci[1]],
    }
)
ci_table

# The bootstrap above resamples all test cases together, so a class with very
# small support is sometimes absent from a resample and is then scored zero,
# which pulls the macro F1 interval downwards. A stratified bootstrap resamples
# within each true class, holding every class support at its observed value and
# removing that artefact. Both are reported so the size of the effect is visible.
def stratified_bootstrap_intervals(y_true, y_pred, n_resamples=2000,
                                   seed=RANDOM_STATE):
    rng = np.random.default_rng(seed)
    index_by_class = [np.flatnonzero(y_true == c) for c in LABELS]
    index_by_class = [idx for idx in index_by_class if len(idx) > 0]
    accs = np.empty(n_resamples)
    macros = np.empty(n_resamples)
    for b in range(n_resamples):
        sample = np.concatenate([rng.choice(idx, len(idx), replace=True)
                                 for idx in index_by_class])
        accs[b] = accuracy_score(y_true[sample], y_pred[sample])
        macros[b] = f1_score(y_true[sample], y_pred[sample], average="macro",
                             labels=LABELS, zero_division=0)
    ci = lambda a: (round(np.percentile(a, 2.5), 4), round(np.percentile(a, 97.5), 4))
    return ci(accs), ci(macros)


strat_acc_ci, strat_macro_ci = stratified_bootstrap_intervals(test_labels, test_pred)

# How often the unstratified scheme loses a class outright, which is the
# mechanism behind the difference between the two intervals.
rng_check = np.random.default_rng(RANDOM_STATE)
omitted = np.array([NUM_CLASSES - len(np.unique(
    test_labels[rng_check.integers(0, len(test_labels), len(test_labels))]))
    for _ in range(2000)])
print(f"Unstratified resamples omitting at least one class: "
      f"{(omitted > 0).mean():.1%}; mean number omitted {omitted.mean():.2f}")

interval_comparison = pd.DataFrame(
    {
        "metric": ["Accuracy", "Accuracy", "Macro F1", "Macro F1"],
        "scheme": ["Unstratified", "Stratified by class"] * 2,
        "point_estimate": [round(test_acc, 4)] * 2 + [round(test_macro, 4)] * 2,
        "ci_lower_95": [acc_ci[0], strat_acc_ci[0], macro_ci[0], strat_macro_ci[0]],
        "ci_upper_95": [acc_ci[1], strat_acc_ci[1], macro_ci[1], strat_macro_ci[1]],
    }
)
interval_comparison["width"] = (interval_comparison["ci_upper_95"]
                                - interval_comparison["ci_lower_95"]).round(4)

# The size of the correction is the quantity of interest, so it is stated.
macro_rows = interval_comparison[interval_comparison["metric"] == "Macro F1"]
lower_shift = round(strat_macro_ci[0] - macro_ci[0], 4)
width_change = round(float(macro_rows["width"].iloc[0]
                           - macro_rows["width"].iloc[1]), 4)
print(f"Stratifying raises the macro F1 lower bound by {lower_shift} and "
      f"narrows the interval by {width_change}")
interval_comparison

Unstratified resamples omitting at least one class: 63.0%; mean number omitted 0.86
Stratifying raises the macro F1 lower bound by 0.0227 and narrows the interval by 0.0192

# Per-class precision, recall, F1, and support on the test split.
report_dict = classification_report(
    test_labels, test_pred, labels=list(range(NUM_CLASSES)),
    output_dict=True, zero_division=0,
)
per_class = pd.DataFrame(
    [
        {
            "class": c,
            "precision": round(report_dict[str(c)]["precision"], 3),
            "recall": round(report_dict[str(c)]["recall"], 3),
            "f1": round(report_dict[str(c)]["f1-score"], 3),
            "support": int(report_dict[str(c)]["support"]),
        }
        for c in range(NUM_CLASSES)
    ]
)
per_class.sort_values("support", ascending=False).head(10)

# Five strongest and five weakest classes by F1, with support shown. Classes
# with zero support cannot be scored and are excluded from this ranking.
scored = per_class[per_class["support"] > 0].copy()
strongest = scored.sort_values(["f1", "support"], ascending=[False, False]).head(5)
weakest = scored.sort_values(["f1", "support"], ascending=[True, False]).head(5)
extremes = pd.concat(
    [strongest.assign(group="strongest"), weakest.assign(group="weakest")]
)
extremes[["group", "class", "precision", "recall", "f1", "support"]]

# Per-class F1 with bootstrap intervals. Predictions are held fixed and the
# paired test labels are resampled, using the same seed and resample count as
# the headline intervals. A class with very small support yields a wide or
# degenerate interval, which is the reason for reporting the interval next to
# the point estimate.
def per_class_f1_intervals(y_true, y_pred, n_resamples=2000, seed=RANDOM_STATE):
    rng = np.random.default_rng(seed)
    n = len(y_true)
    draws = np.empty((n_resamples, NUM_CLASSES))
    for b in range(n_resamples):
        sample = rng.integers(0, n, n)
        draws[b] = f1_score(y_true[sample], y_pred[sample], average=None,
                            labels=LABELS, zero_division=0)
    return np.percentile(draws, 2.5, axis=0), np.percentile(draws, 97.5, axis=0)


f1_lower, f1_upper = per_class_f1_intervals(test_labels, test_pred)
per_class["f1_ci_lower"] = np.round(f1_lower, 3)
per_class["f1_ci_upper"] = np.round(f1_upper, 3)
per_class["f1_ci_width"] = np.round(f1_upper - f1_lower, 3)

extremes_ci = extremes[["group", "class", "f1", "support"]].merge(
    per_class[["class", "f1_ci_lower", "f1_ci_upper", "f1_ci_width"]], on="class"
)
extremes_ci

# Support-qualified ranking. With support n a single misclassification moves
# recall by 1/n, so a threshold of 20 test cases holds that movement at or below
# 5 percentage points. Classes below the threshold remain in the table above,
# where their intervals show how little a point estimate conveys.
MIN_SUPPORT = 20
well_supported = per_class[per_class["support"] >= MIN_SUPPORT].copy()
qualified = pd.concat([
    well_supported.sort_values("f1", ascending=False).head(5).assign(group="strongest"),
    well_supported.sort_values("f1", ascending=True).head(5).assign(group="weakest"),
])

# Rank correlation between support and F1, over all scored classes and then
# within the qualified subset. Comparing the two separates a genuine effect
# of support from the arithmetic of scoring a class that holds a few cases.
scored_classes = per_class[per_class["support"] > 0]
rho_all = scored_classes[["support", "f1"]].corr(method="spearman").iloc[0, 1]
rho_qualified = well_supported[["support", "f1"]].corr(method="spearman").iloc[0, 1]

print(f"{len(well_supported)} of {NUM_CLASSES} classes reach the support "
      f"threshold of {MIN_SUPPORT}; together they hold "
      f"{int(well_supported['support'].sum())} of {len(test_labels)} test cases.")
print(f"Spearman correlation between support and F1: {rho_all:.3f} over the "
      f"{len(scored_classes)} scored classes, {rho_qualified:.3f} within the "
      f"{len(well_supported)} qualified classes.")
qualified[["group", "class", "precision", "recall", "f1", "support",
           "f1_ci_lower", "f1_ci_upper"]]

15 of 46 classes reach the support threshold of 20; together they hold 2005 of 2246 test cases.
Spearman correlation between support and F1: 0.360 over the 46 scored classes, 0.222 within the 15 qualified classes.

# Most frequent off-diagonal confusion pairs across all classes.
cm = confusion_matrix(test_labels, test_pred, labels=list(range(NUM_CLASSES)))
pairs = []
for i in range(NUM_CLASSES):
    for j in range(NUM_CLASSES):
        if i != j and cm[i, j] > 0:
            pairs.append({"true_class": i, "predicted_class": j, "count": int(cm[i, j])})
confusion_pairs = (pd.DataFrame(pairs)
                   .sort_values("count", ascending=False)
                   .head(10)
                   .reset_index(drop=True))
confusion_pairs

# Figure 5: row-normalised confusion matrix for the most supported classes,
# selected by a documented rule: the twelve classes with the largest test
# support. The matrix is computed over all 46 classes first, rows are then
# normalised by the complete row total, and a final column collects predictions
# that fall outside the twelve. Restricting the matrix before normalising would
# drop those predictions and inflate the diagonal, so every row here sums to one
# and no error is hidden by the restriction.
cm_full = confusion_matrix(test_labels, test_pred, labels=LABELS)
top_support = sorted(per_class.sort_values("support", ascending=False)
                              .head(12)["class"].tolist())

row_totals = cm_full[top_support].sum(axis=1, keepdims=True).clip(min=1)
block = cm_full[np.ix_(top_support, top_support)]
outside = cm_full[top_support].sum(axis=1, keepdims=True) - block.sum(axis=1, keepdims=True)
cm_norm = np.hstack([block, outside]) / row_totals
assert np.allclose(cm_norm.sum(axis=1), 1.0)

fig, ax = plt.subplots(figsize=(8.4, 6))
sns.heatmap(cm_norm, annot=True, fmt=".2f", cmap="Blues", cbar=True,
            xticklabels=[str(c) for c in top_support] + ["other"],
            yticklabels=top_support, ax=ax, annot_kws={"size": 8})
ax.set_xlabel("Predicted class"); ax.set_ylabel("True class")
ax.set_title("Figure 5. Row-normalised confusion matrix for the twelve most supported classes")
plt.show()
plt.close(fig)

# Share of each row that leaves the displayed block, which the restricted
# matrix would otherwise conceal.
outside_share = pd.DataFrame({
    "class": top_support,
    "support": cm_full[top_support].sum(axis=1),
    "outside_share": np.round((outside / row_totals).ravel(), 3),
})
outside_share

# Figure 6: predicted confidence for correct and incorrect test predictions.
confidence = test_proba.max(axis=1)
correct_mask = test_pred == test_labels
conf_df = pd.DataFrame({
    "confidence": confidence,
    "outcome": np.where(correct_mask, "correct", "incorrect"),
})

fig, ax = plt.subplots(figsize=(7.5, 3.6))
sns.boxplot(data=conf_df, x="outcome", y="confidence",
            hue="outcome", palette=[PALETTE[2], PALETTE[3]], legend=False, ax=ax)
ax.set_xlabel("Prediction outcome"); ax.set_ylabel("Predicted class probability")
ax.set_title("Figure 6. Prediction confidence for correct and incorrect predictions")
plt.show()
plt.close(fig)

conf_summary = conf_df.groupby("outcome")["confidence"].agg(["mean", "median"]).round(3)
conf_summary

# Figure 7: reliability diagram. Perfect calibration lies on the diagonal, where
# mean confidence in a bin equals the empirical accuracy of that bin.
ece_value, reliability = expected_calibration_error(
    test_proba.max(axis=1), (test_pred == test_labels).astype(float)
)
plotted = reliability.dropna(subset=["accuracy"])
centres = (plotted["bin_low"] + plotted["bin_high"]) / 2

fig, ax = plt.subplots(figsize=(5.6, 4.6))
ax.plot([0, 1], [0, 1], color="grey", linestyle="--", linewidth=1,
        label="Perfect calibration")
ax.plot(centres, plotted["accuracy"], marker="o", color=PALETTE[0],
        label="Observed accuracy")
ax.bar(centres, plotted["count"] / plotted["count"].sum(), width=0.08,
       color=PALETTE[0], alpha=0.25, label="Share of predictions")
ax.set_xlabel("Predicted confidence"); ax.set_ylabel("Empirical accuracy")
ax.set_title(f"Figure 7. Reliability diagram (expected calibration error {ece_value:.3f})")
ax.set_xlim(0, 1); ax.set_ylim(0, 1); ax.legend(loc="upper left")
plt.show()
plt.close(fig)

# Size of the error that the representation forces. Newswires sharing a
# multi-hot vector are indistinguishable to any model built on this input, so
# the best available rule is to predict the majority label within each group of
# identical vectors. What that rule still gets wrong is the floor the
# representation imposes, and it can be compared with the observed error rate.
def representation_error_floor(sequences, labels, dimension):
    groups = defaultdict(list)
    for sequence, label in zip(sequences, labels):
        groups[frozenset(t for t in sequence if 0 <= t < dimension)].append(int(label))
    forced = sum(len(v) - Counter(v).most_common(1)[0][1] for v in groups.values())
    return forced, len(groups)


floor_rows = []
for scope, seqs, labs in [("Test split", test_data, test_labels),
                          ("Training split", train_data, train_labels)]:
    forced, distinct = representation_error_floor(seqs, labs, final_vocab)
    floor_rows.append({"scope": scope, "examples": len(labs),
                       "distinct_vectors": distinct, "forced_errors": forced,
                       "error_floor": round(forced / len(labs), 4)})

floor_table = pd.DataFrame(floor_rows)
test_error_rate = 1 - test_acc
test_floor = float(floor_table.loc[floor_table["scope"] == "Test split",
                                   "error_floor"].iloc[0])
print(f"Observed test error rate {test_error_rate:.4f}; the representation "
      f"forces {test_floor:.4f} of it, a share of {test_floor / test_error_rate:.1%}")
floor_table

Observed test error rate 0.1968; the representation forces 0.0085 of it, a share of 4.3%

# The appendecises were generated with the help of artificial intelligence, specifically skimming the notebook and ..
# .. generating the provenance table and gathering the data points for the summary table. The AI was prompted with ..
# .. the notebook and the instructions to extract the relevant information. The AI's output was then reviewed and .. 
# .. edited by a human to ensure accuracy and clarity.

provenance = pd.DataFrame(
    {
        "element": [
            "Multi-hot vectorisation function",
            "Reference two-layer 64-64 architecture",
            "Universal workflow structure",
            "Categorical cross-entropy and softmax output",
            "Macro and weighted F1, classification report, confusion matrix",
            "One-hot targets via to_categorical",
            "Stratified split and learning-curve subsets",
            "Diagnostic metric suite, macro one-vs-rest AUC, calibration error",
            "Repeated-split sensitivity study and paired comparison",
            "Per-class bootstrap intervals and support-qualified ranking",
            "Row-normalised confusion block with an outside-class column",
            "Class-stratified bootstrap intervals",
            "Representation error floor from identical multi-hot vectors",
            "Experiment runner, phase orchestration, selection rule, "
            "stability and bootstrap analysis",
        ],
        "nature": [
            "Adapted from Chollet (2017), Chapter 3",
            "Conceptual influence from Chollet (2017) Reuters example",
            "Followed from Chollet (2017), Section 4.5",
            "Standard Keras API use",
            "Standard scikit-learn API use",
            "Standard Keras API use",
            "Standard scikit-learn and NumPy use",
            "Original contribution built on scikit-learn and NumPy",
            "Original contribution for this report",
            "Original contribution built on scikit-learn and NumPy",
            "Original contribution built on scikit-learn and NumPy",
            "Original contribution built on scikit-learn and NumPy",
            "Original contribution for this report",
            "Original contribution for this report",
        ],
        "location": [
            "Section 2.4", "Section 2.7.1", "Section 2", "Sections 2.4 to 3",
            "Sections 2.5 to 4", "Section 2.4", "Sections 2.3 and 2.7.6",
            "Sections 2.2, 3, and 4", "Section 2.7.5", "Section 4",
            "Section 4", "Section 3", "Section 4.1", "Sections 2.5 to 3",
        ],
    }
)
provenance

# Single collected record of the headline values quoted in the prose, so each
# claim in the HTML document can be checked against one table.
v5 = float(phase3_df.loc[phase3_df["vocab"] == VOCAB_SMALL, "val_macro_f1"].iloc[0])
v10 = float(phase3_df.loc[phase3_df["vocab"] == VOCAB_MAIN, "val_macro_f1"].iloc[0])
ranked_final = stability_summary.sort_values("macro_f1_mean", ascending=False).reset_index(drop=True)
lead_row, second_row = ranked_final.iloc[0], ranked_final.iloc[1]
lc_sorted = learning_summary.sort_values("fraction")
probe_final_gap = round(float(probe_hist["accuracy"][-1] - probe_hist["val_accuracy"][-1]), 4)

split_lead = split_summary.loc[split_summary["name"] == lead_name].iloc[0]
split_other = split_summary.loc[split_summary["name"] == other_name].iloc[0]
top_raw = extremes_ci[extremes_ci["group"] == "strongest"].iloc[0]
qual_strong = qualified[qualified["group"] == "strongest"].iloc[0]
qual_weak = qualified[qualified["group"] == "weakest"].iloc[0]

report_values = {
    "n_train": len(train_data),
    "n_test": len(test_data),
    "med_len": int(np.median(train_lengths)),
    "largest_share": f"{class_counts.max() / len(train_labels):.1%}",
    "smallest_support": int(class_counts.min()),
    "uniform_acc": round(uniform_acc, 4),
    "majority_acc": round(majority_acc, 4),
    "majority_macro_f1": round(majority_macro, 4),
    "baseline_macro_f1": baseline_record["val_macro_f1"],
    "baseline_acc": baseline_record["val_acc"],
    "probe_best_epoch": int(probe_best_epoch),
    "probe_final_gap": probe_final_gap,
    "phase1_best": phase1_best["name"],
    "dropout_macro_f1_by_rate": phase2_df["val_macro_f1"].tolist(),
    "dropout_gap_by_rate": phase2_df["acc_gap"].tolist(),
    "dropout_acc_by_rate": phase2_df["val_acc"].tolist(),
    "vocab5k_macro_f1": round(v5, 4),
    "vocab10k_macro_f1": round(v10, 4),
    "vocab_diff": round(v10 - v5, 4),
    "finalist_lead": lead_row["name"],
    "finalist_lead_macro_f1": round(float(lead_row["macro_f1_mean"]), 4),
    "finalist_lead_std": round(float(lead_row["macro_f1_std"]), 4),
    "finalist_second": second_row["name"],
    "finalist_second_macro_f1": round(float(second_row["macro_f1_mean"]), 4),
    "split_lead_macro_f1_mean": round(float(split_lead["macro_f1_mean"]), 4),
    "split_lead_macro_f1_std": round(float(split_lead["macro_f1_std"]), 4),
    "split_other_macro_f1_mean": round(float(split_other["macro_f1_mean"]), 4),
    "split_lead_range": round(split_range, 4),
    "seed_lead_range": round(seed_range, 4),
    "fixed_split_margin": round(fixed_split_margin, 4),
    "split_paired_diff_mean": round(float(paired_diff.mean()), 4),
    "split_paired_diff_std": round(float(paired_diff.std()), 4),
    "split_wins_for_lead": int((paired_diff > 0).sum()),
    "split_macro_f1_min": round(float(split_wide[lead_name].min()), 4),
    "split_macro_f1_max": round(float(split_wide[lead_name].max()), 4),
    "final_units": final_units,
    "final_dropout": final_dropout,
    "final_vocab": final_vocab,
    "final_epochs": final_epochs,
    "lc_macro_f1_mean": lc_sorted["macro_f1_mean"].tolist(),
    "lc_macro_f1_step": lc_sorted["macro_f1_step"].dropna().tolist(),
    "lc_acc_step": lc_sorted["acc_step"].dropna().tolist(),
    "lc_macro_f1_std": lc_sorted["macro_f1_std"].tolist(),
    "lc_acc_mean": lc_sorted["acc_mean"].tolist(),
    "test_loss": round(test_loss, 4),
    "test_acc": round(test_acc, 4),
    "test_acc_ci": list(acc_ci),
    "test_macro_f1": round(test_macro, 4),
    "test_macro_f1_ci": list(macro_ci),
    "test_weighted_f1": round(test_weighted, 4),
    "test_bal_acc": round(test_bal_acc, 4),
    "test_bal_acc_ci": list(bal_ci),
    "test_kappa": round(test_kappa, 4),
    "test_mcc": round(test_mcc, 4),
    "test_top3": round(test_top3, 4),
    "test_top5": round(test_top5, 4),
    "test_auc": round(test_auc, 4),
    "test_ece": round(test_ece, 4),
    "top_raw_class": int(top_raw["class"]),
    "top_raw_support": int(top_raw["support"]),
    "top_raw_ci": [float(top_raw["f1_ci_lower"]), float(top_raw["f1_ci_upper"])],
    "qualified_classes": int(len(well_supported)),
    "qualified_coverage": int(well_supported["support"].sum()),
    "qualified_strongest": [int(qual_strong["class"]), float(qual_strong["f1"]),
                            int(qual_strong["support"])],
    "qualified_weakest": [int(qual_weak["class"]), float(qual_weak["f1"]),
                          int(qual_weak["support"])],
    "split_choice": split_choice,
    "rho_support_f1_all": round(float(rho_all), 3),
    "rho_support_f1_qualified": round(float(rho_qualified), 3),
    "strat_acc_ci": list(strat_acc_ci),
    "strat_macro_f1_ci": list(strat_macro_ci),
    "bootstrap_class_omission_rate": round(float((omitted > 0).mean()), 3),
    "phase1_margin": phase1_margin,
    "strat_lower_shift": lower_shift,
    "strat_width_change": width_change,
    "lc_final_step_mean": round(float(final_step.mean()), 4),
    "lc_final_step_std": round(float(final_step.std(ddof=1)), 4),
    "lc_final_step_t": round(float(final_step.mean() / step_se), 2),
    "test_error_rate": round(float(test_error_rate), 4),
    "test_repr_floor": round(test_floor, 4),
    "test_repr_floor_share": round(float(test_floor / test_error_rate), 3),
    "train_repr_floor": float(floor_table.loc[1, "error_floor"]),
    "fig5_max_outside_share": float(outside_share["outside_share"].max()),
    "fig5_mean_outside_share": round(float(outside_share["outside_share"].mean()), 3),
    "conf_correct_median": round(float(conf_summary.loc["correct", "median"]), 4),
    "conf_incorrect_median": round(float(conf_summary.loc["incorrect", "median"]), 4),
    "n_fits": FIT_COUNTER["n"],
}
report_values_df = pd.DataFrame(
    {"quantity": list(report_values.keys()),
     "value": [str(v) for v in report_values.values()]}
)
report_values_df

	name	macro_f1_mean	macro_f1_std	acc_mean	acc_std	loss_mean	loss_std
0	vocab_5000	0.5003	0.0262	0.8044	0.0066	0.9038	0.0362
1	wide_128_128	0.5022	0.0351	0.8068	0.0053	0.8984	0.0318

	fraction	dev_examples	macro_f1_mean	macro_f1_std	acc_mean	acc_std	loss_mean	macro_f1_step	acc_step
0	0.25	1797	0.3463	0.0190	0.7637	0.0019	1.0844	NaN	NaN
1	0.50	3597	0.4296	0.0160	0.7877	0.0020	0.9757	0.0833	0.0240
2	0.75	5388	0.4825	0.0315	0.8004	0.0035	0.9162	0.0529	0.0127
3	1.00	7185	0.5491	0.0495	0.8142	0.0062	0.8675	0.0666	0.0138

	metric	point_estimate	ci_lower_95	ci_upper_95
0	Accuracy	0.8032	0.7867	0.8197
1	Macro F1	0.5537	0.4779	0.5804
2	Balanced accuracy	0.5134	0.4675	0.5621

	class	precision	recall	f1	support
3	3	0.924	0.941	0.932	813
4	4	0.828	0.876	0.851	474
19	19	0.678	0.744	0.710	133
1	1	0.741	0.790	0.765	105
16	16	0.645	0.808	0.717	99
11	11	0.540	0.807	0.647	83
20	20	0.648	0.500	0.565	70
8	8	0.730	0.711	0.720	38
13	13	0.618	0.568	0.592	37
25	25	0.893	0.806	0.847	31

	group	class	f1	support	f1_ci_lower	f1_ci_upper	f1_ci_width
0	strongest	45	1.000	1	0.000	1.000	1.000
1	strongest	3	0.932	813	0.919	0.945	0.025
2	strongest	10	0.900	30	0.807	0.971	0.164
3	strongest	6	0.897	14	0.741	1.000	0.259
4	strongest	33	0.889	5	0.500	1.000	0.500
5	weakest	22	0.000	7	0.000	0.000	0.000
6	weakest	14	0.000	2	0.000	0.000	0.000
7	weakest	37	0.000	2	0.000	0.000	0.000
8	weakest	36	0.125	11	0.000	0.375	0.375
9	weakest	40	0.182	10	0.000	0.500	0.500

Topic Classification of Reuters Newswires with Densely Connected Networks¶

A systematic application of the universal deep learning workflow¶

Abstract¶

1. Introduction and scope¶

1.1 Aim¶

1.2 Assignment restrictions¶

1.3 Research questions¶

1.4 Structure¶

Environment, reproducibility, and global controls¶

2. The universal deep learning workflow¶

2.1 Step 1: Defining the problem and assembling the dataset¶

2.2 Step 2: Choosing a measure of success¶

2.3 Step 3: Deciding on an evaluation protocol¶

2.4 Step 4: Preparing the data¶

2.5 Step 5: Developing a model that performs above a baseline¶

2.6 Step 6: Scaling up and establishing overfitting¶

2.7 Step 7: Regularising the model and tuning hyperparameters¶

2.7.1 Phase 1: Capacity¶

2.7.2 Phase 2: Dropout¶

2.7.3 Phase 3: Vocabulary size¶

2.7.4 Stability across training seeds¶

2.7.5 Sensitivity to the choice of validation split¶

2.7.6 Learning curve¶

3. Final model and untouched test evaluation¶

4. Interpretation and error analysis¶

4.1 On the level of accuracy and the decision not to regularise¶

5. Limitations and reproducibility¶

6. Conclusion¶

References¶

Appendix A. Code provenance¶

Appendix B. Summary of reported headline values¶

	component	version
0	Python	3.12.0
1	TensorFlow	2.18.0
2	Keras	3.6.0
3	NumPy	2.1.2
4	pandas	2.2.2
5	scikit-learn	1.5.2
6	Matplotlib	3.9.2
7	seaborn	0.13.2

	quantity	value
0	Training examples	8982
1	Test examples	2246
2	Number of classes	46
3	Vocabulary limit	10000
4	Median newswire length (tokens)	95
5	Mean newswire length (tokens)	145.5
6	Maximum newswire length (tokens)	2376
7	Largest class share	35.2% (class 3)
8	Smallest class support	10 (class 35)

	measure	role
0	Categorical cross-entropy	Optimisation objective and epoch selection
1	Macro F1	Primary model-selection measure
2	Accuracy	Secondary, interpretable overall measure
3	Weighted F1	Secondary, support-weighted quality
4	Generalisation gap	Diagnostic for overfitting
5	Parameter count	Tie-break and efficiency
6	Balanced accuracy	Final diagnostic, imbalance-robust recall
7	Cohen's kappa	Final diagnostic, chance-corrected agreement
8	Matthews correlation coefficient	Final diagnostic, balanced multiclass correlation
9	Top-3 and top-5 accuracy	Final diagnostic, ranked-shortlist quality
10	Macro one-vs-rest ROC AUC	Final diagnostic, threshold-independent separability
11	Expected calibration error	Final diagnostic, confidence calibration

	partition	examples	share of training
0	Development	7185	80%
1	Validation	1797	20%
2	Test (reserved)	2246	held out

	name	hidden_units	params	best_epoch	val_loss	val_acc	val_macro_f1	acc_gap	seconds
0	small_16	[16]	160798	30	0.8871	0.8058	0.4855	0.1342	3.2
1	reference_64_64	[64, 64]	647214	11	0.8899	0.8075	0.4867	0.1248	2.9
2	wide_128_128	[128, 128]	1302574	9	0.8577	0.8141	0.5298	0.1245	3.4
3	deep_64_64_64	[64, 64, 64]	651374	10	1.0042	0.7963	0.4071	0.1168	2.8
4	bottleneck_64_4	[64, 4]	640554	22	1.2937	0.6956	0.1343	0.0895	4.4

	name	dropout	params	best_epoch	val_loss	val_acc	val_macro_f1	acc_gap	seconds
0	drop_0.0	0.00	1302574	9	0.8577	0.8141	0.5298	0.1245	3.4
1	drop_0.2	0.20	1302574	9	0.8672	0.8158	0.4898	0.0982	3.4
2	drop_0.35	0.35	1302574	11	0.8943	0.8164	0.4577	0.0836	3.9
3	drop_0.5	0.50	1302574	14	0.9322	0.8058	0.3792	0.0646	4.5

	name	vocab	params	best_epoch	val_loss	val_acc	val_macro_f1	acc_gap	seconds
0	vocab_5000	5000	662574	10	0.8615	0.8086	0.5261	0.1326	2.2
1	vocab_10000	10000	1302574	9	0.8577	0.8141	0.5298	0.1245	3.3

	name	seed	best_epoch	val_loss	val_acc	val_macro_f1
0	wide_128_128	7	9	0.8859	0.8047	0.4888
1	wide_128_128	42	9	0.8577	0.8141	0.5298
2	wide_128_128	113	10	0.8557	0.8208	0.5999
3	wide_128_128	2026	12	0.8788	0.8186	0.6008
4	wide_128_128	31415	9	0.8595	0.8130	0.5261
5	vocab_5000	7	8	0.8967	0.8047	0.4594
6	vocab_5000	42	10	0.8615	0.8086	0.5261
7	vocab_5000	113	10	0.8604	0.8136	0.5582
8	vocab_5000	2026	10	0.8670	0.8141	0.5505
9	vocab_5000	31415	9	0.8650	0.8152	0.5089

name	vocab_5000	wide_128_128
split_seed
7	0.4788	0.5358
42	0.5261	0.5298
113	0.5295	0.5162
2026	0.4937	0.4642
31415	0.4735	0.4648

	property	value
0	Hidden units	[128, 128]
1	Dropout rate	0.0
2	Vocabulary size	10000
3	Output layer	Dense(46, softmax)
4	Loss	categorical_crossentropy
5	Optimizer	rmsprop
6	Training epochs	9
7	Epoch rule	Rounded median best epoch over five seeds

	baseline	val_accuracy	val_macro_f1
0	Uniform chance	0.0217	n/a
1	Majority class	0.3517	0.0113

	true_class	predicted_class	count
0	3	4	29
1	4	3	24
2	20	19	12
3	36	11	9
4	19	11	9
5	13	16	9
6	4	19	8
7	16	3	8
8	17	16	7
9	19	3	7

	quantity	value
0	n_train	8982
1	n_test	2246
2	med_len	95
3	largest_share	35.2%
4	smallest_support	10
5	uniform_acc	0.0217
6	majority_acc	0.3517
7	majority_macro_f1	0.0113
8	baseline_macro_f1	0.4855
9	baseline_acc	0.8058
10	probe_best_epoch	7
11	probe_final_gap	0.147
12	phase1_best	wide_128_128
13	dropout_macro_f1_by_rate	[0.5298, 0.4898, 0.4577, 0.3792]
14	dropout_gap_by_rate	[0.1245, 0.0982, 0.0836, 0.0646]
15	dropout_acc_by_rate	[0.8141, 0.8158, 0.8164, 0.8058]
16	vocab5k_macro_f1	0.5261
17	vocab10k_macro_f1	0.5298
18	vocab_diff	0.0037
19	finalist_lead	wide_128_128
20	finalist_lead_macro_f1	0.5491
21	finalist_lead_std	0.0495
22	finalist_second	vocab_5000
23	finalist_second_macro_f1	0.5206
24	split_lead_macro_f1_mean	0.5022
25	split_lead_macro_f1_std	0.0351
26	split_other_macro_f1_mean	0.5003
27	split_lead_range	0.0716
28	seed_lead_range	0.112
29	fixed_split_margin	0.0285
30	split_paired_diff_mean	0.0018
31	split_paired_diff_std	0.033
32	split_wins_for_lead	2
33	split_macro_f1_min	0.4642
34	split_macro_f1_max	0.5358
35	final_units	[128, 128]
36	final_dropout	0.0
37	final_vocab	10000
38	final_epochs	9
39	lc_macro_f1_mean	[0.3463, 0.4296, 0.4825, 0.5491]

	metric	value
0	Cross-entropy	0.9072
1	Accuracy	0.8032
2	Balanced accuracy	0.5134
3	Top-3 accuracy	0.9052
4	Top-5 accuracy	0.9341
5	Macro precision	0.7066
6	Macro recall	0.5134
7	Macro F1	0.5537
8	Weighted precision	0.8036
9	Weighted recall	0.8032
10	Weighted F1	0.7925
11	Cohen's kappa	0.7560
12	Matthews corrcoef	0.7564
13	Macro OVR ROC AUC (46 classes)	0.9611
14	Expected calibration error	0.0581

	metric	scheme	point_estimate	ci_lower_95	ci_upper_95	width
0	Accuracy	Unstratified	0.8032	0.7867	0.8197	0.0330
1	Accuracy	Stratified by class	0.8032	0.7881	0.8175	0.0294
2	Macro F1	Unstratified	0.5537	0.4779	0.5804	0.1025
3	Macro F1	Stratified by class	0.5537	0.5006	0.5839	0.0833

	class	support	outside_share
0	1	105	0.019
1	3	813	0.005
2	4	474	0.011
3	8	38	0.053
4	10	30	0.033
5	11	83	0.060
6	13	37	0.000
7	16	99	0.010
8	19	133	0.023
9	20	70	0.014
10	21	27	0.111
11	25	31	0.000

	element	nature	location
0	Multi-hot vectorisation function	Adapted from Chollet (2017), Chapter 3	Section 2.4
1	Reference two-layer 64-64 architecture	Conceptual influence from Chollet (2017) Reuters example	Section 2.7.1
2	Universal workflow structure	Followed from Chollet (2017), Section 4.5	Section 2
3	Categorical cross-entropy and softmax output	Standard Keras API use	Sections 2.4 to 3
4	Macro and weighted F1, classification report, confusion matrix	Standard scikit-learn API use	Sections 2.5 to 4
5	One-hot targets via to_categorical	Standard Keras API use	Section 2.4
6	Stratified split and learning-curve subsets	Standard scikit-learn and NumPy use	Sections 2.3 and 2.7.6
7	Diagnostic metric suite, macro one-vs-rest AUC, calibration error	Original contribution built on scikit-learn and NumPy	Sections 2.2, 3, and 4
8	Repeated-split sensitivity study and paired comparison	Original contribution for this report	Section 2.7.5
9	Per-class bootstrap intervals and support-qualified ranking	Original contribution built on scikit-learn and NumPy	Section 4
10	Row-normalised confusion block with an outside-class column	Original contribution built on scikit-learn and NumPy	Section 4
11	Class-stratified bootstrap intervals	Original contribution built on scikit-learn and NumPy	Section 3
12	Representation error floor from identical multi-hot vectors	Original contribution for this report	Section 4.1
13	Experiment runner, phase orchestration, selection rule, stability and bootstrap analysis	Original contribution for this report	Sections 2.5 to 3