Overparameterization & Generalization (Double Descent)

import numpy as np

n_train, n_test, max_d = 80, 2000, 300
noise = 0.2
dims = [10, 30, 60, 80, 100, 150, 300]

def fit_linear(X, y):
    n, d = X.shape
    if d < n:
        return np.linalg.lstsq(X, y, rcond=None)[0]
    return X.T @ np.linalg.solve(X @ X.T + 1e-8 * np.eye(n), y)

def trial(seed, d):
    rs = np.random.RandomState(seed)
    w_true = rs.randn(max_d) / np.sqrt(max_d)
    Xtr_full = rs.randn(n_train, max_d)
    Xte_full = rs.randn(n_test, max_d)
    ytr = Xtr_full @ w_true + noise * rs.randn(n_train)
    yte = Xte_full @ w_true + noise * rs.randn(n_test)

    Xtr, Xte = Xtr_full[:, :d], Xte_full[:, :d]
    w_hat = fit_linear(Xtr, ytr)
    train_mse = float(np.mean((Xtr @ w_hat - ytr) ** 2))
    test_mse = float(np.mean((Xte @ w_hat - yte) ** 2))
    return train_mse, test_mse

curve = []
for d in dims:
    runs = np.array([trial(seed, d) for seed in range(20)])
    train_mse, test_mse = runs.mean(axis=0)
    curve.append((d, train_mse, test_mse))
    print("d =", f"{d:>3}", "train_mse =", round(train_mse, 3), "test_mse =", round(test_mse, 3))

by_d = {d: (train, test) for d, train, test in curve}
assert by_d[80][0] < 1e-6
assert by_d[80][1] > by_d[60][1] and by_d[80][1] > by_d[100][1]
assert by_d[300][1] < by_d[80][1]

This is a fixed synthetic linear task, not a theorem about every large model. It makes the interpolation threshold visible: training error reaches zero near $d=n$, test error spikes there, and a larger minimum-norm interpolant can recover lower test error.