Vocabulary Scaling Law

import numpy as np
from scipy.optimize import minimize

# EVOLVE-BLOCK-START
def scaling_law_func(data_points, params):
    """
    Lossu ≈ C0·P^(–a) + C1·(D / V^g)^(–b) + C2·log(P) + C3
    params = [C0, C1, C2, C3, a, b, g]
    """
    X = np.atleast_2d(data_points)
    P, V, D = X[:,0], X[:,1], X[:,2]
    C0, C1, C2, C3, a, b, g = params
    R = D / (V**g)
    return C0 * P**(-a) + C1 * R**(-b) + C2 * np.log(P) + C3

def fit_scaling_law(data_points, loss_values):
    """
    Fit [C0,C1,C2,C3,a,b,g] by:
     1) optimizing exponents (a,b,g) in log-space via L-BFGS-B,
     2) recovering linear coefficients [C0,C1,C2,C3] at each step
        via ridge-regularized least squares,
     3) final solve at optimal exponents.
    """
    X = np.atleast_2d(data_points)
    P, V, D = X[:,0], X[:,1], X[:,2]
    y = np.asarray(loss_values).ravel()
    ridge = 1e-12

    def mse_for_log_exps(log_exps):
        a, b, g = np.exp(log_exps)
        R = D / (V**g)
        M = np.vstack((P**(-a), R**(-b), np.log(P), np.ones_like(P))).T
        A = M.T.dot(M) + ridge * np.eye(4)
        C = np.linalg.solve(A, M.T.dot(y))
        y_pred = M.dot(C)
        return np.mean((y_pred - y)**2)

    # init exponents around 0.5
    init_log = np.log([0.5, 0.5, 0.5])
    bounds = [(-8,8), (-8,8), (-5,5)]
    res = minimize(mse_for_log_exps, init_log,
                   method='L-BFGS-B', bounds=bounds)
    a_opt, b_opt, g_opt = np.exp(res.x)

    # final linear solve for C0,C1,C2,C3
    R_opt = D / (V**g_opt)
    M_final = np.vstack((P**(-a_opt), R_opt**(-b_opt),
                         np.log(P), np.ones_like(P))).T
    A_final = M_final.T.dot(M_final) + ridge * np.eye(4)
    C0, C1, C2, C3 = np.linalg.solve(A_final, M_final.T.dot(y))

    return np.array([C0, C1, C2, C3, a_opt, b_opt, g_opt])
# EVOLVE-BLOCK-END

import numpy as np
from scipy.optimize import minimize

# EVOLVE-BLOCK-START
def scaling_law_func(data_points, params):
    """
    6-parameter dual-term power law:
      Lossu = C0
            + C1 * P^{-alpha} * D^{-delta}
            + C2 * V^{-beta}  * D^{-delta}
    params = [C0, C1, C2, alpha, beta, delta]
    """
    X = np.atleast_2d(data_points)
    P, V, D = X[:,0], X[:,1], X[:,2]
    C0, C1, C2, alpha, beta, delta = params
    return C0 + C1 * P**(-alpha) * D**(-delta) + C2 * V**(-beta) * D**(-delta)

def fit_scaling_law(data_points, loss_values):
    """
    Fit exponents in log-space (L-BFGS-B) then solve linear coeffs by least squares.
    """
    X = np.atleast_2d(data_points)
    y = np.asarray(loss_values).ravel()
    P, V, D = X[:,0], X[:,1], X[:,2]

    def mse_log_exps(log_e):
        a, b, d = np.exp(log_e)
        t1 = P**(-a)*D**(-d)
        t2 = V**(-b)*D**(-d)
        M = np.vstack((np.ones_like(y), t1, t2)).T
        c, *_ = np.linalg.lstsq(M, y, rcond=None)
        return np.mean((M.dot(c)-y)**2)

    # init exponents ~0.5
    res = minimize(mse_log_exps, np.log([0.5,0.5,0.5]),
                   method='L-BFGS-B', bounds=[(-10,10)]*3)
    alpha, beta, delta = np.exp(res.x)

    # final linear solve
    t1 = P**(-alpha)*D**(-delta)
    t2 = V**(-beta)*D**(-delta)
    M_final = np.vstack((np.ones_like(y), t1, t2)).T
    C0, C1, C2 = np.linalg.lstsq(M_final, y, rcond=None)[0]

    return np.array([C0, C1, C2, alpha, beta, delta])
# EVOLVE-BLOCK-END

import numpy as np
from scipy.optimize import minimize

# EVOLVE-BLOCK-START
def scaling_law_func(data_points, params):
    """
    Simplified power-law scaling:
      Lossu ≈ C0 + C1·P^{-α}·D^{-β} + C2·V^{-γ}·D^{-β}
    where
      P = non-vocab parameter count,
      V = vocabulary size,
      D = number of characters.
    params = [C0, C1, C2, α, β, γ]
    """
    X = np.atleast_2d(data_points)
    P = X[:, 0]
    V = X[:, 1]
    D = X[:, 2]
    C0, C1, C2, alpha, beta, gamma = params

    # log-space for numerical stability
    logP = np.log(P)
    logV = np.log(V)
    logD = np.log(D)

    # compute two interaction terms with shared D exponent
    f1 = np.exp(-alpha * logP - beta * logD)
    f2 = np.exp(-gamma * logV - beta * logD)

    return C0 + C1 * f1 + C2 * f2

def fit_scaling_law(data_points, loss_values):
    """
    Fit the 6-parameter simplified model:
      Lossu ≈ C0 + C1·P^{-α}·D^{-β} + C2·V^{-γ}·D^{-β}
    1) Optimize exponents [α, β, γ] in log-space via L-BFGS-B
    2) Solve linear coefficients [C0, C1, C2] by least squares
    Returns array([C0, C1, C2, α, β, γ]).
    """
    X = np.atleast_2d(data_points)
    y = np.asarray(loss_values).ravel()
    logP = np.log(X[:, 0])
    logV = np.log(X[:, 1])
    logD = np.log(X[:, 2])

    # Objective: MSE after fitting linear coeffs for given exponents
    def objective(log_exps):
        alpha, beta, gamma = np.exp(log_exps)
        f1 = np.exp(-alpha * logP - beta * logD)
        f2 = np.exp(-gamma * logV - beta * logD)
        M = np.vstack([f1, f2, np.ones_like(y)]).T
        coeffs, *_ = np.linalg.lstsq(M, y, rcond=None)
        y_pred = M.dot(coeffs)
        return np.mean((y_pred - y) ** 2)

    # Initialize log-exponents
    init_log = np.log([0.5, 0.5, 0.5])
    res = minimize(objective, init_log, method='L-BFGS-B')
    alpha_opt, beta_opt, gamma_opt = np.exp(res.x)

    # Solve for linear coefficients with optimized exponents
    f1 = np.exp(-alpha_opt * logP - beta_opt * logD)
    f2 = np.exp(-gamma_opt * logV - beta_opt * logD)
    M_final = np.vstack([f1, f2, np.ones_like(y)]).T
    C1_opt, C2_opt, C0_opt = np.linalg.lstsq(M_final, y, rcond=None)[0]

    return np.array([C0_opt, C1_opt, C2_opt, alpha_opt, beta_opt, gamma_opt])
# EVOLVE-BLOCK-END

import numpy as np
from scipy.optimize import minimize

# EVOLVE-BLOCK-START
def scaling_law_func(data_points, params):
    """
    Predict unigram-normalized loss (Lossu) given:
      data_points: array of shape (N,3) with columns 
                   [P_non_vocab, vocab_size, num_characters]
      params: 6 parameters [C0, C1, C2, alpha, beta, gamma]
    Model:
      Lossu = C0 
            + C1 * P^{-alpha} * D^{-beta}
            + C2 * V^{-gamma} * D^{-beta}
    where P = non-vocab parameters, V = vocab size, D = number of characters.
    """
    X = np.atleast_2d(np.asarray(data_points, dtype=float))
    P = X[:, 0]
    V = X[:, 1]
    D = X[:, 2]
    C0, C1, C2, alpha, beta, gamma = params
    term_p = P**(-alpha) * D**(-beta)
    term_v = V**(-gamma) * D**(-beta)
    return C0 + C1 * term_p + C2 * term_v

def fit_scaling_law(data_points, loss_values):
    """
    Fit the 6-parameter scaling law model:
      params = [C0, C1, C2, alpha, beta, gamma]
    We optimize exponents (alpha, beta, gamma) in log-space
    and solve for linear coefficients (C0, C1, C2) via least squares.
    """
    X = np.atleast_2d(np.asarray(data_points, dtype=float))
    y = np.asarray(loss_values, dtype=float).ravel()
    P = X[:, 0]
    V = X[:, 1]
    D = X[:, 2]

    def mse_exps(log_exps):
        # enforce positivity of exponents
        a, b, g = np.exp(log_exps)
        t_p = P**(-a) * D**(-b)
        t_v = V**(-g) * D**(-b)
        # design matrix: [ones, t_p, t_v]
        M = np.vstack([np.ones_like(y), t_p, t_v]).T
        coefs, *_ = np.linalg.lstsq(M, y, rcond=None)
        y_pred = M.dot(coefs)
        return np.mean((y - y_pred)**2)

    # initialize log-exponents
    init = np.log([0.5, 0.5, 0.5])
    res = minimize(mse_exps, init, method='L-BFGS-B')
    alpha, beta, gamma = np.exp(res.x)

    # final linear solve for C0, C1, C2
    t_p = P**(-alpha) * D**(-beta)
    t_v = V**(-gamma) * D**(-beta)
    M_final = np.vstack([np.ones_like(y), t_p, t_v]).T
    C0, C1, C2 = np.linalg.lstsq(M_final, y, rcond=None)[0]

    return np.array([C0, C1, C2, alpha, beta, gamma])
# EVOLVE-BLOCK-END