神经网络与机器学习入门直觉

┌─────────────────────────────────────────────┐
│  人工智能 AI（让机器表现出"智能行为"的所有方法）      │
│                                               │
│   ┌───────────────────────────────────────┐ │
│   │  机器学习 ML（从数据中自动总结规律的方法）    │ │
│   │                                         │ │
│   │    ┌─────────────────────────────┐    │ │
│   │    │ 深度学习 DL（用多层神经网络    │    │ │
│   │    │ 实现机器学习的一类具体方法）    │    │ │
│   │    └─────────────────────────────┘    │ │
│   │                                         │ │
│   └───────────────────────────────────────┘ │
└─────────────────────────────────────────────┘

规则（人写的代码）+ 数据（输入）  ──→  程序运行  ──→  结果

数据（输入）+ 结果（标签/答案） ──→  训练算法  ──→  规则（模型/参数）

def is_spam_rule_based(email_text):
    spam_keywords = ["免费领取", "点击链接", "限时抢购", "中奖", "转账汇款"]
    for keyword in spam_keywords:
        if keyword in email_text:
            return True
    return False

def is_spam_ml(features, weights, bias, threshold=0.5):
    """
    features: 从邮件里抽取的数字特征，如 [链接数, 感叹号数, 是否已知联系人...]
    weights:  训练算法从数据里学出来的权重（不是人写死的）
    bias:     偏置
    """
    score = sum(w * x for w, x in zip(weights, features)) + bias
    prob = 1 / (1 + 2.718281828 ** (-score))   # sigmoid 把分数压到 0~1
    return prob > threshold, prob

# 这些 weights/bias 是"训练"出来的，不是人拍脑袋定的
weights = [2.3, 0.8, -3.1]     # 分别对应：含链接、感叹号多、来自已知联系人
bias = -1.0
new_email = [1, 0, 0]          # 有链接、无感叹号、非已知联系人
print(is_spam_ml(new_email, weights, bias))   # (True, 0.78) → 判为垃圾邮件

z = w1*x1 + w2*x2 + ... + wn*xn + b
output = activation(z)

import math

def sigmoid(z):
    """sigmoid 激活函数：把任意实数压缩到 (0, 1) 之间，常用于二分类输出"""
    return 1 / (1 + math.exp(-z))

def relu(z):
    """ReLU 激活函数：负数变 0，正数保持不变，是目前隐藏层最常用的激活函数"""
    return max(0, z)

def tanh(z):
    """tanh 激活函数：把输入压到 (-1, 1)，以 0 为中心，收敛常比 sigmoid 快"""
    return math.tanh(z)

def neuron_forward(inputs, weights, bias, activation_fn):
    """
    模拟单个神经元的前向计算
    inputs: 输入特征列表，比如 [0.5, 1.2, -0.3]
    weights: 每个输入对应的权重列表
    bias: 偏置标量
    activation_fn: 激活函数
    """
    # 第一步：加权求和
    z = sum(w * x for w, x in zip(weights, inputs)) + bias
    # 第二步：激活
    return activation_fn(z)

# 示例：一个判断"是否值得投篮"的玩具神经元
# 输入特征：[距离篮筐远近(归一化), 防守压力(归一化)]
inputs = [0.3, 0.6]
weights = [-1.5, -0.8]   # 距离越远、防守越紧，权重为负，抑制输出
bias = 1.0

score_sigmoid = neuron_forward(inputs, weights, bias, sigmoid)
score_relu = neuron_forward(inputs, weights, bias, relu)

print(f"sigmoid 输出: {score_sigmoid:.3f}")  # 输出一个 0~1 之间的"投篮信心值"
print(f"relu 输出: {score_relu:.3f}")

import numpy as np

def relu(x):
    return np.maximum(0, x)

def forward_two_layer(X, W1, b1, W2, b2):
    """
    一个两层（一个隐藏层）神经网络的前向传播
    X:  输入 (样本数, 输入维度)
    W1: 输入层到隐藏层的权重 (输入维度, 隐藏维度)
    W2: 隐藏层到输出层的权重 (隐藏维度, 输出维度)
    """
    z1 = X @ W1 + b1        # 隐藏层加权求和
    a1 = relu(z1)           # 隐藏层激活
    z2 = a1 @ W2 + b2       # 输出层加权求和
    return z2, a1           # z2 是最终输出（如回归值或分类 logits）

# 造一批假数据试运行
np.random.seed(0)
X = np.random.randn(5, 3)          # 5 个样本，每个 3 维特征
W1 = np.random.randn(3, 4) * 0.1   # 隐藏层 4 个神经元
b1 = np.zeros(4)
W2 = np.random.randn(4, 1) * 0.1   # 输出 1 个值
b2 = np.zeros(1)

output, hidden = forward_two_layer(X, W1, b1, W2, b2)
print("隐藏层输出形状:", hidden.shape)   # (5, 4)
print("网络最终输出:\n", output)

# 简化版梯度下降（以拟合 y = w*x + b 为例）
def train_simple_model(x_data, y_data, learning_rate=0.01, epochs=1000):
    w, b = 0.0, 0.0  # 随机初始化（这里简化为从 0 开始）
    n = len(x_data)

    for epoch in range(epochs):
        # 1. 前向计算：用当前参数做预测
        predictions = [w * x + b for x in x_data]

        # 2. 计算损失（均方误差）
        loss = sum((pred - y) ** 2 for pred, y in zip(predictions, y_data)) / n

        # 3. 计算梯度：损失函数对 w 和 b 的偏导数
        grad_w = sum(2 * (pred - y) * x for pred, y, x in zip(predictions, y_data, x_data)) / n
        grad_b = sum(2 * (pred - y) for pred, y in zip(predictions, y_data)) / n

        # 4. 沿梯度反方向更新参数，学习率控制步子迈多大
        w = w - learning_rate * grad_w
        b = b - learning_rate * grad_b

        if epoch % 200 == 0:
            print(f"第{epoch}轮，loss={loss:.4f}, w={w:.3f}, b={b:.3f}")

    return w, b

import numpy as np

np.random.seed(42)
# 造一个非线性可分的玩具二分类数据（异或问题的简化版）
X = np.array([[0,0],[0,1],[1,0],[1,1]], dtype=float)
y = np.array([[0],[1],[1],[0]], dtype=float)   # XOR，线性模型无法解决

def sigmoid(x): return 1 / (1 + np.exp(-x))

# 初始化参数
W1 = np.random.randn(2, 4)
b1 = np.zeros((1, 4))
W2 = np.random.randn(4, 1)
b2 = np.zeros((1, 1))
lr = 0.5

for epoch in range(5000):
    # ---- 前向传播 ----
    z1 = X @ W1 + b1
    a1 = sigmoid(z1)
    z2 = a1 @ W2 + b2
    a2 = sigmoid(z2)                 # 预测概率

    # ---- 损失（二元交叉熵）----
    loss = -np.mean(y*np.log(a2+1e-9) + (1-y)*np.log(1-a2+1e-9))

    # ---- 反向传播（链式法则逐层回传误差）----
    dz2 = a2 - y                      # 输出层误差
    dW2 = a1.T @ dz2
    db2 = dz2.sum(axis=0, keepdims=True)
    da1 = dz2 @ W2.T
    dz1 = da1 * a1 * (1 - a1)         # sigmoid 的导数
    dW1 = X.T @ dz1
    db1 = dz1.sum(axis=0, keepdims=True)

    # ---- 梯度下降更新 ----
    W2 -= lr * dW2; b2 -= lr * db2
    W1 -= lr * dW1; b1 -= lr * db1

    if epoch % 1000 == 0:
        print(f"epoch {epoch}, loss={loss:.4f}")

print("最终预测:\n", np.round(sigmoid((sigmoid(X@W1+b1))@W2+b2), 2))
# 训练后能正确输出接近 [0,1,1,0] 的结果——单个神经元永远做不到

import numpy as np

def train_val_test_split(X, y, ratios=(0.7, 0.15, 0.15), seed=0):
    """按比例把数据随机切成训练/验证/测试三份"""
    n = len(X)
    idx = np.random.RandomState(seed).permutation(n)  # 先打乱，避免顺序偏差
    n_train = int(n * ratios[0])
    n_val = int(n * ratios[1])
    tr, va, te = idx[:n_train], idx[n_train:n_train+n_val], idx[n_train+n_val:]
    return (X[tr], y[tr]), (X[va], y[va]), (X[te], y[te])

X = np.random.randn(1000, 5)
y = (X.sum(axis=1) > 0).astype(int)
(train, val, test) = train_val_test_split(X, y)
print("训练/验证/测试样本数:", len(train[0]), len(val[0]), len(test[0]))
# 输出：700 150 150

import numpy as np

# 1) L2 正则化：在损失里加上权重平方和的惩罚
def loss_with_l2(base_loss, weights, lambda_=0.01):
    l2_penalty = lambda_ * sum(np.sum(w ** 2) for w in weights)
    return base_loss + l2_penalty

# 2) Dropout：训练时随机置零一部分神经元输出
def dropout(a, keep_prob=0.8, training=True):
    if not training:
        return a
    mask = (np.random.rand(*a.shape) < keep_prob) / keep_prob  # 反向缩放保持期望不变
    return a * mask

# 3) 早停：验证损失连续 patience 轮不改善就停止
class EarlyStopping:
    def __init__(self, patience=5):
        self.patience = patience
        self.best = float('inf')
        self.counter = 0
    def should_stop(self, val_loss):
        if val_loss < self.best:
            self.best = val_loss
            self.counter = 0
        else:
            self.counter += 1
        return self.counter >= self.patience

stopper = EarlyStopping(patience=3)
for epoch, v in enumerate([0.9, 0.7, 0.6, 0.61, 0.62, 0.63]):
    if stopper.should_stop(v):
        print(f"第 {epoch} 轮触发早停，最佳验证损失 {stopper.best}")
        break

dy/dx = (dy/du) * (du/dx)

dL/dw = (dL/da) * (da/dz) * (dz/dw)

z = w * x + b
a = σ(z) = 1 / (1 + e^(-z))
L = (a - y)^2

dL/da = 2 * (a - y)
da/dz = σ(z) * (1 - σ(z)) = a * (1 - a)   # sigmoid 导数的经典简化式
dz/dw = x
dz/db = 1

dL/dw = 2 * (a - y) * a * (1 - a) * x
dL/db = 2 * (a - y) * a * (1 - a)

import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def forward_loss(w, b, x, y):
    z = w * x + b
    a = sigmoid(z)
    return (a - y) ** 2, a

# 解析梯度（反向传播）
x, y = 1.5, 1.0
w, b = 0.4, -0.2
loss, a = forward_loss(w, b, x, y)
grad_w_analytic = 2 * (a - y) * a * (1 - a) * x
grad_b_analytic = 2 * (a - y) * a * (1 - a)

# 数值梯度（扰动法）
eps = 1e-6
grad_w_numeric = (forward_loss(w + eps, b, x, y)[0] - forward_loss(w - eps, b, x, y)[0]) / (2 * eps)
grad_b_numeric = (forward_loss(w, b + eps, x, y)[0] - forward_loss(w, b - eps, x, y)[0]) / (2 * eps)

print(f"解析 dL/dw = {grad_w_analytic:.8f}, 数值 dL/dw = {grad_w_numeric:.8f}")
print(f"解析 dL/db = {grad_b_analytic:.8f}, 数值 dL/db = {grad_b_numeric:.8f}")
# 两者应几乎完全一致，相对误差 < 1e-7 才说明反向传播实现正确

import numpy as np

np.random.seed(1)
N, D_in, H, D_out = 16, 4, 8, 3

# 造数据
X = np.random.randn(N, D_in)
y_true = np.random.randn(N, D_out)

# He 初始化（后面章节详解）
W1 = np.random.randn(D_in, H) * np.sqrt(2.0 / D_in)
b1 = np.zeros((1, H))
W2 = np.random.randn(H, D_out) * np.sqrt(2.0 / H)
b2 = np.zeros((1, D_out))

lr = 0.01
for step in range(2000):
    # ---- 前向 ----
    z1 = X @ W1 + b1          # (N, H)
    a1 = np.maximum(0, z1)    # ReLU
    z2 = a1 @ W2 + b2         # (N, D_out)
    # 回归任务，输出层不加激活
    loss = np.mean((z2 - y_true) ** 2)

    # ---- 反向 ----
    dz2 = 2 * (z2 - y_true) / N        # (N, D_out)，MSE 对 z2 的梯度
    dW2 = a1.T @ dz2                    # (H, D_out)
    db2 = dz2.sum(axis=0, keepdims=True)
    da1 = dz2 @ W2.T                    # (N, H)
    dz1 = da1 * (z1 > 0)               # ReLU 的导数：正数处为 1，负数处为 0
    dW1 = X.T @ dz1                     # (D_in, H)
    db1 = dz1.sum(axis=0, keepdims=True)

    # ---- 更新 ----
    W2 -= lr * dW2; b2 -= lr * db2
    W1 -= lr * dW1; b1 -= lr * db1

    if step % 400 == 0:
        print(f"step {step}, loss={loss:.4f}")

Sigmoid:  σ(x) = 1 / (1 + e^(-x))            输出 (0, 1)
Tanh:     tanh(x) = (e^x - e^-x)/(e^x + e^-x) 输出 (-1, 1)
ReLU:     max(0, x)                           输出 [0, +∞)
GELU:     x * Φ(x)   Φ 是标准正态的累积分布    近似平滑 ReLU
Swish:    x * σ(βx)  β 通常取 1                自门控、平滑

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def tanh(x):
    return np.tanh(x)

def relu(x):
    return np.maximum(0, x)

def leaky_relu(x, alpha=0.01):
    return np.where(x > 0, x, alpha * x)

def gelu(x):
    # tanh 近似版本，Transformer 里常用
    return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x ** 3)))

def swish(x, beta=1.0):
    return x * sigmoid(beta * x)

# 在几个代表性输入点上对比输出
xs = np.array([-3.0, -1.0, 0.0, 1.0, 3.0])
print("输入:      ", xs)
print("sigmoid:   ", np.round(sigmoid(xs), 3))
print("tanh:      ", np.round(tanh(xs), 3))
print("relu:      ", np.round(relu(xs), 3))
print("leaky_relu:", np.round(leaky_relu(xs), 3))
print("gelu:      ", np.round(gelu(xs), 3))
print("swish:     ", np.round(swish(xs), 3))

import numpy as np

def d_sigmoid(x):
    s = 1 / (1 + np.exp(-x))
    return s * (1 - s)          # 最大值仅 0.25，多层连乘极易梯度消失

def d_tanh(x):
    return 1 - np.tanh(x) ** 2  # 最大值 1，比 sigmoid 好但仍会饱和

def d_relu(x):
    return (x > 0).astype(float) # 正区间恒为 1，梯度不衰减；负区间恒为 0

xs = np.array([-2.0, 0.0, 2.0])
print("sigmoid 导数:", np.round(d_sigmoid(xs), 4))  # [0.105 0.25  0.105]
print("tanh 导数:   ", np.round(d_tanh(xs), 4))     # [0.0707 1.  0.0707]
print("relu 导数:   ", np.round(d_relu(xs), 4))     # [0. 0. 1.]

import numpy as np

def mse_loss(y_pred, y_true):
    return np.mean((y_pred - y_true) ** 2)

def mse_grad(y_pred, y_true):
    return 2 * (y_pred - y_true) / y_true.shape[0]

y_true = np.array([3.0, -0.5, 2.0, 7.0])
y_pred = np.array([2.5, 0.0, 2.0, 8.0])
print("MSE:", mse_loss(y_pred, y_true))   # 0.375

import numpy as np

def binary_cross_entropy(y_pred, y_true, eps=1e-9):
    y_pred = np.clip(y_pred, eps, 1 - eps)   # 防止 log(0)
    return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))

y_true = np.array([1, 0, 1, 1])
y_pred = np.array([0.9, 0.1, 0.8, 0.6])
print("BCE:", binary_cross_entropy(y_pred, y_true))   # 约 0.216

import numpy as np

def softmax(logits):
    # 减去每行最大值防止 exp 溢出，结果不变
    shifted = logits - np.max(logits, axis=1, keepdims=True)
    exp = np.exp(shifted)
    return exp / np.sum(exp, axis=1, keepdims=True)

def cross_entropy(logits, labels):
    """labels 是整数类别索引，如 [2, 0, 1]"""
    N = logits.shape[0]
    probs = softmax(logits)
    # 取每个样本正确类别的预测概率，取负对数
    log_likelihood = -np.log(probs[np.arange(N), labels] + 1e-9)
    return np.mean(log_likelihood)

def softmax_ce_grad(logits, labels):
    """softmax + 交叉熵合并后的梯度，形式极简：p - onehot"""
    N = logits.shape[0]
    grad = softmax(logits)
    grad[np.arange(N), labels] -= 1
    return grad / N

logits = np.array([[2.0, 1.0, 0.1], [0.5, 2.5, 0.3], [1.2, 0.7, 3.1]])
labels = np.array([0, 1, 2])
print("交叉熵损失:", round(cross_entropy(logits, labels), 4))
print("梯度形状:", softmax_ce_grad(logits, labels).shape)  # (3, 3)

import numpy as np

def focal_loss(y_pred, y_true, gamma=2.0, alpha=0.25, eps=1e-9):
    """
    gamma: 聚焦参数，越大对简单样本压制越强，常用 2.0
    alpha: 正负样本平衡因子，常用 0.25
    """
    y_pred = np.clip(y_pred, eps, 1 - eps)
    # pt 是模型对"真实类别"的预测概率
    pt = np.where(y_true == 1, y_pred, 1 - y_pred)
    alpha_t = np.where(y_true == 1, alpha, 1 - alpha)
    loss = -alpha_t * (1 - pt) ** gamma * np.log(pt)
    return np.mean(loss)

y_true = np.array([1, 0, 0, 0, 1])
y_pred = np.array([0.9, 0.05, 0.03, 0.4, 0.6])
print("Focal Loss:", round(focal_loss(y_pred, y_true), 4))

import numpy as np

def sgd_step(params, grads, lr=0.01):
    for k in params:
        params[k] -= lr * grads[k]
    return params

import numpy as np

class SGDMomentum:
    def __init__(self, lr=0.01, momentum=0.9):
        self.lr = lr
        self.momentum = momentum
        self.velocity = {}
    def step(self, params, grads):
        for k in params:
            if k not in self.velocity:
                self.velocity[k] = np.zeros_like(params[k])
            # v = 动量 * 旧速度 - 学习率 * 梯度
            self.velocity[k] = self.momentum * self.velocity[k] - self.lr * grads[k]
            params[k] += self.velocity[k]
        return params

import numpy as np

class Adam:
    def __init__(self, lr=0.001, beta1=0.9, beta2=0.999, eps=1e-8):
        self.lr = lr
        self.beta1 = beta1      # 一阶矩衰减率
        self.beta2 = beta2      # 二阶矩衰减率
        self.eps = eps
        self.m = {}             # 一阶矩（动量）
        self.v = {}             # 二阶矩（自适应缩放）
        self.t = 0              # 时间步

    def step(self, params, grads):
        self.t += 1
        for k in params:
            if k not in self.m:
                self.m[k] = np.zeros_like(params[k])
                self.v[k] = np.zeros_like(params[k])
            g = grads[k]
            # 更新一阶、二阶矩的指数移动平均
            self.m[k] = self.beta1 * self.m[k] + (1 - self.beta1) * g
            self.v[k] = self.beta2 * self.v[k] + (1 - self.beta2) * (g ** 2)
            # 偏差修正（训练初期 m、v 偏向 0，要修正）
            m_hat = self.m[k] / (1 - self.beta1 ** self.t)
            v_hat = self.v[k] / (1 - self.beta2 ** self.t)
            # 每个参数用各自的自适应学习率更新
            params[k] -= self.lr * m_hat / (np.sqrt(v_hat) + self.eps)
        return params

# AdamW 与 Adam 的唯一区别：更新参数时额外减去 lr * weight_decay * 参数本身
# params[k] -= self.lr * (m_hat / (np.sqrt(v_hat) + self.eps) + weight_decay * params[k])

import numpy as np

def xavier_init(fan_in, fan_out):
    """Xavier/Glorot 初始化，适配 tanh / sigmoid"""
    limit = np.sqrt(6.0 / (fan_in + fan_out))
    return np.random.uniform(-limit, limit, size=(fan_in, fan_out))

def he_init(fan_in, fan_out):
    """He/Kaiming 初始化，适配 ReLU 系激活"""
    return np.random.randn(fan_in, fan_out) * np.sqrt(2.0 / fan_in)

# 验证：好的初始化能让每层激活的方差保持稳定
np.random.seed(0)
x = np.random.randn(1000, 512)
W_bad = np.random.randn(512, 512) * 0.5   # 方差过大
W_he = he_init(512, 512)

a_bad = np.maximum(0, x @ W_bad)
a_he = np.maximum(0, x @ W_he)
print(f"随意初始化后激活标准差: {a_bad.std():.3f}")   # 偏离 1，可能爆炸
print(f"He 初始化后激活标准差:  {a_he.std():.3f}")   # 接近合理范围

import numpy as np

def batch_norm_forward(x, gamma, beta, eps=1e-5):
    """x: (N, D)，对每个特征维在 batch 上归一化"""
    mu = x.mean(axis=0)                    # 每个特征的均值 (D,)
    var = x.var(axis=0)                    # 每个特征的方差 (D,)
    x_hat = (x - mu) / np.sqrt(var + eps)  # 归一化
    return gamma * x_hat + beta            # 缩放平移

np.random.seed(0)
x = np.random.randn(32, 10) * 5 + 3       # 均值约 3，标准差约 5
gamma = np.ones(10); beta = np.zeros(10)
out = batch_norm_forward(x, gamma, beta)
print(f"归一化前均值 {x.mean():.2f} 标准差 {x.std():.2f}")
print(f"归一化后均值 {out.mean():.2f} 标准差 {out.std():.2f}")  # 均值≈0 标准差≈1

import numpy as np

def layer_norm_forward(x, gamma, beta, eps=1e-5):
    """x: (N, D)，对每个样本自己的 D 维特征归一化"""
    mu = x.mean(axis=1, keepdims=True)    # 每个样本的均值 (N, 1)
    var = x.var(axis=1, keepdims=True)    # 每个样本的方差 (N, 1)
    x_hat = (x - mu) / np.sqrt(var + eps)
    return gamma * x_hat + beta

x = np.random.randn(4, 8) * 2 + 1
gamma = np.ones(8); beta = np.zeros(8)
out = layer_norm_forward(x, gamma, beta)
print("每个样本归一化后的均值:", np.round(out.mean(axis=1), 4))  # 都接近 0

import numpy as np

# 模拟 20 层网络，每层梯度因子分别为 0.8（消失）和 1.5（爆炸）
layers = 20
vanishing = 0.8 ** layers    # 约 0.0115，再深就趋近 0
exploding = 1.5 ** layers    # 约 3325，继续放大会溢出
print(f"20 层后，因子 0.8 连乘: {vanishing:.6f}  (梯度消失)")
print(f"20 层后，因子 1.5 连乘: {exploding:.1f}   (梯度爆炸)")

import numpy as np

def clip_grad_norm(grads, max_norm=1.0):
    total_norm = np.sqrt(sum(np.sum(g ** 2) for g in grads))
    if total_norm > max_norm:
        scale = max_norm / (total_norm + 1e-6)
        grads = [g * scale for g in grads]
    return grads

import numpy as np

def conv2d_single(image, kernel, stride=1):
    """单通道二维卷积（valid 模式，无 padding）"""
    ih, iw = image.shape
    kh, kw = kernel.shape
    oh = (ih - kh) // stride + 1
    ow = (iw - kw) // stride + 1
    out = np.zeros((oh, ow))
    for i in range(oh):
        for j in range(ow):
            region = image[i*stride:i*stride+kh, j*stride:j*stride+kw]
            out[i, j] = np.sum(region * kernel)   # 局部加权求和
    return out

# 用一个垂直边缘检测核处理小图
image = np.array([
    [10, 10, 10, 0, 0, 0],
    [10, 10, 10, 0, 0, 0],
    [10, 10, 10, 0, 0, 0],
    [10, 10, 10, 0, 0, 0],
], dtype=float)
sobel_vertical = np.array([[1, 0, -1], [1, 0, -1], [1, 0, -1]], dtype=float)
print("卷积输出（边缘处响应最强）:\n", conv2d_single(image, sobel_vertical))

import numpy as np

def max_pool2d(x, size=2, stride=2):
    h, w = x.shape
    oh = (h - size) // stride + 1
    ow = (w - size) // stride + 1
    out = np.zeros((oh, ow))
    for i in range(oh):
        for j in range(ow):
            region = x[i*stride:i*stride+size, j*stride:j*stride+size]
            out[i, j] = np.max(region)
    return out

x = np.array([[1, 3, 2, 4], [5, 6, 1, 2], [7, 2, 3, 0], [1, 2, 8, 4]], dtype=float)
print("2x2 最大池化:\n", max_pool2d(x))   # 每个 2x2 窗口取最大值

import numpy as np

def rnn_step(x_t, h_prev, Wx, Wh, b):
    """单个时间步的 RNN 前向"""
    return np.tanh(x_t @ Wx + h_prev @ Wh + b)

np.random.seed(0)
seq_len, input_dim, hidden_dim = 5, 3, 4
Wx = np.random.randn(input_dim, hidden_dim) * 0.1
Wh = np.random.randn(hidden_dim, hidden_dim) * 0.1
b = np.zeros(hidden_dim)

h = np.zeros(hidden_dim)
for t in range(seq_len):
    x_t = np.random.randn(input_dim)
    h = rnn_step(x_t, h, Wx, Wh, b)
print("最终隐藏状态:", np.round(h, 3))

import numpy as np

def sigmoid(x): return 1 / (1 + np.exp(-x))

def lstm_step(x_t, h_prev, c_prev, params):
    """单步 LSTM，params 含各门的权重"""
    z = np.concatenate([x_t, h_prev])
    f = sigmoid(z @ params['Wf'] + params['bf'])   # 遗忘门：丢弃哪些旧记忆
    i = sigmoid(z @ params['Wi'] + params['bi'])   # 输入门：接纳哪些新信息
    o = sigmoid(z @ params['Wo'] + params['bo'])   # 输出门：输出多少
    c_tilde = np.tanh(z @ params['Wc'] + params['bc'])  # 候选记忆
    c = f * c_prev + i * c_tilde                    # 更新细胞状态
    h = o * np.tanh(c)                              # 新隐藏状态
    return h, c

import numpy as np

def step_decay(base_lr, epoch, drop=0.5, every=10):
    """阶梯衰减：每 every 轮把学习率乘以 drop"""
    return base_lr * (drop ** (epoch // every))

def cosine_decay(base_lr, epoch, total_epochs):
    """余弦退火：学习率沿余弦曲线平滑降到 0，Transformer 常用"""
    return 0.5 * base_lr * (1 + np.cos(np.pi * epoch / total_epochs))

def warmup_cosine(base_lr, step, warmup_steps, total_steps):
    """预热 + 余弦：先线性升温再余弦降温，大模型标配"""
    if step < warmup_steps:
        return base_lr * step / warmup_steps            # 线性预热
    progress = (step - warmup_steps) / (total_steps - warmup_steps)
    return 0.5 * base_lr * (1 + np.cos(np.pi * progress))

for e in [0, 5, 10, 20, 30]:
    print(f"epoch {e}: step={step_decay(0.1, e):.4f}, cosine={cosine_decay(0.1, e, 30):.4f}")

import numpy as np

np.random.seed(42)

# ---------- 1. 造一个三分类的螺旋数据集 ----------
def make_spiral(points_per_class=100, num_classes=3):
    X = np.zeros((points_per_class * num_classes, 2))
    y = np.zeros(points_per_class * num_classes, dtype=int)
    for c in range(num_classes):
        ix = range(points_per_class * c, points_per_class * (c + 1))
        r = np.linspace(0.0, 1, points_per_class)
        t = np.linspace(c * 4, (c + 1) * 4, points_per_class) + np.random.randn(points_per_class) * 0.2
        X[ix] = np.c_[r * np.sin(t), r * np.cos(t)]
        y[ix] = c
    return X, y

X, y = make_spiral()      # 300 个样本，2 维特征，3 类，线性不可分
print("数据形状:", X.shape, "标签类别:", np.unique(y))

# ---------- 2. 初始化一个 2 -> 64 -> 64 -> 3 的三层 MLP ----------
def he(fan_in, fan_out):
    return np.random.randn(fan_in, fan_out) * np.sqrt(2.0 / fan_in)

params = {
    'W1': he(2, 64),  'b1': np.zeros((1, 64)),
    'W2': he(64, 64), 'b2': np.zeros((1, 64)),
    'W3': he(64, 3),  'b3': np.zeros((1, 3)),
}

# ---------- 3. Adam 优化器状态 ----------
m = {k: np.zeros_like(v) for k, v in params.items()}
v = {k: np.zeros_like(v) for k, v in params.items()}
beta1, beta2, eps, lr = 0.9, 0.999, 1e-8, 0.001

def softmax(logits):
    shifted = logits - logits.max(axis=1, keepdims=True)
    e = np.exp(shifted)
    return e / e.sum(axis=1, keepdims=True)

# ---------- 4. 训练循环 ----------
N = X.shape[0]
batch_size = 64
num_classes = 3
t = 0

for epoch in range(3001):
    # 打乱并按 mini-batch 训练
    perm = np.random.permutation(N)
    for start in range(0, N, batch_size):
        idx = perm[start:start + batch_size]
        xb, yb = X[idx], y[idx]
        B = xb.shape[0]

        # ---- 前向 ----
        z1 = xb @ params['W1'] + params['b1']; a1 = np.maximum(0, z1)
        z2 = a1 @ params['W2'] + params['b2']; a2 = np.maximum(0, z2)
        z3 = a2 @ params['W3'] + params['b3']
        probs = softmax(z3)

        # ---- 反向 ----
        dz3 = probs.copy(); dz3[np.arange(B), yb] -= 1; dz3 /= B
        grads = {}
        grads['W3'] = a2.T @ dz3
        grads['b3'] = dz3.sum(axis=0, keepdims=True)
        da2 = dz3 @ params['W3'].T; dz2 = da2 * (z2 > 0)
        grads['W2'] = a1.T @ dz2
        grads['b2'] = dz2.sum(axis=0, keepdims=True)
        da1 = dz2 @ params['W2'].T; dz1 = da1 * (z1 > 0)
        grads['W1'] = xb.T @ dz1
        grads['b1'] = dz1.sum(axis=0, keepdims=True)

        # ---- Adam 更新 ----
        t += 1
        for k in params:
            m[k] = beta1 * m[k] + (1 - beta1) * grads[k]
            v[k] = beta2 * v[k] + (1 - beta2) * (grads[k] ** 2)
            m_hat = m[k] / (1 - beta1 ** t)
            v_hat = v[k] / (1 - beta2 ** t)
            params[k] -= lr * m_hat / (np.sqrt(v_hat) + eps)

    # ---- 每 500 轮评估整体准确率 ----
    if epoch % 500 == 0:
        z1 = np.maximum(0, X @ params['W1'] + params['b1'])
        z2 = np.maximum(0, z1 @ params['W2'] + params['b2'])
        logits = z2 @ params['W3'] + params['b3']
        pred = logits.argmax(axis=1)
        acc = (pred == y).mean()
        loss = -np.log(softmax(logits)[np.arange(N), y] + 1e-9).mean()
        print(f"epoch {epoch:4d}  loss={loss:.4f}  acc={acc:.3f}")

import torch
import torch.nn as nn

model = nn.Sequential(
    nn.Linear(2, 64), nn.ReLU(),
    nn.Linear(64, 64), nn.ReLU(),
    nn.Linear(64, 3),
)
criterion = nn.CrossEntropyLoss()          # 内部含 softmax
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

X_t = torch.tensor(X, dtype=torch.float32)
y_t = torch.tensor(y, dtype=torch.long)

for epoch in range(3001):
    logits = model(X_t)                     # 前向
    loss = criterion(logits, y_t)
    optimizer.zero_grad()
    loss.backward()                         # 自动反向传播
    optimizer.step()                        # 自动参数更新
    if epoch % 500 == 0:
        acc = (logits.argmax(1) == y_t).float().mean()
        print(f"epoch {epoch}, loss={loss.item():.4f}, acc={acc:.3f}")