深度学习四大基石：反向传播、CNN、Embedding 与注意力

import numpy as np

# XOR 数据集：输入两位，输出它们的异或
X = np.array([[0, 0],
              [0, 1],
              [1, 0],
              [1, 1]], dtype=float)
y = np.array([0, 1, 1, 0], dtype=float)

# 尝试用一个单层线性模型（逻辑回归）去拟合
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

np.random.seed(0)
w = np.random.randn(2)   # 只有一条分界线的权重
b = 0.0
lr = 0.5

for step in range(5000):
    z = X @ w + b
    pred = sigmoid(z)
    error = pred - y
    # 梯度下降更新
    w -= lr * (X.T @ error) / len(y)
    b -= lr * error.mean()

pred = sigmoid(X @ w + b)
print("单层模型预测:", np.round(pred, 2))
# 输出大致是 [0.5 0.5 0.5 0.5]，四个点全被压到0.5附近，完全分不开——
# 这就是"一条直线切不开XOR"的直接体现

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(a):
    # a 是sigmoid的输出，导数可以直接用输出表示
    return a * (1 - a)

# 前向传播
def forward(X, W1, b1, W2, b2):
    z1 = X @ W1 + b1
    a1 = sigmoid(z1)          # 隐藏层输出
    z2 = a1 @ W2 + b2
    a2 = sigmoid(z2)          # 最终预测
    cache = (X, z1, a1, z2, a2)
    return a2, cache

# 反向传播：链式法则逐层往回传误差
def backward(y_true, W2, cache):
    X, z1, a1, z2, a2 = cache

    # 第一步：输出层的误差梯度（损失对a2的偏导 * a2对z2的偏导）
    d_loss_a2 = a2 - y_true                     # 均方误差简化后的梯度
    d_a2_z2 = sigmoid_derivative(a2)
    delta2 = d_loss_a2 * d_a2_z2                # 输出层的"误差信号"

    # 第二步：用delta2反推W2、b2的梯度
    dW2 = a1.T @ delta2
    db2 = delta2.sum(axis=0)

    # 第三步：把误差信号通过W2传回隐藏层（链式法则的关键一步）
    d_loss_a1 = delta2 @ W2.T
    d_a1_z1 = sigmoid_derivative(a1)
    delta1 = d_loss_a1 * d_a1_z1                # 隐藏层的"误差信号"

    # 第四步：反推W1、b1的梯度
    dW1 = X.T @ delta1
    db1 = delta1.sum(axis=0)

    return dW1, db1, dW2, db2

# 训练循环：不断前向算误差、反向算梯度、更新参数
def train_step(X, y_true, W1, b1, W2, b2, lr=0.1):
    a2, cache = forward(X, W1, b1, W2, b2)
    dW1, db1, dW2, db2 = backward(y_true, W2, cache)

    W1 -= lr * dW1
    b1 -= lr * db1
    W2 -= lr * dW2
    b2 -= lr * db2
    return W1, b1, W2, b2

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(a):
    return a * (1 - a)

X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
y = np.array([[0], [1], [1], [0]], dtype=float)

np.random.seed(42)
# 隐藏层4个神经元，输出层1个神经元
W1 = np.random.randn(2, 4)
b1 = np.zeros((1, 4))
W2 = np.random.randn(4, 1)
b2 = np.zeros((1, 1))

lr = 0.5
for epoch in range(10000):
    # 前向
    a1 = sigmoid(X @ W1 + b1)
    a2 = sigmoid(a1 @ W2 + b2)

    # 反向
    delta2 = (a2 - y) * sigmoid_derivative(a2)
    dW2 = a1.T @ delta2
    db2 = delta2.sum(axis=0, keepdims=True)

    delta1 = (delta2 @ W2.T) * sigmoid_derivative(a1)
    dW1 = X.T @ delta1
    db1 = delta1.sum(axis=0, keepdims=True)

    # 更新
    W2 -= lr * dW2
    b2 -= lr * db2
    W1 -= lr * dW1
    b1 -= lr * db1

pred = sigmoid(sigmoid(X @ W1 + b1) @ W2 + b2)
print("两层网络预测:", np.round(pred.ravel(), 2))
# 输出大致是 [0.02 0.98 0.98 0.02]，四个点被完美分开——
# 隐藏层让网络学会了非线性的分界，这就是"深度"的威力

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    s = sigmoid(x)
    return s * (1 - s)

def relu(x):
    return np.maximum(0, x)

def relu_grad(x):
    return (x > 0).astype(float)

np.random.seed(0)
depth = 20            # 20层深的网络
x = np.random.randn(1, 64)

# 用sigmoid做20层前向+反向，观察回传到第一层的梯度幅度
def measure_gradient(activation, activation_grad, scale=1.0):
    a = x.copy()
    caches = []
    for _ in range(depth):
        W = np.random.randn(a.shape[1], 64) * scale
        z = a @ W
        caches.append((a, W, z))
        a = activation(z)
    # 从输出往回传一个全1的梯度信号
    grad = np.ones_like(a)
    first_layer_grad_norm = None
    for a_prev, W, z in reversed(caches):
        grad = grad * activation_grad(z)   # 乘上本层激活导数
        grad = grad @ W.T                  # 传回上一层
        first_layer_grad_norm = np.linalg.norm(grad)
    return first_layer_grad_norm

np.random.seed(0)
g_sigmoid = measure_gradient(sigmoid, sigmoid_grad, scale=1.0)
np.random.seed(0)
g_relu = measure_gradient(relu, relu_grad, scale=0.5)

print(f"20层后 sigmoid 回传梯度范数: {g_sigmoid:.2e}")   # 极小，梯度几乎消失
print(f"20层后 relu    回传梯度范数: {g_relu:.2e}")      # 明显大得多
# 典型结果：sigmoid 是 1e-8 级别，relu 能保持在 1e-1 到 1e0 级别

import numpy as np

def batch_norm_forward(x, gamma, beta, eps=1e-5):
    """
    x: 形状 (N, D)，N是batch大小，D是特征维度
    对每一个特征维度，在batch方向上做归一化
    """
    mu = x.mean(axis=0)                     # 每个特征的均值 (D,)
    var = x.var(axis=0)                     # 每个特征的方差 (D,)
    x_hat = (x - mu) / np.sqrt(var + eps)   # 归一化到均值0方差1
    out = gamma * x_hat + beta              # 再缩放平移，恢复表达能力
    cache = (x, x_hat, mu, var, gamma, eps)
    return out, cache

# 验证：归一化后每列均值接近0、方差接近1
np.random.seed(0)
x = np.random.randn(128, 16) * 5 + 3       # 一批分布很偏的数据
gamma = np.ones(16)
beta = np.zeros(16)
out, _ = batch_norm_forward(x, gamma, beta)
print("归一化后均值:", np.round(out.mean(axis=0)[:4], 3))   # 接近 0
print("归一化后方差:", np.round(out.var(axis=0)[:4], 3))    # 接近 1

import numpy as np

def relu(x):
    return np.maximum(0, x)

def residual_block_forward(x, W1, W2):
    """
    一个最简化的残差块：out = relu(x + F(x))
    F(x) 是两层变换，x 通过捷径直接加到输出上
    """
    h = relu(x @ W1)      # 第一层变换
    f = h @ W2            # 第二层变换（残差 F(x)）
    out = relu(x + f)     # 关键：把输入x直接加回来
    return out

# 对比：观察残差连接如何让深层梯度不衰减
np.random.seed(0)
d = 64
x = np.random.randn(1, d)

def deep_forward_backward(use_residual, depth=30):
    a = x.copy()
    caches = []
    for _ in range(depth):
        W = np.random.randn(d, d) * 0.1
        z = a @ W
        h = relu(z)
        out = a + h if use_residual else h   # 有无捷径的区别
        caches.append((a, W, z, use_residual))
        a = out
    grad = np.ones_like(a)
    for a_prev, W, z, res in reversed(caches):
        g = grad * (z > 0)                    # relu导数
        g = g @ W.T
        grad = g + grad if res else g          # 捷径让梯度多一条直达路径
    return np.linalg.norm(grad)

np.random.seed(1)
g_plain = deep_forward_backward(False)
np.random.seed(1)
g_res = deep_forward_backward(True)
print(f"30层无残差 梯度范数: {g_plain:.2e}")
print(f"30层有残差 梯度范数: {g_res:.2e}")   # 明显更大、更健康

import numpy as np

def conv2d(image, kernel, stride=1):
    """
    对单通道灰度图做一次简单卷积（不做padding）
    image: 形状 (H, W) 的二维数组
    kernel: 形状 (kh, kw) 的卷积核
    """
    H, W = image.shape
    kh, kw = kernel.shape
    out_h = (H - kh) // stride + 1
    out_w = (W - kw) // stride + 1
    output = np.zeros((out_h, out_w))

    for i in range(0, out_h):
        for j in range(0, out_w):
            row_start = i * stride
            col_start = j * stride
            # 取出当前卷积核覆盖的图像区域
            region = image[row_start:row_start + kh, col_start:col_start + kw]
            # 逐元素相乘再求和，这就是一次"卷积"
            output[i, j] = np.sum(region * kernel)
    return output

# 一个经典的垂直边缘检测卷积核（Sobel算子的简化版）
vertical_edge_kernel = np.array([
    [-1, 0, 1],
    [-1, 0, 1],
    [-1, 0, 1]
])

# 造一张左黑右白的图，卷积后应该在中间边界处产生强响应
img = np.zeros((6, 6))
img[:, 3:] = 1
print("边缘检测结果:\n", conv2d(img, vertical_edge_kernel))
# 会看到中间列出现明显的非零值，正好对应黑白交界处

import numpy as np

def conv2d_pad(image, kernel, stride=1, padding=0):
    """支持 padding 和 stride 的卷积"""
    if padding > 0:
        image = np.pad(image, padding, mode='constant', constant_values=0)
    H, W = image.shape
    kh, kw = kernel.shape
    out_h = (H - kh) // stride + 1
    out_w = (W - kw) // stride + 1
    output = np.zeros((out_h, out_w))
    for i in range(out_h):
        for j in range(out_w):
            r, c = i * stride, j * stride
            output[i, j] = np.sum(image[r:r+kh, c:c+kw] * kernel)
    return output

img = np.random.randn(5, 5)
k = np.ones((3, 3)) / 9.0    # 3x3均值模糊核
same = conv2d_pad(img, k, stride=1, padding=1)  # padding=1保持5x5
print("输入尺寸:", img.shape, " same卷积输出尺寸:", same.shape)
# 输入 (5,5) -> 输出 (5,5)，尺寸被保住了

import numpy as np

def im2col(image, kh, kw, stride=1):
    """把每个卷积窗口展平成一行，堆成矩阵"""
    H, W = image.shape
    out_h = (H - kh) // stride + 1
    out_w = (W - kw) // stride + 1
    cols = np.zeros((out_h * out_w, kh * kw))
    idx = 0
    for i in range(out_h):
        for j in range(out_w):
            r, c = i * stride, j * stride
            patch = image[r:r+kh, c:c+kw].ravel()  # 窗口拉直成一行
            cols[idx] = patch
            idx += 1
    return cols, out_h, out_w

def conv2d_im2col(image, kernel, stride=1):
    kh, kw = kernel.shape
    cols, out_h, out_w = im2col(image, kh, kw, stride)
    # 卷积核也拉直，卷积变成一次矩阵乘法
    out = cols @ kernel.ravel()
    return out.reshape(out_h, out_w)

# 验证 im2col 版本和朴素版本结果一致
img = np.random.randn(8, 8)
k = np.random.randn(3, 3)
def conv2d_naive(image, kernel, stride=1):
    H, W = image.shape; kh, kw = kernel.shape
    oh, ow = (H-kh)//stride+1, (W-kw)//stride+1
    o = np.zeros((oh, ow))
    for i in range(oh):
        for j in range(ow):
            o[i, j] = np.sum(image[i*stride:i*stride+kh, j*stride:j*stride+kw] * kernel)
    return o

print("两种实现是否一致:",
      np.allclose(conv2d_naive(img, k), conv2d_im2col(img, k)))
# 输出 True：结果完全相同，但矩阵乘法版本可以被BLAS/GPU大幅加速

import numpy as np

def max_pool2d(feature_map, pool_size=2, stride=2):
    H, W = feature_map.shape
    out_h = (H - pool_size) // stride + 1
    out_w = (W - pool_size) // stride + 1
    output = np.zeros((out_h, out_w))
    for i in range(out_h):
        for j in range(out_w):
            r, c = i * stride, j * stride
            region = feature_map[r:r+pool_size, c:c+pool_size]
            output[i, j] = np.max(region)     # 保留最大值
    return output

def avg_pool2d(feature_map, pool_size=2, stride=2):
    H, W = feature_map.shape
    out_h = (H - pool_size) // stride + 1
    out_w = (W - pool_size) // stride + 1
    output = np.zeros((out_h, out_w))
    for i in range(out_h):
        for j in range(out_w):
            r, c = i * stride, j * stride
            region = feature_map[r:r+pool_size, c:c+pool_size]
            output[i, j] = np.mean(region)    # 保留平均值
    return output

def global_avg_pool(feature_map):
    return feature_map.mean()                 # 整张图压成一个数

fm = np.arange(16, dtype=float).reshape(4, 4)
print("原特征图:\n", fm)
print("最大池化:\n", max_pool2d(fm))
print("平均池化:\n", avg_pool2d(fm))
print("全局平均池化:", global_avg_pool(fm))

def receptive_field(kernels, strides):
    """
    根据每层的卷积核大小和步长，逐层递推感受野
    kernels: 每层卷积核大小列表，比如 [3,3,3]
    strides: 每层步长列表，比如 [1,1,1]
    """
    rf = 1            # 初始感受野
    jump = 1          # 相邻输出点在原图上的间隔
    for k, s in zip(kernels, strides):
        rf = rf + (k - 1) * jump
        jump = jump * s
    return rf

print("三个3x3卷积堆叠的感受野:", receptive_field([3, 3, 3], [1, 1, 1]))
# 输出 7，正好等价于一个7x7卷积的视野
print("加入一次步长2的池化后:", receptive_field([3, 2, 3], [1, 2, 1]))

import numpy as np

vocab = ["猫", "狗", "汽车", "飞机"]
vocab_size = len(vocab)
word2id = {w: i for i, w in enumerate(vocab)}

# One-Hot：每个词一个超稀疏向量，维度=词表大小
def one_hot(word):
    v = np.zeros(vocab_size)
    v[word2id[word]] = 1
    return v

print("猫的One-Hot:", one_hot("猫"))     # [1 0 0 0]
print("狗的One-Hot:", one_hot("狗"))     # [0 1 0 0]
# 任意两个One-Hot点积都是0（正交），看不出"猫和狗更相似"

# Embedding：一张查找表，每个词对应一个低维稠密向量
embedding_dim = 3
np.random.seed(0)
embedding_table = np.random.randn(vocab_size, embedding_dim)

def embed(word):
    return embedding_table[word2id[word]]   # 直接查表

print("猫的Embedding:", np.round(embed("猫"), 2))
# 训练之后，语义相近的词向量会靠得更近，这是One-Hot永远做不到的

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def skipgram_training_step(center_word_id, context_word_id,
                            center_embeddings, context_embeddings,
                            vocab_size, lr=0.01):
    """
    center_embeddings: 形状 (vocab_size, dim)，每个词的"中心词"向量
    context_embeddings: 形状 (vocab_size, dim)，每个词的"上下文词"向量
    目标：让 中心词向量 与 真实上下文词向量 的点积尽量大，
         与随机采样的"负样本"词向量的点积尽量小。
    """
    v_center = center_embeddings[center_word_id]
    v_context = context_embeddings[context_word_id]

    # 正样本：真实共现的词对，希望相似度（点积）越大越好
    pos_score = sigmoid(np.dot(v_center, v_context))
    grad_pos = (pos_score - 1) * v_context

    # 负样本：随机采样几个"不该出现在这个上下文里"的词
    negative_ids = np.random.choice(vocab_size, size=5)
    grad_neg_total = np.zeros_like(v_center)
    for neg_id in negative_ids:
        v_neg = context_embeddings[neg_id]
        neg_score = sigmoid(np.dot(v_center, v_neg))
        grad_neg_total += neg_score * v_neg
        context_embeddings[neg_id] -= lr * neg_score * v_center

    # 用梯度更新中心词向量和上下文词向量（简化的梯度下降）
    grad_center = grad_pos + grad_neg_total
    center_embeddings[center_word_id] -= lr * grad_center
    context_embeddings[context_word_id] -= lr * (pos_score - 1) * v_center

    return center_embeddings, context_embeddings

import numpy as np

# 玩具语料：两类主题的句子（饮品 / 交通）
corpus = [
    "我 喝 咖啡", "我 喝 茶", "我 喝 果汁",
    "他 开 汽车", "他 开 飞机", "他 开 火车",
]
tokens = [w for s in corpus for w in s.split()]
vocab = sorted(set(tokens))
word2id = {w: i for i, w in enumerate(vocab)}
V, dim = len(vocab), 8

np.random.seed(1)
center_emb = np.random.randn(V, dim) * 0.1
context_emb = np.random.randn(V, dim) * 0.1

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

# 构造(中心词, 上下文词)训练对，窗口大小1
pairs = []
for s in corpus:
    ws = s.split()
    for i, w in enumerate(ws):
        for j in (i - 1, i + 1):
            if 0 <= j < len(ws):
                pairs.append((word2id[w], word2id[ws[j]]))

lr = 0.05
for epoch in range(3000):
    for c, ctx in pairs:
        vc, vo = center_emb[c], context_emb[ctx]
        # 正样本梯度
        g = (sigmoid(vc @ vo) - 1)
        center_emb[c] -= lr * g * vo
        context_emb[ctx] -= lr * g * vc
        # 负采样
        for neg in np.random.choice(V, 3):
            vn = context_emb[neg]
            gn = sigmoid(vc @ vn)
            center_emb[c] -= lr * gn * vn
            context_emb[neg] -= lr * gn * vc

def cos(a, b):
    return a @ b / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)

print("咖啡-茶 相似度:", round(cos(center_emb[word2id["咖啡"]], center_emb[word2id["茶"]]), 3))
print("咖啡-汽车 相似度:", round(cos(center_emb[word2id["咖啡"]], center_emb[word2id["汽车"]]), 3))
# 训练后通常会看到 咖啡-茶 的相似度明显高于 咖啡-汽车

import numpy as np

def cosine_similarity(a, b):
    """余弦相似度：只看方向不看长度，范围[-1,1]，越大越相似"""
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)

def nearest_neighbors(query_vec, embedding_table, id2word, topk=3):
    """在词向量表里找出和query最相似的topk个词"""
    sims = []
    for i, vec in enumerate(embedding_table):
        sims.append((id2word[i], cosine_similarity(query_vec, vec)))
    sims.sort(key=lambda x: x[1], reverse=True)
    return sims[:topk]

# 构造一个玩具向量表（真实场景里是训练出来的）
id2word = {0: "国王", 1: "女王", 2: "男人", 3: "女人", 4: "苹果"}
np.random.seed(0)
table = np.random.randn(5, 4)
# 人为让"国王"和"女王"方向接近
table[1] = table[0] + np.random.randn(4) * 0.1

q = table[0]  # 用"国王"作为查询
print("和'国王'最相似的词:", nearest_neighbors(q, table, id2word))
# "女王"会排在前面，因为我们让它俩方向接近了

import numpy as np

def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9)

# 手工构造一组"带性别方向"的向量来演示原理
# 维度含义假设为: [王室程度, 性别(男正女负), 其他]
vec = {
    "国王": np.array([1.0,  1.0, 0.2]),
    "女王": np.array([1.0, -1.0, 0.2]),
    "男人": np.array([0.0,  1.0, 0.1]),
    "女人": np.array([0.0, -1.0, 0.1]),
}

# 语义代数：国王 - 男人 + 女人
result = vec["国王"] - vec["男人"] + vec["女人"]
print("类比结果向量:", result)

# 看它和谁最接近
for w, v in vec.items():
    print(f"  与 {w} 的相似度: {cosine_similarity(result, v):.3f}")
# "女王"的相似度最高，验证了 国王-男人+女人 ≈ 女王

import numpy as np

def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)  # 数值稳定性处理
    exp_x = np.exp(x)
    return exp_x / np.sum(exp_x, axis=-1, keepdims=True)

def simple_self_attention(X, Wq, Wk, Wv):
    """
    X: 形状 (seq_len, d_model)，句子中每个词的初始向量表示
    Wq, Wk, Wv: 分别把X投影成 Query、Key、Value 的权重矩阵
    """
    Q = X @ Wq   # (seq_len, d_k) 每个词的"问题"
    K = X @ Wk   # (seq_len, d_k) 每个词的"标签"
    V = X @ Wv   # (seq_len, d_v) 每个词的"内容"

    d_k = K.shape[-1]
    # 第一步：算出每个词的Query与所有词的Key的匹配度（点积衡量相似度）
    scores = Q @ K.T / np.sqrt(d_k)     # 除以sqrt(d_k)防止数值过大导致softmax梯度消失

    # 第二步：把匹配度分数变成"加起来等于1"的注意力权重
    attention_weights = softmax(scores)  # 形状 (seq_len, seq_len)

    # 第三步：用注意力权重对所有词的Value做加权求和
    output = attention_weights @ V       # 每个词都得到融合了上下文的新表示

    return output, attention_weights

import numpy as np

np.random.seed(0)
d_k = 128
q = np.random.randn(d_k)
k = np.random.randn(d_k)

raw = q @ k                      # 未缩放的点积
scaled = (q @ k) / np.sqrt(d_k)  # 缩放后的点积
print(f"未缩放点积: {raw:.2f}")      # 数值较大，方差约等于d_k
print(f"缩放后点积: {scaled:.2f}")   # 方差被拉回1附近

# 用一组分数看softmax的"尖锐"程度差异
scores = np.random.randn(5) * 10   # 模拟未缩放的大分数
print("大分数softmax:", np.round(np.exp(scores)/np.exp(scores).sum(), 3))
# 会看到几乎所有权重集中在一个位置，其余接近0——梯度消失

import numpy as np

def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)
    e = np.exp(x)
    return e / e.sum(axis=-1, keepdims=True)

def multi_head_attention(X, Wq, Wk, Wv, Wo, num_heads):
    """
    X: (seq_len, d_model)
    Wq, Wk, Wv: (d_model, d_model) 投影矩阵
    Wo: (d_model, d_model) 输出融合矩阵
    num_heads: 头的数量，d_model 必须能被它整除
    """
    seq_len, d_model = X.shape
    d_head = d_model // num_heads   # 每个头的维度

    # 先整体投影，再切成多个头
    Q = X @ Wq   # (seq_len, d_model)
    K = X @ Wk
    V = X @ Wv

    outputs = []
    all_weights = []
    for h in range(num_heads):
        # 取出第h个头对应的那一段维度
        s = h * d_head
        e = (h + 1) * d_head
        Qh, Kh, Vh = Q[:, s:e], K[:, s:e], V[:, s:e]

        scores = Qh @ Kh.T / np.sqrt(d_head)   # 每个头独立算注意力
        w = softmax(scores)
        outputs.append(w @ Vh)                  # (seq_len, d_head)
        all_weights.append(w)

    # 把所有头的输出在特征维度上拼接起来
    concat = np.concatenate(outputs, axis=-1)   # (seq_len, d_model)
    out = concat @ Wo                           # 再做一次线性融合
    return out, all_weights

# 跑一个小例子
np.random.seed(0)
seq_len, d_model, heads = 4, 8, 2
X = np.random.randn(seq_len, d_model)
Wq = np.random.randn(d_model, d_model)
Wk = np.random.randn(d_model, d_model)
Wv = np.random.randn(d_model, d_model)
Wo = np.random.randn(d_model, d_model)
out, weights = multi_head_attention(X, Wq, Wk, Wv, Wo, heads)
print("多头注意力输出形状:", out.shape)          # (4, 8)
print("头数:", len(weights), " 每个头的注意力矩阵形状:", weights[0].shape)

import numpy as np

def sinusoidal_position_encoding(seq_len, d_model):
    """
    生成正弦位置编码
    返回形状 (seq_len, d_model)，可直接加到词向量上
    """
    pe = np.zeros((seq_len, d_model))
    position = np.arange(seq_len)[:, np.newaxis]           # (seq_len, 1)
    # 不同维度用不同频率：偶数维用sin，奇数维用cos
    div_term = np.exp(np.arange(0, d_model, 2) * (-np.log(10000.0) / d_model))
    pe[:, 0::2] = np.sin(position * div_term)   # 偶数维
    pe[:, 1::2] = np.cos(position * div_term)   # 奇数维
    return pe

pe = sinusoidal_position_encoding(seq_len=6, d_model=8)
print("位置编码形状:", pe.shape)
print("第0个位置的编码:", np.round(pe[0], 2))
print("第1个位置的编码:", np.round(pe[1], 2))
# 每个位置得到一个独一无二的向量，且相邻位置编码相似、远处位置差异大

# 用法：词向量 + 位置编码 一起送进注意力
np.random.seed(0)
word_emb = np.random.randn(6, 8)
X = word_emb + pe   # 现在X既带语义又带位置信息

import numpy as np

def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)
    e = np.exp(x)
    return e / e.sum(axis=-1, keepdims=True)

def causal_self_attention(X, Wq, Wk, Wv):
    Q, K, V = X @ Wq, X @ Wk, X @ Wv
    d_k = K.shape[-1]
    scores = Q @ K.T / np.sqrt(d_k)          # (seq_len, seq_len)

    seq_len = X.shape[0]
    # 构造上三角掩码：i行j列，j>i的位置（未来）设为负无穷
    mask = np.triu(np.ones((seq_len, seq_len)), k=1).astype(bool)
    scores[mask] = -np.inf                    # 屏蔽未来位置

    weights = softmax(scores)
    return weights @ V, weights

np.random.seed(0)
seq_len, d = 4, 6
X = np.random.randn(seq_len, d)
Wq, Wk, Wv = np.random.randn(d, d), np.random.randn(d, d), np.random.randn(d, d)
out, w = causal_self_attention(X, Wq, Wk, Wv)
print("因果注意力权重(下三角，上三角为0):")
print(np.round(w, 2))
# 你会看到权重矩阵是下三角的：每个位置只关注自己和之前的位置

import numpy as np

def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)
    e = np.exp(x)
    return e / e.sum(axis=-1, keepdims=True)

def layer_norm(x, eps=1e-6):
    mu = x.mean(axis=-1, keepdims=True)
    var = x.var(axis=-1, keepdims=True)
    return (x - mu) / np.sqrt(var + eps)

def self_attention(X, Wq, Wk, Wv):
    Q, K, V = X @ Wq, X @ Wk, X @ Wv
    d_k = K.shape[-1]
    w = softmax(Q @ K.T / np.sqrt(d_k))
    return w @ V

def feed_forward(x, W1, b1, W2, b2):
    # 升维 -> ReLU -> 降维
    return np.maximum(0, x @ W1 + b1) @ W2 + b2

def transformer_encoder_block(X, params):
    # 子层1：自注意力 + 残差 + 层归一化
    attn = self_attention(X, params['Wq'], params['Wk'], params['Wv'])
    X = layer_norm(X + attn)                       # 残差连接后归一化

    # 子层2：前馈网络 + 残差 + 层归一化
    ff = feed_forward(X, params['W1'], params['b1'], params['W2'], params['b2'])
    X = layer_norm(X + ff)
    return X

# 组装参数跑一遍
np.random.seed(0)
seq_len, d_model, d_ff = 5, 16, 64
X = np.random.randn(seq_len, d_model)
params = {
    'Wq': np.random.randn(d_model, d_model) * 0.1,
    'Wk': np.random.randn(d_model, d_model) * 0.1,
    'Wv': np.random.randn(d_model, d_model) * 0.1,
    'W1': np.random.randn(d_model, d_ff) * 0.1,
    'b1': np.zeros(d_ff),
    'W2': np.random.randn(d_ff, d_model) * 0.1,
    'b2': np.zeros(d_model),
}
out = transformer_encoder_block(X, params)
print("Transformer编码块输出形状:", out.shape)  # (5, 16)，形状不变，可无限堆叠

import torch
import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, in_dim, hidden, out_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim, hidden),
            nn.ReLU(),              # 非线性，否则多层等价于一层
            nn.Linear(hidden, out_dim),
        )

    def forward(self, x):
        return self.net(x)

model = MLP(784, 256, 10)
loss_fn = nn.CrossEntropyLoss()
opt = torch.optim.Adam(model.parameters(), lr=1e-3)

# 一步训练：前向 -> 算损失 -> 反向 -> 更新
logits = model(x_batch)
loss = loss_fn(logits, y_batch)
opt.zero_grad()   # 清空上一步梯度，否则会累加
loss.backward()   # 反向传播：链式法则自动算出每个参数的梯度
opt.step()        # 沿负梯度方向更新参数

conv_block = nn.Sequential(
    nn.Conv2d(3, 32, kernel_size=3, padding=1),  # 局部连接+权重共享
    nn.BatchNorm2d(32),                          # 批归一化，稳住分布
    nn.ReLU(),
    nn.MaxPool2d(2),                             # 下采样，扩大感受野
)
# 输入 (N,3,32,32) -> 输出 (N,32,16,16)

import torch
import torch.nn.functional as F

def attention(Q, K, V, mask=None):
    d_k = Q.size(-1)
    # 打分：Q 和每个 K 做点积，再除以 sqrt(d_k) 防止梯度过小
    scores = Q @ K.transpose(-2, -1) / (d_k ** 0.5)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, float('-inf'))
    weights = F.softmax(scores, dim=-1)  # 归一成注意力权重
    return weights @ V                   # 按权重聚合 V