【AI数学】30分钟拆解AI核心公式-复习笔记

import numpy as np

def softmax(x):
    """
    输入: x, shape (K,) 或 (batch, K)
    输出: 概率分布，shape同输入
    """
    # 减去最大值防止数值溢出（数学上等价）
    exp_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
    return exp_x / np.sum(exp_x, axis=-1, keepdims=True)

# 示例
logits = np.array([2.0, 1.0, 0.1])
probs = softmax(logits)
print(probs)  # [0.659, 0.242, 0.099]
print(probs.sum())  # 1.0

def cross_entropy_loss(probs, label):
    """
    probs: softmax输出的概率, shape (K,)
    label: 正确类别的索引, int
    """
    return -np.log(probs[label] + 1e-10)  # 加小数防止log(0)

# 示例
probs = np.array([0.7, 0.2, 0.1])
loss = cross_entropy_loss(probs, label=0)  # 正确类别是0
print(loss)  # 0.357（概率0.7对应的损失）

import numpy as np

def attention(Q, K, V):
    """
    Q: (seq_len_q, d_k)
    K: (seq_len_k, d_k)
    V: (seq_len_k, d_v)
    返回: (seq_len_q, d_v)
    """
    d_k = K.shape[-1]
    
    # 1. 计算注意力分数
    scores = Q @ K.T  # (seq_len_q, seq_len_k)
    
    # 2. 缩放
    scores = scores / np.sqrt(d_k)
    
    # 3. Softmax得到注意力权重
    weights = softmax(scores)  # 每行和为1
    
    # 4. 加权求和
    output = weights @ V  # (seq_len_q, d_v)
    
    return output, weights

# 示例：3个位置，维度4
Q = np.random.randn(3, 4)
K = np.random.randn(3, 4)
V = np.random.randn(3, 4)

output, weights = attention(Q, K, V)
print(f"输出shape: {output.shape}")  # (3, 4)
print(f"注意力权重:\n{weights}")  # 每行和为1

def multi_head_attention(Q, K, V, num_heads=8):
    """简化版多头注意力"""
    d_model = Q.shape[-1]
    d_k = d_model // num_heads
    
    outputs = []
    for i in range(num_heads):
        # 每个头处理一部分维度
        start, end = i * d_k, (i + 1) * d_k
        head_out, _ = attention(Q[:, start:end], K[:, start:end], V[:, start:end])
        outputs.append(head_out)
    
    return np.concatenate(outputs, axis=-1)

import torch

# 1. 定义参数
theta = torch.tensor([1.0, 2.0], requires_grad=True)

# 2. 定义损失函数（示例：L = theta[0]^2 + theta[1]^2）
def loss_fn(theta):
    return (theta ** 2).sum()

# 3. 梯度下降
learning_rate = 0.1
for step in range(10):
    # 前向：计算损失
    loss = loss_fn(theta)
    
    # 反向：计算梯度
    loss.backward()
    
    # 更新参数（不记录梯度）
    with torch.no_grad():
        theta -= learning_rate * theta.grad
    
    # 清零梯度（重要！）
    theta.grad.zero_()
    
    print(f"Step {step}: theta = {theta.data.numpy()}, loss = {loss.item():.4f}")

# PyTorch中使用Adam
optimizer = torch.optim.Adam([theta], lr=0.01)

for step in range(100):
    loss = loss_fn(theta)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()  # 自动更新参数

import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

class LSTMCell:
    def __init__(self, input_size, hidden_size):
        # 初始化权重（简化：合并为一个大矩阵）
        self.W = np.random.randn(4 * hidden_size, input_size + hidden_size) * 0.01
        self.b = np.zeros(4 * hidden_size)
        self.hidden_size = hidden_size
    
    def forward(self, x, h_prev, c_prev):
        """
        x: (input_size,)
        h_prev: (hidden_size,)
        c_prev: (hidden_size,)
        """
        hs = self.hidden_size
        
        # 拼接输入
        concat = np.concatenate([h_prev, x])
        
        # 计算所有门（一次矩阵乘法）
        gates = self.W @ concat + self.b
        
        # 分割成4个门
        f = sigmoid(gates[0:hs])           # 遗忘门
        i = sigmoid(gates[hs:2*hs])        # 输入门
        c_tilde = np.tanh(gates[2*hs:3*hs])  # 候选细胞
        o = sigmoid(gates[3*hs:4*hs])      # 输出门
        
        # 更新细胞状态
        c = f * c_prev + i * c_tilde
        
        # 计算隐藏状态
        h = o * np.tanh(c)
        
        return h, c

# 示例
lstm = LSTMCell(input_size=10, hidden_size=20)
x = np.random.randn(10)
h = np.zeros(20)
c = np.zeros(20)

h_new, c_new = lstm.forward(x, h, c)
print(f"隐藏状态shape: {h_new.shape}")  # (20,)

import torch
import torch.nn as nn

class ResidualBlock(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.conv1 = nn.Conv2d(channels, channels, kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(channels)
        self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(channels)
        self.relu = nn.ReLU()
    
    def forward(self, x):
        # 保存输入
        identity = x
        
        # 主路径
        out = self.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        
        # 残差连接
        out = out + identity  # 关键！
        out = self.relu(out)
        
        return out

# 示例
block = ResidualBlock(64)
x = torch.randn(1, 64, 32, 32)
y = block(x)
print(y.shape)  # torch.Size([1, 64, 32, 32])

# Transformer层的结构
x = x + self_attention(x)  # 残差连接1
x = x + feed_forward(x)    # 残差连接2

import numpy as np

def kl_divergence(p, q):
    """
    离散KL散度
    p, q: 概率分布，shape相同，和为1
    """
    # 避免log(0)
    q = np.clip(q, 1e-10, 1)
    p = np.clip(p, 1e-10, 1)
    return np.sum(p * np.log(p / q))

# 示例
p = np.array([0.4, 0.3, 0.3])  # 真实分布
q = np.array([0.5, 0.3, 0.2])  # 近似分布

kl = kl_divergence(p, q)
print(f"KL(P||Q) = {kl:.4f}")  # 约0.036

def vae_kl_loss(mu, log_var):
    """
    VAE的KL散度损失（解析形式）
    mu: 编码器输出的均值, shape (batch, latent_dim)
    log_var: 编码器输出的log方差, shape (batch, latent_dim)
    """
    # -0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
    kl = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp(), dim=1)
    return kl.mean()

# 示例
mu = torch.randn(32, 64)  # batch=32, latent_dim=64
log_var = torch.randn(32, 64)
kl_loss = vae_kl_loss(mu, log_var)
print(f"KL Loss: {kl_loss.item():.4f}")

import torch
import torch.nn as nn

# 2D卷积
conv = nn.Conv2d(
    in_channels=3,    # 输入通道（如RGB）
    out_channels=64,  # 输出通道（卷积核数量）
    kernel_size=3,    # 卷积核大小
    stride=1,         # 步长
    padding=1         # 填充
)

x = torch.randn(1, 3, 32, 32)  # (batch, channels, H, W)
y = conv(x)
print(y.shape)  # torch.Size([1, 64, 32, 32])

# 手动计算输出尺寸
H_in, k, p, s = 32, 3, 1, 1
H_out = (H_in + 2*p - k) // s + 1  # 32

import torch
import torch.nn as nn

# PyTorch内置
ln = nn.LayerNorm(normalized_shape=64)
x = torch.randn(32, 10, 64)  # (batch, seq_len, hidden)
y = ln(x)

# 手动实现
def layer_norm(x, gamma, beta, eps=1e-5):
    mean = x.mean(dim=-1, keepdim=True)
    var = x.var(dim=-1, keepdim=True, unbiased=False)
    x_norm = (x - mean) / torch.sqrt(var + eps)
    return gamma * x_norm + beta

import numpy as np
import torch

def positional_encoding(max_len, d_model):
    """
    生成位置编码矩阵
    max_len: 最大序列长度
    d_model: 模型维度
    """
    pe = np.zeros((max_len, d_model))
    position = np.arange(max_len)[:, np.newaxis]
    div_term = np.exp(np.arange(0, d_model, 2) * (-np.log(10000.0) / d_model))
    
    pe[:, 0::2] = np.sin(position * div_term)  # 偶数维度
    pe[:, 1::2] = np.cos(position * div_term)  # 奇数维度
    
    return torch.FloatTensor(pe)

# 示例
pe = positional_encoding(100, 512)
print(pe.shape)  # torch.Size([100, 512])

# 使用：x = x + pe[:seq_len, :]

import torch
import torch.nn.functional as F

def vae_loss(x, x_recon, mu, log_var):
    """
    VAE损失 = 重构损失 + KL散度
    x: 原始输入
    x_recon: 重构输出
    mu, log_var: 编码器输出的均值和log方差
    """
    # 重构损失（二值交叉熵或MSE）
    recon_loss = F.mse_loss(x_recon, x, reduction='sum')
    
    # KL散度（解析形式）
    kl_loss = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())
    
    return recon_loss + kl_loss