第 2 章：神经网络如何学习 —— 梯度下降算法¶

import numpy as np

# 假设网络预测某张图片是各数字的概率
prediction = np.array([0.05, 0.02, 0.03, 0.10, 0.05, 0.60, 0.05, 0.03, 0.02, 0.05])
# 正确答案是数字 5（索引从0开始）
true_label = 5

# 交叉熵损失：最常用的分类损失函数
loss = -np.log(prediction[true_label])
print(f"预测数字5的概率: {prediction[true_label]:.2f}")
print(f"损失值: {loss:.4f}")

import numpy as np
import matplotlib.pyplot as plt

def f(x):
    return x ** 2

def gradient(x):
    """$f(x) = x^2$ 的导数是 $f'(x) = 2x$"""
    return 2 * x

# 梯度下降
x = 3.0                # 起始位置（随便选）
learning_rate = 0.1    # 学习率（步长）
history = [x]          # 记录路径

for step in range(20):
    grad = gradient(x)           # 计算当前位置的梯度
    x = x - learning_rate * grad # 沿负梯度方向移动
    history.append(x)
    if step % 5 == 0:
        print(f"步骤 {step:2d}: x = {x:.4f}, f(x) = {f(x):.6f}, 梯度 = {grad:.4f}")

# 可视化
x_vals = np.linspace(-4, 4, 100)
plt.figure(figsize=(10, 4))
plt.plot(x_vals, f(x_vals), 'b-', label=r'$f(x) = x^2$')
plt.scatter(history, [f(h) for h in history], c='red', s=50, zorder=5)
plt.plot(history, [f(h) for h in history], 'r--', alpha=0.5)
plt.xlabel('x')
plt.ylabel('f(x)')
plt.title('梯度下降：从 x=3 走向 x=0')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

def gradient_descent_demo(start_x, learning_rate, steps=10):
    x = start_x
    history = [x]
    for _ in range(steps):
        grad = 2 * x
        x = x - learning_rate * grad
        history.append(x)
    return history

# 三种学习率对比
rates = [0.01, 0.1, 1.01]
histories = {}
for lr in rates:
    histories[lr] = gradient_descent_demo(3.0, lr, steps=15)

# 打印对比
for lr, hist in histories.items():
    final_x = hist[-1]
    print(f"学习率 {lr:.2f}: 起始 x=3.0 → 最终 x={final_x:.4f}, f(x)={final_x**2:.6f}")

def f_2d(x, y):
    """一个碗状的二维函数"""
    return x**2 + y**2

def gradient_2d(x, y):
    """偏导数: $\frac{\partial f}{\partial x} = 2x$, $\frac{\partial f}{\partial y} = 2y$"""
    return np.array([2*x, 2*y])

# 二维梯度下降
point = np.array([3.0, 4.0])  # 起始点
lr = 0.1
path_2d = [point.copy()]

for _ in range(30):
    grad = gradient_2d(point[0], point[1])
    point = point - lr * grad
    path_2d.append(point.copy())

path_2d = np.array(path_2d)
print(f"起始点: (3.0, 4.0), f = {f_2d(3, 4)}")
print(f"最终点: ({path_2d[-1][0]:.4f}, {path_2d[-1][1]:.4f}), f = {f_2d(*path_2d[-1]):.6f}")

import numpy as np

class TrainableNN:
    """可训练的简单神经网络"""

    def __init__(self, input_size=784, hidden_size=128, output_size=10):
        self.W1 = np.random.randn(input_size, hidden_size) * 0.01
        self.b1 = np.zeros((1, hidden_size))
        self.W2 = np.random.randn(hidden_size, output_size) * 0.01
        self.b2 = np.zeros((1, output_size))

    def sigmoid(self, z):
        return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

    def softmax(self, z):
        exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
        return exp_z / np.sum(exp_z, axis=1, keepdims=True)

    def forward(self, X):
        self.z1 = np.dot(X, self.W1) + self.b1
        self.a1 = self.sigmoid(self.z1)
        self.z2 = np.dot(self.a1, self.W2) + self.b2
        self.a2 = self.softmax(self.z2)
        return self.a2

    def compute_loss(self, predictions, y_true):
        """交叉熵损失"""
        n_samples = predictions.shape[0]
        correct_probs = predictions[np.arange(n_samples), y_true]
        return -np.mean(np.log(correct_probs + 1e-8))

    def compute_gradients(self, X, y_true):
        """
        计算损失函数对每个参数的梯度
        这是反向传播的核心——第3章会详细讲解
        """
        n_samples = X.shape[0]

        # 输出层梯度
        dz2 = self.a2.copy()
        dz2[np.arange(n_samples), y_true] -= 1
        dz2 /= n_samples

        dW2 = np.dot(self.a1.T, dz2)
        db2 = np.sum(dz2, axis=0, keepdims=True)

        # 隐藏层梯度
        da1 = np.dot(dz2, self.W2.T)
        dz1 = da1 * self.a1 * (1 - self.a1)

        dW1 = np.dot(X.T, dz1)
        db1 = np.sum(dz1, axis=0, keepdims=True)

        return {"W1": dW1, "b1": db1, "W2": dW2, "b2": db2}

    def train_step(self, X, y_true, learning_rate=0.1):
        """执行一步梯度下降"""
        # 前向传播
        predictions = self.forward(X)
        loss = self.compute_loss(predictions, y_true)

        # 计算梯度
        grads = self.compute_gradients(X, y_true)

        # 更新所有参数（梯度下降的核心步骤）
        self.W1 -= learning_rate * grads["W1"]
        self.b1 -= learning_rate * grads["b1"]
        self.W2 -= learning_rate * grads["W2"]
        self.b2 -= learning_rate * grads["b2"]

        return loss

# 模拟训练
np.random.seed(42)
nn = TrainableNN()

# 生成 100 个假样本用于演示
X_fake = np.random.randn(100, 784)
y_fake = np.random.randint(0, 10, 100)

print("开始训练...")
print(f"{'轮次':<6} {'损失':<12}")
print("-" * 20)

for epoch in range(10):
    loss = nn.train_step(X_fake, y_fake, learning_rate=0.1)
    print(f"{epoch+1:<6} {loss:<12.6f}")

print(f"\n最终损失: {loss:.6f}")

import torch
import torch.nn as nn
import torch.optim as optim

class TrainableNN(nn.Module):
    """与 2.5 节 NumPy 版本相同的可训练网络"""
    def __init__(self):
        super(TrainableNN, self).__init__()
        self.fc1 = nn.Linear(784, 128)
        self.sigmoid = nn.Sigmoid()
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.fc1(x)
        x = self.sigmoid(x)
        x = self.fc2(x)
        return x

torch.manual_seed(42)
model = TrainableNN()

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)

X_fake = torch.randn(100, 784)
y_fake = torch.randint(0, 10, (100,))

print("开始训练（PyTorch 自动求导）...")
print(f"{'轮次':<6} {'损失':<12}")
print("-" * 20)

for epoch in range(10):
    # 1. 前向传播
    output = model(X_fake)
    loss = criterion(output, y_fake)

    # 2. 反向传播（自动计算所有梯度！）
    optimizer.zero_grad()
    loss.backward()

    # 3. 参数更新
    optimizer.step()

    print(f"{epoch+1:<6} {loss.item():<12.6f}")

print(f"\n最终损失: {loss.item():.6f}")