Loading
Loading
Backpropagation is the algorithm that makes neural networks learn — a clever application of the Chain Rule from calculus.
A neural network may have billions of parameters. After each wrong prediction, we need to know: how does each weight affect the error?
Answer: Gradient Descent + Backpropagation.
W_new = W_old - α × (∂Loss/∂W)
Mini-batch GD (32–256 samples) balances speed and stability.
Computes gradients from Output back to Input using Chain Rule:
Forward: x → layers → Loss
Backward: Loss → gradients → ∂W (update each weight)
Adam (most common): combines Momentum + adaptive learning rates per parameter.
AdamW: Adam with correct weight decay — best choice in 2024.
import numpy as np
import matplotlib
matplotlib.use("Agg")
print("=" * 55)
print("Backpropagation: مقارنة Optimizers")
print("=" * 55)
# ─────────────────────────────────────────
# مشكلة بسيطة: تصنيف ثنائي
# ─────────────────────────────────────────
np.random.seed(42)
n = 300
# بيانات: دائرتان
r1 = np.random.normal(0, 0.5, (n//2, 2))
r2 = np.random.normal(2, 0.5, (n//2, 2))
X = np.vstack([r1, r2])
y = np.hstack([np.zeros(n//2), np.ones(n//2)]).reshape(-1, 1)
# ─────────────────────────────────────────
# Optimizers
# ─────────────────────────────────────────
class SGDOptimizer:
def __init__(self, lr=0.01):
self.lr = lr
def update(self, params, grads):
for p, g in zip(params, grads):
p -= self.lr * g
class MomentumOptimizer:
def __init__(self, lr=0.01, beta=0.9):
self.lr, self.beta = lr, beta
self.v = None
def update(self, params, grads):
if self.v is None:
self.v = [np.zeros_like(p) for p in params]
for i, (p, g) in enumerate(zip(params, grads)):
self.v[i] = self.beta * self.v[i] + (1 - self.beta) * g
p -= self.lr * self.v[i]
class AdamOptimizer:
def __init__(self, lr=0.01, b1=0.9, b2=0.999, eps=1e-8):
self.lr, self.b1, self.b2, self.eps = lr, b1, b2, eps
self.m = self.v = None
self.t = 0
def update(self, params, grads):
if self.m is None:
self.m = [np.zeros_like(p) for p in params]
self.v = [np.zeros_like(p) for p in params]
self.t += 1
for i, (p, g) in enumerate(zip(params, grads)):
self.m[i] = self.b1 * self.m[i] + (1 - self.b1) * g
self.v[i] = self.b2 * self.v[i] + (1 - self.b2) * g**2
m_hat = self.m[i] / (1 - self.b1**self.t)
v_hat = self.v[i] / (1 - self.b2**self.t)
p -= self.lr * m_hat / (np.sqrt(v_hat) + self.eps)
# ─────────────────────────────────────────
# شبكة بسيطة
# ─────────────────────────────────────────
def train(optimizer_name: str, optimizer, epochs: int = 200) -> list[float]:
np.random.seed(42)
W1 = np.random.randn(2, 8) * 0.1
b1 = np.zeros((1, 8))
W2 = np.random.randn(8, 1) * 0.1
b2 = np.zeros((1, 1))
losses = []
for _ in range(epochs):
# Forward
z1 = X @ W1 + b1
a1 = np.maximum(0, z1)
z2 = a1 @ W2 + b2
a2 = 1 / (1 + np.exp(-z2))
loss = -np.mean(y * np.log(a2 + 1e-8) + (1 - y) * np.log(1 - a2 + 1e-8))
losses.append(loss)
# Backward
n_s = len(y)
dz2 = (a2 - y) / n_s
dW2 = a1.T @ dz2
db2 = dz2.sum(0, keepdims=True)
da1 = dz2 @ W2.T
dz1 = da1 * (z1 > 0)
dW1 = X.T @ dz1
db1 = dz1.sum(0, keepdims=True)
optimizer.update([W1, b1, W2, b2], [dW1, db1, dW2, db2])
# Accuracy
z1 = X @ W1 + b1
a1 = np.maximum(0, z1)
z2 = a1 @ W2 + b2
a2 = 1 / (1 + np.exp(-z2))
acc = ((a2 >= 0.5) == y).mean()
print(f" {optimizer_name:12s}: Final Loss={losses[-1]:.4f}, Accuracy={acc:.1%}")
return losses
print("\nمقارنة Optimizers (200 epochs, lr=0.01):")
results = {}
results["SGD"] = train("SGD", SGDOptimizer(lr=0.01))
results["Momentum"] = train("Momentum", MomentumOptimizer(lr=0.01))
results["Adam"] = train("Adam", AdamOptimizer(lr=0.01))
# تحليل السرعة
print("\nمقارنة: epochs للوصول لـ Loss < 0.3")
for name, losses in results.items():
epochs_needed = next((i for i, l in enumerate(losses) if l < 0.3), len(losses))
print(f" {name:12s}: {epochs_needed} epoch")
# ─────────────────────────────────────────
# Early Stopping
# ─────────────────────────────────────────
print("\nEarly Stopping — مثال:")
best_loss, patience, counter = float("inf"), 10, 0
fake_val_losses = [0.8, 0.7, 0.6, 0.5, 0.52, 0.51, 0.53, 0.54, 0.55, 0.56, 0.57]
for epoch, val_loss in enumerate(fake_val_losses, 1):
if val_loss < best_loss:
best_loss = val_loss
counter = 0
print(f" Epoch {epoch}: Loss={val_loss} ✅ best")
else:
counter += 1
print(f" Epoch {epoch}: Loss={val_loss} ({counter}/{patience})")
if counter >= patience:
print(f" ⏹️ Early Stop at Epoch {epoch}")
break