🎥Video 45 min3 / 4

Derivatives and gradients

Derivatives and Gradients

Derivatives and gradients are the mathematical tools that neural networks use to learn.

Derivative:

Measures the rate of change of a function. f'(x) = how much f changes when x changes slightly.

Gradient:

Derivative of a multi-variable function. Points in the direction of greatest increase in the function.

Gradient Descent:

The fundamental learning algorithm: w = w - lr × ∇f(w) where lr is the Learning Rate and the gradient points toward increase (we subtract it to decrease)

Backpropagation = Chain Rule:

To compute the gradient of a composite function y = f(g(x)) apply the chain rule: dy/dx = dy/dg × dg/dx

Learning Rate Effect:

Too small → very slow learning
Too large → doesn't converge, oscillates
Just right → fast and smooth convergence

python

import math
from typing import Callable, List, Tuple

# ─── مشتق عددي ─────────────────────────────────────────────
def deriv(f: Callable[[float], float], x: float, h: float = 1e-5) -> float:
    """المشتق العددي بالفرق المركزي"""
    return (f(x + h) - f(x - h)) / (2 * h)

def gradient(f: Callable[[List[float]], float],
             params: List[float], h: float = 1e-5) -> List[float]:
    """تدرج دالة متعددة المتغيرات"""
    grads = []
    for i in range(len(params)):
        p1, p2 = params[:], params[:]
        p1[i] += h
        p2[i] -= h
        grads.append((f(p1) - f(p2)) / (2 * h))
    return grads

# ─── دوال التفعيل ──────────────────────────────────────────
def sigmoid(x: float)  -> float: return 1 / (1 + math.exp(-x))
def relu(x: float)     -> float: return max(0.0, x)
def tanh_fn(x: float)  -> float: return math.tanh(x)

# ─── Gradient Descent ──────────────────────────────────────
def gd_demo(f, df, w0: float, lr: float, epochs: int, label: str):
    """محاكاة Gradient Descent"""
    w = w0
    losses = [f(w)]
    for _ in range(epochs):
        w = w - lr * df(w)
        losses.append(f(w))
    init_str  = f"{losses[0]:.4f}"
    final_str = f"{losses[-1]:.4f}"
    w_str     = f"{w:.4f}"
    conv = "✅" if abs(losses[-1] - losses[0]) > abs(losses[0]) * 0.5 else "⚠️ "
    print(f"  {label}")
    print(f"    Loss: {init_str} → {final_str}  |  w* = {w_str}  {conv}")
    return w

# ─── عرض مشتقات دوال التفعيل ──────────────────────────────
print("📐 مشتقات دوال التفعيل:")
print("=" * 52)

funcs = [("sigmoid", sigmoid), ("relu", relu), ("tanh", tanh_fn)]
points = [-2.0, -1.0, 0.0, 1.0, 2.0]

for name, fn in funcs:
    print(f"\n  {name}(x) → f(x)  |  f'(x):")
    for x in points:
        val  = fn(x)
        grad = deriv(fn, x)
        bar  = "▓" * max(0, int(abs(grad) * 8))
        print(f"    x={x:+.1f} → {val:.3f}  |  {grad:.3f}  {bar}")

# ─── Gradient Descent ──────────────────────────────────────
print(f"\n\n📉 Gradient Descent — تقليل f(w) = w² + 2w - 3:")
print("  (الحد الأدنى الحقيقي: w = -1)")
print("-" * 52)

def f(w):  return w**2 + 2*w - 3
def df(w): return 2*w + 2

configs = [
    (5.0, 0.01, 40, "LR=0.01 (بطيء)   "),
    (5.0, 0.1,  20, "LR=0.10 (مناسب)  "),
    (5.0, 0.95, 15, "LR=0.95 (كبير جداً)"),
]
for w0, lr, ep, label in configs:
    gd_demo(f, df, w0, lr, ep, label)

# ─── Chain Rule ────────────────────────────────────────────
print(f"\n\n⛓️  Chain Rule — قلب Backpropagation:")
print("-" * 52)
x, w, b = 2.0, 0.5, -0.3

z     = x * w + b
r     = relu(z)
y     = sigmoid(r)

dy_dr = deriv(sigmoid, r)
dr_dz = 1.0 if z > 0 else 0.0  # مشتق ReLU
dz_dw = x
dy_dw = dy_dr * dr_dz * dz_dw

print(f"  الشبكة: y = sigmoid(relu(x·w + b))")
print(f"  x={x}, w={w}, b={b}")
print(f"  z = {z:.3f}, r = {r:.3f}, y = {y:.4f}")
print(f"  dy/dr = {dy_dr:.4f}")
print(f"  dr/dz = {dr_dz:.1f}")
print(f"  dz/dw = {dz_dw:.1f}")
dw_str = f"{dy_dw:.5f}"
print(f"  dy/dw = {dw_str}  ← نُحدّث w بهذا التدرج")
print(f"\n✅ Chain Rule يُمكّن تعلم الشبكات العصبية!")

PreviousNext

Derivatives and Gradients

Derivatives and gradients are the mathematical tools that neural networks use to learn.

Derivative:

Measures the rate of change of a function. f'(x) = how much f changes when x changes slightly.

Gradient:

Derivative of a multi-variable function. Points in the direction of greatest increase in the function.

Gradient Descent:

The fundamental learning algorithm: w = w - lr × ∇f(w) where lr is the Learning Rate and the gradient points toward increase (we subtract it to decrease)

Backpropagation = Chain Rule:

To compute the gradient of a composite function y = f(g(x)) apply the chain rule: dy/dx = dy/dg × dg/dx

Learning Rate Effect:

Too small → very slow learning

Too large → doesn't converge, oscillates

Just right → fast and smooth convergence

import math from typing import Callable, List, Tuple # ─── مشتق عددي ───────────────────────────────────────────── def deriv(f: Callable[[float], float], x: float, h: float = 1e-5) -> float: """المشتق العددي بالفرق المركزي""" return (f(x + h) - f(x - h)) / (2 * h) def gradient(f: Callable[[List[float]], float], params: List[float], h: float = 1e-5) -> List[float]: """تدرج دالة متعددة المتغيرات""" grads = [] for i in range(len(params)): p1, p2 = params[:], params[:] p1[i] += h p2[i] -= h grads.append((f(p1) - f(p2)) / (2 * h)) return grads # ─── دوال التفعيل ────────────────────────────────────────── def sigmoid(x: float) -> float: return 1 / (1 + math.exp(-x)) def relu(x: float) -> float: return max(0.0, x) def tanh_fn(x: float) -> float: return math.tanh(x) # ─── Gradient Descent ────────────────────────────────────── def gd_demo(f, df, w0: float, lr: float, epochs: int, label: str): """محاكاة Gradient Descent""" w = w0 losses = [f(w)] for _ in range(epochs): w = w - lr * df(w) losses.append(f(w)) init_str = f"{losses[0]:.4f}" final_str = f"{losses[-1]:.4f}" w_str = f"{w:.4f}" conv = "✅" if abs(losses[-1] - losses[0]) > abs(losses[0]) * 0.5 else "⚠️ " print(f" {label}") print(f" Loss: {init_str} → {final_str} | w* = {w_str} {conv}") return w # ─── عرض مشتقات دوال التفعيل ────────────────────────────── print("📐 مشتقات دوال التفعيل:") print("=" * 52) funcs = [("sigmoid", sigmoid), ("relu", relu), ("tanh", tanh_fn)] points = [-2.0, -1.0, 0.0, 1.0, 2.0] for name, fn in funcs: print(f"\n {name}(x) → f(x) | f'(x):") for x in points: val = fn(x) grad = deriv(fn, x) bar = "▓" * max(0, int(abs(grad) * 8)) print(f" x={x:+.1f} → {val:.3f} | {grad:.3f} {bar}") # ─── Gradient Descent ────────────────────────────────────── print(f"\n\n📉 Gradient Descent — تقليل f(w) = w² + 2w - 3:") print(" (الحد الأدنى الحقيقي: w = -1)") print("-" * 52) def f(w): return w**2 + 2*w - 3 def df(w): return 2*w + 2 configs = [ (5.0, 0.01, 40, "LR=0.01 (بطيء) "), (5.0, 0.1, 20, "LR=0.10 (مناسب) "), (5.0, 0.95, 15, "LR=0.95 (كبير جداً)"), ] for w0, lr, ep, label in configs: gd_demo(f, df, w0, lr, ep, label) # ─── Chain Rule ──────────────────────────────────────────── print(f"\n\n⛓️ Chain Rule — قلب Backpropagation:") print("-" * 52) x, w, b = 2.0, 0.5, -0.3 z = x * w + b r = relu(z) y = sigmoid(r) dy_dr = deriv(sigmoid, r) dr_dz = 1.0 if z > 0 else 0.0 # مشتق ReLU dz_dw = x dy_dw = dy_dr * dr_dz * dz_dw print(f" الشبكة: y = sigmoid(relu(x·w + b))") print(f" x={x}, w={w}, b={b}") print(f" z = {z:.3f}, r = {r:.3f}, y = {y:.4f}") print(f" dy/dr = {dy_dr:.4f}") print(f" dr/dz = {dr_dz:.1f}") print(f" dz/dw = {dz_dw:.1f}") dw_str = f"{dy_dw:.5f}" print(f" dy/dw = {dw_str} ← نُحدّث w بهذا التدرج") print(f"\n✅ Chain Rule يُمكّن تعلم الشبكات العصبية!")