Loading
Loading
Building the model is half the work — the other half is knowing whether it's actually good. Here you'll learn how to evaluate models scientifically.
Instead of a single train/test split, K-Fold divides data k times and averages results — providing a more reliable performance estimate with confidence intervals.
Underfitting (High Bias) ←────→ Overfitting (High Variance)
Both curves low & close Train high, Val low — large gap
Regression: MAE (same units), RMSE (penalizes large errors), R² (variance explained), MAPE (% error).
Classification: Accuracy (misleading when imbalanced), Precision/Recall/F1, AUC-ROC (works well with imbalanced classes).
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import (
cross_val_score, StratifiedKFold, learning_curve, GridSearchCV
)
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.datasets import make_classification
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
print("=" * 55)
print("تقييم النماذج: الدليل الكامل")
print("=" * 55)
X, y = make_classification(
n_samples=1500, n_features=15, n_informative=8,
n_classes=2, random_state=42
)
# ─────────────────────────────────────────
# 1. K-Fold Cross-Validation
# ─────────────────────────────────────────
print("\n1. Stratified K-Fold Cross-Validation (k=10):")
model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
for metric in ["accuracy", "precision", "recall", "f1", "roc_auc"]:
scores = cross_val_score(model, X, y, cv=cv, scoring=metric, n_jobs=-1)
print(f" {metric:12s}: {scores.mean():.3f} ± {scores.std():.3f} "
f"[{scores.min():.3f} - {scores.max():.3f}]")
# ─────────────────────────────────────────
# 2. Learning Curves
# ─────────────────────────────────────────
print("\n2. Learning Curves:")
train_sizes, train_scores, val_scores = learning_curve(
model, X, y,
train_sizes=np.linspace(0.1, 1.0, 10),
cv=5, scoring="accuracy", n_jobs=-1
)
print(" حجم التدريب | Train Acc | Val Acc | فجوة Overfit")
print(" " + "-" * 52)
for i, size in enumerate(train_sizes):
t_mean = train_scores[i].mean()
v_mean = val_scores[i].mean()
gap = t_mean - v_mean
print(f" {int(size):11d} | {t_mean:.3f} | {v_mean:.3f} | {gap:.3f}")
# ─────────────────────────────────────────
# 3. Hyperparameter Tuning (Grid Search)
# ─────────────────────────────────────────
print("\n3. Grid Search لإيجاد أفضل المعاملات:")
param_grid = {
"n_estimators": [50, 100],
"max_depth": [None, 5, 10],
"min_samples_leaf": [1, 5],
}
gs = GridSearchCV(
RandomForestClassifier(random_state=42, n_jobs=-1),
param_grid, cv=5, scoring="roc_auc", n_jobs=-1, verbose=0
)
gs.fit(X, y)
print(f" أفضل معاملات: {gs.best_params_}")
print(f" أفضل AUC-ROC: {gs.best_score_:.4f}")
# ─────────────────────────────────────────
# 4. AUC-ROC Analysis
# ─────────────────────────────────────────
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, stratify=y, random_state=42
)
best_model = gs.best_estimator_
best_model.fit(X_train, y_train)
y_proba = best_model.predict_proba(X_test)[:, 1]
auc = roc_auc_score(y_test, y_proba)
print(f"\n4. AUC-ROC على Test Set: {auc:.4f}")
print(" (1.0 = مثالي | 0.5 = عشوائي | > 0.9 = ممتاز)")