# Install required packages (run this cell first in Google Colab)
import sys
if 'google.colab' in sys.modules:
!pip install ipywidgets -q
from google.colab import output
output.enable_custom_widget_manager()
# Import all necessary libraries
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
import ipywidgets as widgets
from IPython.display import display, clear_output
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.datasets import make_classification, load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')
# Set random seed for reproducibility
np.random.seed(42)
# Set plot style
plt.style.use('seaborn-v0_8-darkgrid')
print("โ
All libraries loaded successfully!")Week 4: Interactive Logistic Regression Demonstrations
MPS311/439 Machine Learning
This notebook contains interactive visualizations to help you understand: - Why linear regression fails for classification - How the sigmoid function works - Decision boundaries and confidence - Precision-Recall tradeoffs - When logistic regression succeeds and fails
๐ฏ Interactive Demo 1: Why Linear Regression Fails for Classification
Key Learning Goal: See how linear regression produces invalid probabilities (< 0 or > 1) and is sensitive to outliers.
def demo_regression_vs_classification(add_outlier=False, outlier_position=10):
"""
Compare linear regression and logistic regression for binary classification.
"""
# Generate simple 1D classification data
np.random.seed(42)
X_class0 = np.random.normal(2, 0.5, 30).reshape(-1, 1)
X_class1 = np.random.normal(5, 0.5, 30).reshape(-1, 1)
X = np.vstack([X_class0, X_class1])
y = np.hstack([np.zeros(30), np.ones(30)])
# Add outlier if requested
if add_outlier:
X = np.vstack([X, [[outlier_position]]])
y = np.append(y, 1)
# Fit both models
lin_reg = LinearRegression().fit(X, y)
log_reg = LogisticRegression().fit(X, y)
# Create prediction space
X_plot = np.linspace(-1, 11, 300).reshape(-1, 1)
y_lin_pred = lin_reg.predict(X_plot)
y_log_pred = log_reg.predict_proba(X_plot)[:, 1]
# Create figure
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
# Plot 1: Linear Regression
ax1.scatter(X[y==0], y[y==0], c='blue', s=100, alpha=0.6, label='Class 0', edgecolors='k')
ax1.scatter(X[y==1], y[y==1], c='red', s=100, alpha=0.6, label='Class 1', edgecolors='k')
ax1.plot(X_plot, y_lin_pred, 'g-', linewidth=3, label='Linear Regression')
ax1.axhline(y=0, color='gray', linestyle='--', alpha=0.3)
ax1.axhline(y=1, color='gray', linestyle='--', alpha=0.3)
# Highlight invalid predictions
invalid_below = y_lin_pred < 0
invalid_above = y_lin_pred > 1
ax1.fill_between(X_plot.ravel(), -0.3, 0, where=invalid_below.ravel(),
alpha=0.3, color='orange', label='Invalid: P < 0')
ax1.fill_between(X_plot.ravel(), 1, 1.3, where=invalid_above.ravel(),
alpha=0.3, color='orange', label='Invalid: P > 1')
ax1.set_xlabel('Feature Value (x)', fontsize=12)
ax1.set_ylabel('Prediction', fontsize=12)
ax1.set_title('โ Linear Regression: Invalid Probabilities', fontsize=14, fontweight='bold')
ax1.legend(loc='best')
ax1.set_ylim([-0.3, 1.3])
ax1.grid(True, alpha=0.3)
# Plot 2: Logistic Regression
ax2.scatter(X[y==0], y[y==0], c='blue', s=100, alpha=0.6, label='Class 0', edgecolors='k')
ax2.scatter(X[y==1], y[y==1], c='red', s=100, alpha=0.6, label='Class 1', edgecolors='k')
ax2.plot(X_plot, y_log_pred, 'purple', linewidth=3, label='Logistic Regression')
ax2.axhline(y=0.5, color='black', linestyle='--', linewidth=2, alpha=0.5, label='Decision Threshold (0.5)')
ax2.axhline(y=0, color='gray', linestyle='--', alpha=0.3)
ax2.axhline(y=1, color='gray', linestyle='--', alpha=0.3)
# Highlight valid probability range
ax2.fill_between(X_plot.ravel(), 0, 1, alpha=0.1, color='green', label='Valid: 0 โค P โค 1')
ax2.set_xlabel('Feature Value (x)', fontsize=12)
ax2.set_ylabel('P(Class = 1 | x)', fontsize=12)
ax2.set_title('โ
Logistic Regression: Valid Probabilities', fontsize=14, fontweight='bold')
ax2.legend(loc='best')
ax2.set_ylim([-0.3, 1.3])
ax2.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
# Print analysis
print("\n๐ Analysis:")
print(f" โข Linear Regression: {np.sum(invalid_below) + np.sum(invalid_above)} invalid predictions")
print(f" โข Logistic Regression: Always outputs valid probabilities [0, 1]")
if add_outlier:
print(f"\nโ ๏ธ With outlier at x={outlier_position}, linear regression is heavily influenced!")
# Create interactive widget
widgets.interact(
demo_regression_vs_classification,
add_outlier=widgets.Checkbox(value=False, description='Add Outlier'),
outlier_position=widgets.FloatSlider(min=6, max=12, step=0.5, value=10,
description='Outlier X:',
style={'description_width': 'initial'})
);๐ Key Takeaways: - Linear regression can predict probabilities < 0 or > 1 (impossible!) - Outliers dramatically affect linear regression - Logistic regression always outputs valid probabilities
๐ก Try This: 1. Turn on the outlier - see how linear regression breaks! 2. Move the outlier further right - watch the chaos!
๐ฏ Interactive Demo 2: Exploring the Sigmoid Function
Key Learning Goal: Understand how the sigmoid function transforms any number into a valid probability.
def demo_sigmoid_function(slope=1.0, shift=0.0, show_derivative=False):
"""
Interactive sigmoid function visualization.
"""
# Define sigmoid function
def sigmoid(z):
return 1 / (1 + np.exp(-z))
def sigmoid_derivative(z):
s = sigmoid(z)
return s * (1 - s)
# Generate data
z = np.linspace(-10, 10, 400)
z_transformed = slope * (z - shift)
y = sigmoid(z_transformed)
y_deriv = sigmoid_derivative(z_transformed) * slope
# Create figure
fig, ax = plt.subplots(figsize=(12, 6))
# Plot sigmoid
ax.plot(z, y, 'b-', linewidth=3, label=f'ฯ(z) = 1 / (1 + e^(-{slope}*(z-{shift})))')
# Plot derivative if requested
if show_derivative:
ax.plot(z, y_deriv, 'r--', linewidth=2, alpha=0.7, label="ฯ'(z) - Derivative")
# Add reference lines
ax.axhline(y=0.5, color='black', linestyle='--', linewidth=1.5, alpha=0.5, label='Decision Boundary (0.5)')
ax.axhline(y=0, color='gray', linestyle=':', alpha=0.3)
ax.axhline(y=1, color='gray', linestyle=':', alpha=0.3)
ax.axvline(x=shift, color='green', linestyle=':', alpha=0.5, label=f'Center at z={shift}')
# Highlight probability regions
ax.fill_between(z, 0, 0.5, where=(y < 0.5), alpha=0.1, color='blue', label='Class 0 Region')
ax.fill_between(z, 0.5, 1, where=(y >= 0.5), alpha=0.1, color='red', label='Class 1 Region')
# Add annotation for key points
ax.plot(shift, 0.5, 'go', markersize=12, label=f'Center: ({shift:.1f}, 0.5)')
# Labels and formatting
ax.set_xlabel('z = wโ + wโxโ + wโxโ + ... (Linear Combination)', fontsize=13, fontweight='bold')
ax.set_ylabel('ฯ(z) = P(y=1|x)', fontsize=13, fontweight='bold')
ax.set_title('The Sigmoid Function: Squashing Any Number to [0, 1]', fontsize=15, fontweight='bold')
ax.legend(loc='best', fontsize=10)
ax.grid(True, alpha=0.3)
ax.set_ylim([-0.1, 1.1])
plt.tight_layout()
plt.show()
# Print key properties
print("\n๐ Sigmoid Properties:")
print(f" โข Input range: (-โ, +โ)")
print(f" โข Output range: (0, 1) - Always a valid probability!")
print(f" โข ฯ(0) = {sigmoid(0):.3f} (decision point)")
print(f" โข ฯ({shift}) = {sigmoid(0):.3f} (center with current shift)")
print(f" โข Current slope (steepness): {slope}x")
if slope > 1:
print(f" โ Steeper curve = More confident predictions")
elif slope < 1:
print(f" โ Gentler curve = More uncertain predictions")
print(f"\n๐ก In logistic regression: z = wโ + wโxโ + wโxโ + ...")
print(f" Then we apply sigmoid: P(y=1|x) = ฯ(z)")
# Create interactive widget
widgets.interact(
demo_sigmoid_function,
slope=widgets.FloatSlider(min=0.2, max=3.0, step=0.1, value=1.0,
description='Slope (Steepness):',
style={'description_width': 'initial'}),
shift=widgets.FloatSlider(min=-5, max=5, step=0.5, value=0,
description='Horizontal Shift:',
style={'description_width': 'initial'}),
show_derivative=widgets.Checkbox(value=False, description='Show Derivative')
);๐ Key Takeaways: - Sigmoid takes ANY number and outputs a probability [0, 1] - The center point (ฯ = 0.5) is where we make our decision - Slope controls confidence: steeper = more confident - Horizontal shift moves the decision boundary
๐ก Try This: 1. Increase slope to 3 - see how it becomes more like a step function! 2. Shift right/left - see how the decision point moves 3. Check โShow Derivativeโ - see where the function changes fastest
๐ฏ Interactive Demo 3: 2D Decision Boundaries
Key Learning Goal: Visualize how logistic regression creates linear decision boundaries in 2D space.
def demo_decision_boundary(w0=0, w1=1, w2=1, show_probabilities=True):
"""
Interactive 2D decision boundary visualization.
"""
# Generate 2D classification data
np.random.seed(42)
X, y = make_classification(n_samples=200, n_features=2, n_redundant=0,
n_informative=2, n_clusters_per_class=1,
class_sep=1.5, random_state=42)
# Create mesh for plotting
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200),
np.linspace(y_min, y_max, 200))
# Compute probabilities using specified weights
def sigmoid(z):
return 1 / (1 + np.exp(-z))
z = w0 + w1 * xx + w2 * yy
probs = sigmoid(z)
# Create figure
fig, ax = plt.subplots(figsize=(10, 8))
# Plot probability contours if requested
if show_probabilities:
contour_filled = ax.contourf(xx, yy, probs, levels=20, cmap='RdBu_r', alpha=0.6)
cbar = plt.colorbar(contour_filled, ax=ax)
cbar.set_label('P(Class = 1 | x)', fontsize=11, fontweight='bold')
# Plot decision boundary (P = 0.5)
ax.contour(xx, yy, probs, levels=[0.5], colors='black', linewidths=3)
# Plot confidence contours
ax.contour(xx, yy, probs, levels=[0.1, 0.3, 0.7, 0.9], colors='gray',
linewidths=1, linestyles='dashed', alpha=0.5)
# Plot data points
scatter0 = ax.scatter(X[y==0, 0], X[y==0, 1], c='blue', s=80,
edgecolors='k', linewidths=1.5, alpha=0.8, label='Class 0')
scatter1 = ax.scatter(X[y==1, 0], X[y==1, 1], c='red', s=80,
edgecolors='k', linewidths=1.5, alpha=0.8, label='Class 1')
# Add decision boundary equation
equation = f'Decision Boundary: {w0:.1f} + {w1:.1f}ยทxโ + {w2:.1f}ยทxโ = 0'
ax.text(0.5, 0.02, equation, transform=ax.transAxes,
fontsize=12, fontweight='bold',
bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.7),
ha='center')
ax.set_xlabel('Feature 1 (xโ)', fontsize=12, fontweight='bold')
ax.set_ylabel('Feature 2 (xโ)', fontsize=12, fontweight='bold')
ax.set_title('Logistic Regression: Linear Decision Boundary', fontsize=14, fontweight='bold')
ax.legend(loc='upper right', fontsize=11)
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
# Calculate and display statistics
predictions = (probs.ravel() > 0.5).astype(int)
print("\n๐ Model Analysis:")
print(f" โข Decision boundary: {w0:.1f} + {w1:.1f}ยทxโ + {w2:.1f}ยทxโ = 0")
print(f" โข Boundary is {'steep' if abs(w2/w1) > 2 else 'gentle' if abs(w2/w1) < 0.5 else 'moderate'}")
print(f"\n๐ก Interpretation:")
if w1 > 0:
print(f" โข Feature 1 โ โ Probability of Class 1 โ")
else:
print(f" โข Feature 1 โ โ Probability of Class 1 โ")
if w2 > 0:
print(f" โข Feature 2 โ โ Probability of Class 1 โ")
else:
print(f" โข Feature 2 โ โ Probability of Class 1 โ")
# Create interactive widget
widgets.interact(
demo_decision_boundary,
w0=widgets.FloatSlider(min=-3, max=3, step=0.2, value=0,
description='wโ (Intercept):',
style={'description_width': 'initial'}),
w1=widgets.FloatSlider(min=-3, max=3, step=0.2, value=1,
description='wโ (Weight for xโ):',
style={'description_width': 'initial'}),
w2=widgets.FloatSlider(min=-3, max=3, step=0.2, value=1,
description='wโ (Weight for xโ):',
style={'description_width': 'initial'}),
show_probabilities=widgets.Checkbox(value=True, description='Show Probability Gradient')
);๐ Key Takeaways: - Decision boundary is a straight line (linear!) - Points far from boundary = high confidence - Points near boundary = uncertain predictions - Weights control the angle and position of the boundary
๐ก Try This: 1. Set wโ=2, wโ=0 - vertical boundary (only xโ matters) 2. Set wโ=0, wโ=2 - horizontal boundary (only xโ matters) 3. Adjust wโ to shift the boundary without changing angle 4. Try wโ=-2 to flip which side predicts which class!
๐ฏ Interactive Demo 4: Threshold Adjustment & Precision-Recall Tradeoff
Key Learning Goal: Understand the fundamental tradeoff between precision and recall by adjusting the decision threshold.
# Load real breast cancer data
data = load_breast_cancer()
X, y = data.data, data.target
# Use only 2 features for visualization
X_2d = X[:, [0, 1]] # mean radius and mean texture
X_train, X_test, y_train, y_test = train_test_split(X_2d, y, test_size=0.3, random_state=42)
# Standardize features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# Train logistic regression
model = LogisticRegression(random_state=42)
model.fit(X_train_scaled, y_train)
# Get probability predictions
y_proba = model.predict_proba(X_test_scaled)[:, 1]
print("โ
Breast Cancer Model Trained!")
print(f" Dataset: {len(X_train)} training samples, {len(X_test)} test samples")
print(f" Features: Mean Radius & Mean Texture")
print(f" Classes: Malignant (1) vs Benign (0)")def demo_threshold_tradeoff(threshold=0.5):
"""
Interactive demonstration of precision-recall tradeoff.
"""
# Apply threshold
y_pred = (y_proba >= threshold).astype(int)
# Calculate metrics
cm = confusion_matrix(y_test, y_pred)
tn, fp, fn, tp = cm.ravel()
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, zero_division=0)
recall = recall_score(y_test, y_pred, zero_division=0)
f1 = f1_score(y_test, y_pred, zero_division=0)
# Create figure with 3 subplots
fig = plt.figure(figsize=(16, 5))
# Subplot 1: Confusion Matrix
ax1 = plt.subplot(1, 3, 1)
im = ax1.imshow(cm, cmap='Blues', alpha=0.8)
# Add text annotations
for i in range(2):
for j in range(2):
text = ax1.text(j, i, cm[i, j], ha="center", va="center",
fontsize=24, fontweight='bold',
color="white" if cm[i, j] > cm.max()/2 else "black")
ax1.set_xticks([0, 1])
ax1.set_yticks([0, 1])
ax1.set_xticklabels(['Predicted\nBenign (0)', 'Predicted\nMalignant (1)'], fontsize=10)
ax1.set_yticklabels(['Actual\nBenign (0)', 'Actual\nMalignant (1)'], fontsize=10)
ax1.set_title(f'Confusion Matrix\n(Threshold = {threshold:.2f})', fontsize=13, fontweight='bold')
# Add labels for each cell
ax1.text(0, -0.5, f'TN={tn}', ha='center', fontsize=9, color='green', fontweight='bold')
ax1.text(1, -0.5, f'FP={fp}', ha='center', fontsize=9, color='red', fontweight='bold')
ax1.text(0, 1.5, f'FN={fn}', ha='center', fontsize=9, color='red', fontweight='bold')
ax1.text(1, 1.5, f'TP={tp}', ha='center', fontsize=9, color='green', fontweight='bold')
plt.colorbar(im, ax=ax1)
# Subplot 2: Metrics Bar Chart
ax2 = plt.subplot(1, 3, 2)
metrics = ['Accuracy', 'Precision', 'Recall', 'F1-Score']
values = [accuracy, precision, recall, f1]
colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728']
bars = ax2.barh(metrics, values, color=colors, alpha=0.7, edgecolor='black', linewidth=2)
# Add value labels on bars
for i, (bar, value) in enumerate(zip(bars, values)):
ax2.text(value + 0.02, i, f'{value:.3f}', va='center', fontweight='bold', fontsize=11)
ax2.set_xlim([0, 1.1])
ax2.set_xlabel('Score', fontsize=12, fontweight='bold')
ax2.set_title('Performance Metrics', fontsize=13, fontweight='bold')
ax2.grid(axis='x', alpha=0.3)
ax2.axvline(x=0.5, color='gray', linestyle='--', alpha=0.5)
# Subplot 3: Threshold Analysis
ax3 = plt.subplot(1, 3, 3)
# Calculate metrics across thresholds
thresholds = np.linspace(0.05, 0.95, 50)
precisions = []
recalls = []
f1_scores = []
for t in thresholds:
y_pred_t = (y_proba >= t).astype(int)
precisions.append(precision_score(y_test, y_pred_t, zero_division=0))
recalls.append(recall_score(y_test, y_pred_t, zero_division=0))
f1_scores.append(f1_score(y_test, y_pred_t, zero_division=0))
ax3.plot(thresholds, precisions, 'o-', label='Precision', linewidth=2, markersize=4, color='#ff7f0e')
ax3.plot(thresholds, recalls, 's-', label='Recall', linewidth=2, markersize=4, color='#2ca02c')
ax3.plot(thresholds, f1_scores, '^-', label='F1-Score', linewidth=2, markersize=4, color='#d62728')
# Highlight current threshold
ax3.axvline(x=threshold, color='black', linestyle='--', linewidth=2, label=f'Current ({threshold:.2f})')
ax3.plot(threshold, precision, 'o', color='#ff7f0e', markersize=12, markeredgecolor='black', markeredgewidth=2)
ax3.plot(threshold, recall, 's', color='#2ca02c', markersize=12, markeredgecolor='black', markeredgewidth=2)
ax3.plot(threshold, f1, '^', color='#d62728', markersize=12, markeredgecolor='black', markeredgewidth=2)
ax3.set_xlabel('Decision Threshold', fontsize=12, fontweight='bold')
ax3.set_ylabel('Score', fontsize=12, fontweight='bold')
ax3.set_title('Precision-Recall Tradeoff', fontsize=13, fontweight='bold')
ax3.legend(loc='best', fontsize=10)
ax3.grid(True, alpha=0.3)
ax3.set_ylim([0, 1.05])
plt.tight_layout()
plt.show()
# Print detailed interpretation
print("\n" + "="*70)
print(f"๐ ANALYSIS FOR THRESHOLD = {threshold:.2f}")
print("="*70)
print(f"\n๐ข Confusion Matrix Breakdown:")
print(f" โข True Negatives (TN): {tn:3d} - Correctly identified benign cases")
print(f" โข False Positives (FP): {fp:3d} - Benign cases wrongly flagged as malignant")
print(f" โข False Negatives (FN): {fn:3d} - Malignant cases missed (DANGEROUS!)")
print(f" โข True Positives (TP): {tp:3d} - Correctly identified malignant cases")
print(f"\n๐ Performance Metrics:")
print(f" โข Accuracy: {accuracy:.3f} - Overall correctness")
print(f" โข Precision: {precision:.3f} - Of predicted malignant, how many are actually malignant?")
print(f" โข Recall: {recall:.3f} - Of actual malignant, how many did we catch?")
print(f" โข F1-Score: {f1:.3f} - Harmonic mean of precision and recall")
print(f"\n๐ก Clinical Interpretation:")
if threshold < 0.3:
print(" ๐จ AGGRESSIVE screening (Low threshold):")
print(" โ Catches almost all cancer cases (high recall)")
print(" โ Many false alarms (low precision)")
print(" โ Good for initial screening where missing cancer is worse than false alarms")
elif threshold > 0.7:
print(" ๐ฏ CONSERVATIVE approach (High threshold):")
print(" โ Few false alarms (high precision)")
print(" โ Might miss some cancer cases (lower recall)")
print(" โ Use when confirmatory tests are expensive or risky")
else:
print(" โ๏ธ BALANCED approach (Default threshold):")
print(" Reasonable trade-off between catching cancers and avoiding false alarms")
print(f"\n๐ Key Insight:")
print(" There is NO FREE LUNCH! You cannot maximize both precision and recall.")
print(" Choose threshold based on the cost of different types of errors in your application.")
print("="*70)
# Create interactive widget
widgets.interact(
demo_threshold_tradeoff,
threshold=widgets.FloatSlider(min=0.1, max=0.9, step=0.05, value=0.5,
description='Decision Threshold:',
style={'description_width': 'initial'},
readout_format='.2f')
);๐ Key Takeaways: - Default threshold (0.5) isnโt always optimal! - Precision vs Recall Tradeoff: - Lower threshold โ Higher recall (catch more cancers) but lower precision (more false alarms) - Higher threshold โ Higher precision (fewer false alarms) but lower recall (miss some cancers) - Choice depends on application: - Medical screening: Prefer high recall (donโt miss diseases) - Spam filtering: Prefer high precision (real emails must not go to spam)
๐ก Try This: 1. Set threshold to 0.2 - see how recall increases but precision drops 2. Set threshold to 0.8 - see how precision increases but recall drops 3. Find the threshold that maximizes F1-score (balance) 4. Imagine youโre a doctor: which threshold would you choose? Why?
๐ฏ Interactive Demo 5: When Logistic Regression Fails
Key Learning Goal: Understand that linear boundaries cannot solve all classification problems.
def demo_linear_vs_nonlinear(dataset_type='Linear', noise_level=0.1):
"""
Compare logistic regression performance on linearly and non-linearly separable data.
"""
np.random.seed(42)
if dataset_type == 'Linear':
# Generate linearly separable data
X, y = make_classification(n_samples=300, n_features=2, n_redundant=0,
n_informative=2, n_clusters_per_class=1,
class_sep=2.0, flip_y=noise_level*2, random_state=42)
title_suffix = "Linearly Separable"
elif dataset_type == 'Circular':
# Generate circular pattern (non-linear)
n_samples = 300
# Inner circle (class 0)
r_inner = np.random.uniform(0, 1.5, n_samples//2)
theta_inner = np.random.uniform(0, 2*np.pi, n_samples//2)
X_inner = np.column_stack([r_inner * np.cos(theta_inner),
r_inner * np.sin(theta_inner)])
# Outer circle (class 1)
r_outer = np.random.uniform(2.5, 4, n_samples//2)
theta_outer = np.random.uniform(0, 2*np.pi, n_samples//2)
X_outer = np.column_stack([r_outer * np.cos(theta_outer),
r_outer * np.sin(theta_outer)])
X = np.vstack([X_inner, X_outer])
y = np.hstack([np.zeros(n_samples//2), np.ones(n_samples//2)])
# Add noise
X += np.random.normal(0, noise_level*2, X.shape)
title_suffix = "Circular (Non-Linear)"
elif dataset_type == 'XOR':
# Generate XOR pattern (non-linear)
n_samples = 300
# Four clusters
X1 = np.random.randn(n_samples//4, 2) * 0.6 + np.array([1.5, 1.5])
X2 = np.random.randn(n_samples//4, 2) * 0.6 + np.array([-1.5, -1.5])
X3 = np.random.randn(n_samples//4, 2) * 0.6 + np.array([1.5, -1.5])
X4 = np.random.randn(n_samples//4, 2) * 0.6 + np.array([-1.5, 1.5])
X = np.vstack([X1, X2, X3, X4])
y = np.hstack([np.ones(n_samples//2), np.zeros(n_samples//2)])
# Add noise
X += np.random.normal(0, noise_level, X.shape)
title_suffix = "XOR Pattern (Non-Linear)"
else: # Moons
from sklearn.datasets import make_moons
X, y = make_moons(n_samples=300, noise=noise_level, random_state=42)
title_suffix = "Two Moons (Non-Linear)"
# Train logistic regression
model = LogisticRegression(random_state=42)
model.fit(X, y)
# Calculate accuracy
y_pred = model.predict(X)
accuracy = accuracy_score(y, y_pred)
# Create mesh for decision boundary
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200),
np.linspace(y_min, y_max, 200))
Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
Z = Z.reshape(xx.shape)
# Create figure
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
# Plot 1: Data only
ax1.scatter(X[y==0, 0], X[y==0, 1], c='blue', s=60, alpha=0.8,
edgecolors='k', linewidths=1, label='Class 0')
ax1.scatter(X[y==1, 0], X[y==1, 1], c='red', s=60, alpha=0.8,
edgecolors='k', linewidths=1, label='Class 1')
ax1.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax1.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax1.set_title(f'Dataset: {title_suffix}', fontsize=13, fontweight='bold')
ax1.legend(loc='best')
ax1.grid(True, alpha=0.3)
# Plot 2: Decision boundary
contour = ax2.contourf(xx, yy, Z, levels=20, cmap='RdBu_r', alpha=0.6)
ax2.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=3)
# Plot misclassified points
correct = y_pred == y
ax2.scatter(X[correct & (y==0), 0], X[correct & (y==0), 1],
c='blue', s=60, alpha=0.8, edgecolors='k', linewidths=1, label='Class 0 (Correct)')
ax2.scatter(X[correct & (y==1), 0], X[correct & (y==1), 1],
c='red', s=60, alpha=0.8, edgecolors='k', linewidths=1, label='Class 1 (Correct)')
ax2.scatter(X[~correct, 0], X[~correct, 1],
c='yellow', s=100, alpha=1, edgecolors='red', linewidths=3,
marker='X', label='Misclassified', zorder=5)
ax2.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax2.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax2.set_title(f'Logistic Regression Boundary | Accuracy: {accuracy:.1%}',
fontsize=13, fontweight='bold')
ax2.legend(loc='best')
ax2.grid(True, alpha=0.3)
plt.colorbar(contour, ax=ax2, label='P(Class = 1)')
plt.tight_layout()
plt.show()
# Print analysis
print("\n" + "="*70)
print(f"๐ PERFORMANCE ANALYSIS: {title_suffix}")
print("="*70)
print(f"\n โข Accuracy: {accuracy:.1%}")
print(f" โข Misclassified points: {np.sum(~correct)} out of {len(y)}")
if dataset_type == 'Linear':
print("\n โ
SUCCESS! Linear boundary works well for linearly separable data.")
print(" The straight line effectively separates the two classes.")
else:
print("\n โ FAILURE! Linear boundary cannot capture the non-linear pattern.")
print(" A straight line is fundamentally insufficient for this problem.")
print("\n ๐ก Solutions:")
print(" 1. Feature Engineering: Add polynomial features (xโยฒ, xโยฒ, xโยทxโ)")
print(" 2. Use non-linear models: QDA, Decision Trees, Neural Networks")
print(" 3. Kernel methods: SVM with RBF kernel")
print("="*70)
# Create interactive widget
widgets.interact(
demo_linear_vs_nonlinear,
dataset_type=widgets.Dropdown(
options=['Linear', 'Circular', 'XOR', 'Moons'],
value='Linear',
description='Dataset Type:',
style={'description_width': 'initial'}
),
noise_level=widgets.FloatSlider(
min=0.0, max=0.5, step=0.05, value=0.1,
description='Noise Level:',
style={'description_width': 'initial'}
)
);๐ Key Takeaways: - Logistic regression draws a straight line decision boundary - Works great for linearly separable data - Fails on non-linear patterns (circles, XOR, moons) - Need different approaches for complex patterns: - Feature engineering (polynomial features) - Non-linear models (QDA, trees, neural networks)
๐ก Try This: 1. Start with โLinearโ - see the success (high accuracy) 2. Switch to โCircularโ - watch it struggle (low accuracy) 3. Try โXORโ - even worse! A straight line canโt separate opposite corners 4. Try โMoonsโ - see the challenge of curved boundaries 5. Increase noise - see how performance degrades
๐ฎ Preview Week 5: Next week weโll learn about QDA, which can handle curved boundaries!
๐ฏ Bonus Demo: Loss Function Visualization
Key Learning Goal: Understand why we use cross-entropy loss instead of squared error.
def demo_loss_functions(show_both=True):
"""
Visualize cross-entropy vs squared error loss.
"""
# Generate predictions from 0 to 1
predictions = np.linspace(0.01, 0.99, 100)
# Calculate losses for true label = 1
cross_entropy_y1 = -np.log(predictions)
squared_error_y1 = (1 - predictions) ** 2
# Calculate losses for true label = 0
cross_entropy_y0 = -np.log(1 - predictions)
squared_error_y0 = (0 - predictions) ** 2
# Create figure
fig, axes = plt.subplots(1, 2, figsize=(15, 5))
# Plot 1: When true label = 1
ax1 = axes[0]
ax1.plot(predictions, cross_entropy_y1, 'b-', linewidth=3, label='Cross-Entropy Loss')
if show_both:
ax1.plot(predictions, squared_error_y1, 'r--', linewidth=3, label='Squared Error Loss')
# Highlight key regions
ax1.axvline(x=0.5, color='gray', linestyle=':', alpha=0.5)
ax1.fill_between(predictions, 0, 10, where=(predictions < 0.5),
alpha=0.1, color='red', label='Wrong Prediction Region')
ax1.fill_between(predictions, 0, 10, where=(predictions >= 0.5),
alpha=0.1, color='green', label='Correct Prediction Region')
# Add annotations
ax1.annotate('Confident &\nWrong\nโ HUGE Penalty!',
xy=(0.1, cross_entropy_y1[9]), xytext=(0.15, 6),
arrowprops=dict(arrowstyle='->', lw=2, color='red'),
fontsize=11, fontweight='bold', color='red',
bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.7))
ax1.annotate('Confident &\nCorrect\nโ Small Penalty',
xy=(0.95, cross_entropy_y1[-5]), xytext=(0.7, 2),
arrowprops=dict(arrowstyle='->', lw=2, color='green'),
fontsize=11, fontweight='bold', color='green',
bbox=dict(boxstyle='round', facecolor='lightgreen', alpha=0.7))
ax1.set_xlabel('Predicted Probability P(y=1)', fontsize=12, fontweight='bold')
ax1.set_ylabel('Loss', fontsize=12, fontweight='bold')
ax1.set_title('Loss When True Label = 1', fontsize=13, fontweight='bold')
ax1.legend(loc='upper right', fontsize=10)
ax1.grid(True, alpha=0.3)
ax1.set_ylim([0, 8])
# Plot 2: When true label = 0
ax2 = axes[1]
ax2.plot(predictions, cross_entropy_y0, 'b-', linewidth=3, label='Cross-Entropy Loss')
if show_both:
ax2.plot(predictions, squared_error_y0, 'r--', linewidth=3, label='Squared Error Loss')
# Highlight key regions
ax2.axvline(x=0.5, color='gray', linestyle=':', alpha=0.5)
ax2.fill_between(predictions, 0, 10, where=(predictions > 0.5),
alpha=0.1, color='red', label='Wrong Prediction Region')
ax2.fill_between(predictions, 0, 10, where=(predictions <= 0.5),
alpha=0.1, color='green', label='Correct Prediction Region')
# Add annotations
ax2.annotate('Confident &\nWrong\nโ HUGE Penalty!',
xy=(0.9, cross_entropy_y0[-10]), xytext=(0.65, 6),
arrowprops=dict(arrowstyle='->', lw=2, color='red'),
fontsize=11, fontweight='bold', color='red',
bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.7))
ax2.annotate('Confident &\nCorrect\nโ Small Penalty',
xy=(0.05, cross_entropy_y0[4]), xytext=(0.25, 2),
arrowprops=dict(arrowstyle='->', lw=2, color='green'),
fontsize=11, fontweight='bold', color='green',
bbox=dict(boxstyle='round', facecolor='lightgreen', alpha=0.7))
ax2.set_xlabel('Predicted Probability P(y=1)', fontsize=12, fontweight='bold')
ax2.set_ylabel('Loss', fontsize=12, fontweight='bold')
ax2.set_title('Loss When True Label = 0', fontsize=13, fontweight='bold')
ax2.legend(loc='upper left', fontsize=10)
ax2.grid(True, alpha=0.3)
ax2.set_ylim([0, 8])
plt.tight_layout()
plt.show()
# Print explanation
print("\n" + "="*70)
print("๐ WHY CROSS-ENTROPY LOSS?")
print("="*70)
print("\nโ
Cross-Entropy Advantages:")
print(" 1. Heavily penalizes confident wrong predictions")
print(" โ Model learns to be cautious when uncertain")
print(" 2. Convex optimization landscape with sigmoid")
print(" โ Gradient descent finds global optimum")
print(" 3. Encourages well-calibrated probabilities")
print(" โ Predicted probabilities match actual frequencies")
print("\nโ Squared Error Problems:")
print(" 1. Milder penalty for confident wrong predictions")
print(" โ Model not punished enough for mistakes")
print(" 2. Non-convex with sigmoid (multiple local minima)")
print(" โ Gradient descent might get stuck")
print(" 3. Designed for regression, not classification")
print("\n๐ฏ Key Insight:")
print(" Cross-entropy grows exponentially as confidence in wrong answer increases.")
print(" This strong penalty forces the model to learn better decision boundaries!")
print("="*70)
# Create interactive widget
widgets.interact(
demo_loss_functions,
show_both=widgets.Checkbox(value=True, description='Compare with Squared Error')
);๐ Key Takeaways: - Cross-entropy loss heavily penalizes confident wrong predictions - Creates a convex optimization problem (good for training!) - Encourages well-calibrated probabilities - Much better than squared error for classification
๐ก Try This: 1. Uncheck โCompare with Squared Errorโ to focus on cross-entropy alone 2. Notice how the penalty explodes when confident and wrong!
๐ Summary: Key Concepts Covered
What We Learned Today:
- Why Linear Regression Fails โ
- Produces invalid probabilities (< 0 or > 1)
- Sensitive to outliers
- Doesnโt respect probability constraints
- The Sigmoid Function ๐
- Transforms any number to valid probability [0, 1]
- S-shaped curve
- ฯ(z) = 1 / (1 + e^(-z))
- Logistic Regression Model ๐ฏ
- Linear combination: z = wโ + wโxโ + wโxโ + โฆ
- Apply sigmoid: P(y=1|x) = ฯ(z)
- Decision: Predict 1 if P > 0.5, else 0
- Decision Boundaries ๐
- Always a straight line (linear!)
- Points far from boundary = confident
- Points near boundary = uncertain
- Precision vs Recall Tradeoff โ๏ธ
- Precision: Of predicted positives, how many are correct?
- Recall: Of actual positives, how many did we catch?
- Canโt maximize both - must choose based on application!
- Threshold adjustment controls the tradeoff
- Limitations โ ๏ธ
- Only works for linearly separable data
- Fails on non-linear patterns (circles, XOR, etc.)
- Solution: Feature engineering or non-linear models
- Cross-Entropy Loss ๐
- Better than squared error for classification
- Heavily penalizes confident wrong predictions
- Creates convex optimization landscape
๐ Next Week Preview:
Discriminant Analysis (LDA/QDA) - Handle non-linear decision boundaries - Understand probabilistic classification - When to use LDA vs QDA vs Logistic Regression
๐ Practice Exercises
Go back to Demo 4: Try to find the threshold that maximizes F1-score for the cancer dataset
Experiment with Demo 3:
- Can you position the boundary to separate the two classes perfectly?
- What combination of wโ, wโ, wโ works best?
Think About Your Own Problem:
- If you were building a fraud detection system, would you prefer high precision or high recall? Why?
- What about a disease screening test?
- What about a recommendation system?
Challenge: Load your own dataset and apply logistic regression!
- Use
sklearn.datasets.load_*to load a built-in dataset - Or upload your own CSV file
- Try different thresholds and see how metrics change
- Use
๐ฌ Questions?
Feel free to experiment with all the interactive demos above! Try different parameter values and see what happens. The best way to learn is by doing! ๐