# Import required libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import ipywidgets as widgets
from ipywidgets import interact, interactive, fixed
from IPython.display import display
import warnings
warnings.filterwarnings('ignore')
# Set random seed for reproducibility
np.random.seed(42)
# Set plot style
plt.style.use('seaborn-v0_8-darkgrid')
%matplotlib inline
print("✅ All libraries imported successfully!")
print("\n📊 Ready for interactive demonstrations!")Week 5: Linear Discriminant Analysis - Interactive Demonstrations
MPS311/439 Machine Learning
This notebook contains three interactive demonstrations to help you understand: 1. How LDA finds the optimal projection direction 2. How LDA compares to Logistic Regression 3. When to use LDA vs QDA
Instructions: Run each cell and use the sliders to explore different scenarios!
Demo 1: LDA Projection Visualization
Understanding How LDA Finds the Optimal Projection
This demo shows: - Original 2D data with two classes - The projection direction (arrow) found by LDA - The decision boundary (perpendicular to projection) - Projected 1D values as histograms
Play with the sliders to see how different data characteristics affect LDA!
def plot_lda_projection(n_samples=200, class_sep=1.0, cluster_std=1.0, random_state=42):
"""
Interactive visualization of LDA projection
Parameters:
- n_samples: Number of samples per class
- class_sep: Separation between class centers (higher = easier to separate)
- cluster_std: Standard deviation within each class (higher = more spread)
- random_state: Random seed for reproducibility
"""
# Generate synthetic 2D data
X, y = make_classification(
n_samples=n_samples*2,
n_features=2,
n_redundant=0,
n_informative=2,
n_clusters_per_class=1,
class_sep=class_sep,
flip_y=0,
random_state=random_state,
shuffle=True
)
# Scale the cluster spread
X = X * cluster_std
# Fit LDA
lda = LinearDiscriminantAnalysis()
lda.fit(X, y)
# Get projection direction (normalized)
w = lda.coef_[0]
w_normalized = w / np.linalg.norm(w)
# Project data onto LDA direction
X_projected = X @ w.T
# Create figure with subplots
fig = plt.figure(figsize=(16, 6))
# Subplot 1: Original 2D data with projection direction
ax1 = plt.subplot(1, 3, 1)
# Plot data points
scatter1 = ax1.scatter(X[y==0, 0], X[y==0, 1], c='blue', label='Class 0',
alpha=0.6, edgecolors='k', s=50)
scatter2 = ax1.scatter(X[y==1, 0], X[y==1, 1], c='red', label='Class 1',
alpha=0.6, edgecolors='k', s=50)
# Plot projection direction as arrow
center = X.mean(axis=0)
arrow_scale = 3
ax1.arrow(center[0], center[1],
w_normalized[0]*arrow_scale, w_normalized[1]*arrow_scale,
head_width=0.3, head_length=0.3, fc='green', ec='green',
linewidth=3, label='Projection direction w')
# Plot decision boundary (perpendicular to w)
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200),
np.linspace(y_min, y_max, 200))
Z = lda.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
ax1.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=2,
linestyles='--', label='Decision boundary')
ax1.contourf(xx, yy, Z, alpha=0.2, levels=[0, 0.5, 1], colors=['blue', 'red'])
ax1.set_xlabel('Feature 1', fontsize=12)
ax1.set_ylabel('Feature 2', fontsize=12)
ax1.set_title('Original 2D Data with LDA Projection', fontsize=14, fontweight='bold')
ax1.legend(loc='best')
ax1.grid(True, alpha=0.3)
ax1.set_xlim(x_min, x_max)
ax1.set_ylim(y_min, y_max)
# Subplot 2: Projection onto line
ax2 = plt.subplot(1, 3, 2)
# Project points onto the projection direction line
projections_on_line = np.outer(X @ w.T, w_normalized)
# Plot the projection line
line_extent = max(np.abs(X @ w.T)) * 1.2
line_points = np.array([-line_extent, line_extent])
ax2.plot(center[0] + line_points * w_normalized[0],
center[1] + line_points * w_normalized[1],
'g-', linewidth=3, label='Projection line')
# Plot original points (faded)
ax2.scatter(X[y==0, 0], X[y==0, 1], c='blue', alpha=0.2, s=30)
ax2.scatter(X[y==1, 0], X[y==1, 1], c='red', alpha=0.2, s=30)
# Plot projected points on the line
ax2.scatter(center[0] + projections_on_line[y==0, 0],
center[1] + projections_on_line[y==0, 1],
c='blue', s=50, edgecolors='k', label='Class 0 projected')
ax2.scatter(center[0] + projections_on_line[y==1, 0],
center[1] + projections_on_line[y==1, 1],
c='red', s=50, edgecolors='k', label='Class 1 projected')
# Draw lines from points to their projections
for i in range(0, len(X), max(1, len(X)//20)): # Draw only subset for clarity
ax2.plot([X[i, 0], center[0] + projections_on_line[i, 0]],
[X[i, 1], center[1] + projections_on_line[i, 1]],
'gray', alpha=0.3, linewidth=0.5)
ax2.set_xlabel('Feature 1', fontsize=12)
ax2.set_ylabel('Feature 2', fontsize=12)
ax2.set_title('Data Projected onto LDA Direction', fontsize=14, fontweight='bold')
ax2.legend(loc='best')
ax2.grid(True, alpha=0.3)
ax2.set_xlim(x_min, x_max)
ax2.set_ylim(y_min, y_max)
# Subplot 3: 1D projected values (histogram)
ax3 = plt.subplot(1, 3, 3)
# Plot histograms of projected values
ax3.hist(X_projected[y==0], bins=20, color='blue', alpha=0.6,
label='Class 0', edgecolor='black')
ax3.hist(X_projected[y==1], bins=20, color='red', alpha=0.6,
label='Class 1', edgecolor='black')
# Mark class means
mean0 = X_projected[y==0].mean()
mean1 = X_projected[y==1].mean()
ax3.axvline(mean0, color='blue', linewidth=3, linestyle='--', label=f'Mean 0: {mean0:.2f}')
ax3.axvline(mean1, color='red', linewidth=3, linestyle='--', label=f'Mean 1: {mean1:.2f}')
# Mark decision threshold
threshold = (mean0 + mean1) / 2
ax3.axvline(threshold, color='black', linewidth=2, linestyle=':',
label=f'Threshold: {threshold:.2f}')
ax3.set_xlabel('Projected Value (z = w^T x)', fontsize=12)
ax3.set_ylabel('Frequency', fontsize=12)
ax3.set_title('1D Projected Values Distribution', fontsize=14, fontweight='bold')
ax3.legend(loc='best')
ax3.grid(True, alpha=0.3)
# Calculate and display metrics
separation = abs(mean1 - mean0)
std0 = X_projected[y==0].std()
std1 = X_projected[y==1].std()
fisher_ratio = separation / (std0 + std1)
# Add text box with metrics
textstr = f'Separation: {separation:.2f}\nWithin-class std: {(std0+std1)/2:.2f}\nFisher ratio: {fisher_ratio:.2f}'
props = dict(boxstyle='round', facecolor='wheat', alpha=0.8)
ax3.text(0.05, 0.95, textstr, transform=ax3.transAxes, fontsize=10,
verticalalignment='top', bbox=props)
plt.tight_layout()
plt.show()
# Print interpretation
print(f"\n📊 Interpretation:")
print(f" • Projection direction w = [{w[0]:.3f}, {w[1]:.3f}]")
print(f" • Between-class separation: {separation:.3f}")
print(f" • Within-class spread: {(std0+std1)/2:.3f}")
print(f" • Fisher's ratio (separation/spread): {fisher_ratio:.3f}")
print(f"\n💡 Higher Fisher ratio = Better class separation!")
# Create interactive widget
interactive_plot = interactive(
plot_lda_projection,
n_samples=widgets.IntSlider(min=50, max=300, step=50, value=150,
description='Samples/class:', style={'description_width': '150px'}),
class_sep=widgets.FloatSlider(min=0.5, max=3.0, step=0.5, value=1.5,
description='Class separation:', style={'description_width': '150px'}),
cluster_std=widgets.FloatSlider(min=0.5, max=2.0, step=0.25, value=1.0,
description='Cluster spread:', style={'description_width': '150px'}),
random_state=widgets.IntSlider(min=0, max=100, step=1, value=42,
description='Random seed:', style={'description_width': '150px'})
)
display(interactive_plot)🎯 Key Observations from Demo 1:
- Projection Direction (Green Arrow): This is the direction \(\mathbf{w}\) that LDA finds
- Decision Boundary (Dashed Line): Perpendicular to the projection direction
- 1D Histograms: Shows how classes separate after projection
- Fisher’s Ratio: Higher values mean better separation!
Try this: - Increase “Class separation” → See Fisher’s ratio increase - Increase “Cluster spread” → See more overlap, Fisher’s ratio decreases - Change “Random seed” → See different random datasets
Demo 2: LDA vs Logistic Regression
Comparing Two Linear Classifiers
Both LDA and Logistic Regression create linear decision boundaries, but they find them differently: - LDA: Finds optimal projection direction (assumes Gaussian classes) - Logistic Regression: Directly optimizes decision boundary
Explore how they compare under different conditions!
def compare_lda_logreg(n_samples=200, class_sep=1.5, cluster_std=1.0,
noise_level=0.0, random_state=42):
"""
Compare LDA and Logistic Regression decision boundaries
Parameters:
- n_samples: Number of samples per class
- class_sep: Separation between classes
- cluster_std: Within-class standard deviation
- noise_level: Amount of label noise (0-0.3)
- random_state: Random seed
"""
# Generate data
X, y = make_classification(
n_samples=n_samples*2,
n_features=2,
n_redundant=0,
n_informative=2,
n_clusters_per_class=1,
class_sep=class_sep,
flip_y=noise_level,
random_state=random_state,
shuffle=True
)
X = X * cluster_std
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.3, random_state=random_state
)
# Fit models
lda = LinearDiscriminantAnalysis()
logreg = LogisticRegression(max_iter=1000)
lda.fit(X_train, y_train)
logreg.fit(X_train, y_train)
# Make predictions
lda_pred = lda.predict(X_test)
logreg_pred = logreg.predict(X_test)
# Calculate accuracies
lda_acc = accuracy_score(y_test, lda_pred)
logreg_acc = accuracy_score(y_test, logreg_pred)
# Create mesh for decision boundary
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200),
np.linspace(y_min, y_max, 200))
# Create figure
fig, axes = plt.subplots(1, 3, figsize=(18, 5))
# Plot 1: LDA
ax1 = axes[0]
Z_lda = lda.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
ax1.contourf(xx, yy, Z_lda, alpha=0.3, levels=[0, 0.5, 1], colors=['blue', 'red'])
ax1.contour(xx, yy, Z_lda, levels=[0.5], colors='black', linewidths=3, linestyles='-')
ax1.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1],
c='blue', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 0')
ax1.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1],
c='red', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 1')
ax1.scatter(X_test[y_test==0, 0], X_test[y_test==0, 1],
c='blue', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 0')
ax1.scatter(X_test[y_test==1, 0], X_test[y_test==1, 1],
c='red', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 1')
ax1.set_xlabel('Feature 1', fontsize=12)
ax1.set_ylabel('Feature 2', fontsize=12)
ax1.set_title(f'LDA\nAccuracy: {lda_acc:.3f}', fontsize=14, fontweight='bold')
ax1.legend(loc='best', fontsize=8)
ax1.grid(True, alpha=0.3)
ax1.set_xlim(x_min, x_max)
ax1.set_ylim(y_min, y_max)
# Plot 2: Logistic Regression
ax2 = axes[1]
Z_logreg = logreg.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
ax2.contourf(xx, yy, Z_logreg, alpha=0.3, levels=[0, 0.5, 1], colors=['blue', 'red'])
ax2.contour(xx, yy, Z_logreg, levels=[0.5], colors='black', linewidths=3, linestyles='-')
ax2.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1],
c='blue', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 0')
ax2.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1],
c='red', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 1')
ax2.scatter(X_test[y_test==0, 0], X_test[y_test==0, 1],
c='blue', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 0')
ax2.scatter(X_test[y_test==1, 0], X_test[y_test==1, 1],
c='red', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 1')
ax2.set_xlabel('Feature 1', fontsize=12)
ax2.set_ylabel('Feature 2', fontsize=12)
ax2.set_title(f'Logistic Regression\nAccuracy: {logreg_acc:.3f}', fontsize=14, fontweight='bold')
ax2.legend(loc='best', fontsize=8)
ax2.grid(True, alpha=0.3)
ax2.set_xlim(x_min, x_max)
ax2.set_ylim(y_min, y_max)
# Plot 3: Overlay both boundaries
ax3 = axes[2]
ax3.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1],
c='blue', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 0')
ax3.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1],
c='red', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 1')
# Draw both boundaries
ax3.contour(xx, yy, Z_lda, levels=[0.5], colors='blue', linewidths=3,
linestyles='-', label='LDA boundary')
ax3.contour(xx, yy, Z_logreg, levels=[0.5], colors='green', linewidths=3,
linestyles='--', label='LogReg boundary')
ax3.set_xlabel('Feature 1', fontsize=12)
ax3.set_ylabel('Feature 2', fontsize=12)
ax3.set_title('Both Boundaries Overlaid', fontsize=14, fontweight='bold')
ax3.legend(loc='best')
ax3.grid(True, alpha=0.3)
ax3.set_xlim(x_min, x_max)
ax3.set_ylim(y_min, y_max)
plt.tight_layout()
plt.show()
# Print comparison
print(f"\n📊 Model Comparison:")
print(f" • LDA Test Accuracy: {lda_acc:.4f}")
print(f" • Logistic Regression Test Accuracy: {logreg_acc:.4f}")
print(f" • Difference: {abs(lda_acc - logreg_acc):.4f}")
if abs(lda_acc - logreg_acc) < 0.02:
print(f"\n💡 Both methods perform similarly - boundaries are nearly identical!")
elif lda_acc > logreg_acc:
print(f"\n💡 LDA performs better - data likely follows Gaussian assumptions!")
else:
print(f"\n💡 LogReg performs better - fewer assumptions help with noisy data!")
# Create interactive widget
interactive_plot2 = interactive(
compare_lda_logreg,
n_samples=widgets.IntSlider(min=50, max=300, step=50, value=150,
description='Samples/class:', style={'description_width': '150px'}),
class_sep=widgets.FloatSlider(min=0.5, max=3.0, step=0.5, value=1.5,
description='Class separation:', style={'description_width': '150px'}),
cluster_std=widgets.FloatSlider(min=0.5, max=2.0, step=0.25, value=1.0,
description='Cluster spread:', style={'description_width': '150px'}),
noise_level=widgets.FloatSlider(min=0.0, max=0.3, step=0.05, value=0.0,
description='Label noise:', style={'description_width': '150px'}),
random_state=widgets.IntSlider(min=0, max=100, step=1, value=42,
description='Random seed:', style={'description_width': '150px'})
)
display(interactive_plot2)🎯 Key Observations from Demo 2:
- Similar Performance: For well-separated Gaussian data, both methods work similarly
- Different Boundaries: The decision boundaries are usually close but not identical
- Noise Sensitivity: Try increasing “Label noise” - see how each method handles it
Try this: - Set “Label noise” = 0.0 → Both methods agree - Increase “Label noise” → See which method is more robust - High “Cluster spread” + Low “Class separation” → More interesting differences
Demo 3: LDA vs QDA vs Logistic Regression
When Classes Have Different Spreads
This demo shows what happens when we violate LDA’s equal covariance assumption: - LDA: Forces linear boundary (may be suboptimal) - QDA: Allows quadratic (curved) boundary - Logistic Regression: Linear boundary (different from LDA)
See when QDA’s flexibility wins!
def compare_lda_qda_logreg(n_samples=150, mean_separation=3.0,
class0_cov_ratio=1.0, class1_cov_ratio=3.0,
rotation_angle=45, random_state=42):
"""
Compare LDA, QDA, and Logistic Regression with different covariances
Parameters:
- n_samples: Number of samples per class
- mean_separation: Distance between class centers
- class0_cov_ratio: Ratio of variances for class 0 (x-var / y-var)
- class1_cov_ratio: Ratio of variances for class 1 (x-var / y-var)
- rotation_angle: Rotation angle for covariances (degrees)
- random_state: Random seed
"""
np.random.seed(random_state)
# Create rotation matrix
theta = np.radians(rotation_angle)
rotation = np.array([[np.cos(theta), -np.sin(theta)],
[np.sin(theta), np.cos(theta)]])
# Create covariance matrices with different shapes
cov0_base = np.array([[class0_cov_ratio, 0], [0, 1.0]])
cov1_base = np.array([[class1_cov_ratio, 0], [0, 1.0]])
# Apply rotation
cov0 = rotation @ cov0_base @ rotation.T
cov1 = rotation @ cov1_base @ rotation.T
# Generate data with different covariances
mean0 = np.array([0, 0])
mean1 = np.array([mean_separation, 0])
X0 = np.random.multivariate_normal(mean0, cov0, n_samples)
X1 = np.random.multivariate_normal(mean1, cov1, n_samples)
X = np.vstack([X0, X1])
y = np.hstack([np.zeros(n_samples), np.ones(n_samples)])
# Shuffle
shuffle_idx = np.random.permutation(len(X))
X = X[shuffle_idx]
y = y[shuffle_idx]
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.3, random_state=random_state
)
# Fit models
lda = LinearDiscriminantAnalysis()
qda = QuadraticDiscriminantAnalysis()
logreg = LogisticRegression(max_iter=1000)
lda.fit(X_train, y_train)
qda.fit(X_train, y_train)
logreg.fit(X_train, y_train)
# Calculate accuracies
lda_acc = accuracy_score(y_test, lda.predict(X_test))
qda_acc = accuracy_score(y_test, qda.predict(X_test))
logreg_acc = accuracy_score(y_test, logreg.predict(X_test))
# Create mesh
x_min, x_max = X[:, 0].min() - 2, X[:, 0].max() + 2
y_min, y_max = X[:, 1].min() - 2, X[:, 1].max() + 2
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 300),
np.linspace(y_min, y_max, 300))
# Create figure
fig, axes = plt.subplots(2, 2, figsize=(16, 14))
models = [('LDA', lda, lda_acc), ('QDA', qda, qda_acc),
('Logistic Regression', logreg, logreg_acc)]
axes_flat = [axes[0, 0], axes[0, 1], axes[1, 0]]
for idx, (name, model, acc) in enumerate(models):
ax = axes_flat[idx]
# Predict on mesh
Z = model.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
# Plot decision regions
ax.contourf(xx, yy, Z, alpha=0.3, levels=[0, 0.5, 1], colors=['blue', 'red'])
# Plot decision boundary
ax.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=3)
# Plot training data
ax.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1],
c='blue', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 0')
ax.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1],
c='red', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 1')
# Plot test data
ax.scatter(X_test[y_test==0, 0], X_test[y_test==0, 1],
c='blue', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 0')
ax.scatter(X_test[y_test==1, 0], X_test[y_test==1, 1],
c='red', marker='s', s=80, edgecolors='yellow', linewidths=2, label='Test Class 1')
ax.set_xlabel('Feature 1', fontsize=12)
ax.set_ylabel('Feature 2', fontsize=12)
ax.set_title(f'{name}\nTest Accuracy: {acc:.3f}', fontsize=14, fontweight='bold')
ax.legend(loc='best', fontsize=9)
ax.grid(True, alpha=0.3)
ax.set_xlim(x_min, x_max)
ax.set_ylim(y_min, y_max)
# Fourth subplot: All boundaries together
ax4 = axes[1, 1]
# Plot data
ax4.scatter(X_train[y_train==0, 0], X_train[y_train==0, 1],
c='blue', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 0')
ax4.scatter(X_train[y_train==1, 0], X_train[y_train==1, 1],
c='red', marker='o', s=50, alpha=0.6, edgecolors='k', label='Train Class 1')
# Plot all boundaries
Z_lda = lda.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
Z_qda = qda.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
Z_logreg = logreg.predict(np.c_[xx.ravel(), yy.ravel()]).reshape(xx.shape)
ax4.contour(xx, yy, Z_lda, levels=[0.5], colors='blue', linewidths=3,
linestyles='-', label='LDA')
ax4.contour(xx, yy, Z_qda, levels=[0.5], colors='green', linewidths=3,
linestyles='-', label='QDA (curved!)')
ax4.contour(xx, yy, Z_logreg, levels=[0.5], colors='orange', linewidths=3,
linestyles='--', label='LogReg')
ax4.set_xlabel('Feature 1', fontsize=12)
ax4.set_ylabel('Feature 2', fontsize=12)
ax4.set_title('All Decision Boundaries Compared', fontsize=14, fontweight='bold')
ax4.legend(loc='best', fontsize=10)
ax4.grid(True, alpha=0.3)
ax4.set_xlim(x_min, x_max)
ax4.set_ylim(y_min, y_max)
plt.tight_layout()
plt.show()
# Print detailed comparison
print(f"\n📊 Model Performance Comparison:")
print(f" • LDA Test Accuracy: {lda_acc:.4f}")
print(f" • QDA Test Accuracy: {qda_acc:.4f}")
print(f" • Logistic Regression Test Accuracy: {logreg_acc:.4f}")
print(f"\n📐 Covariance Settings:")
print(f" • Class 0 covariance ratio: {class0_cov_ratio:.1f}")
print(f" • Class 1 covariance ratio: {class1_cov_ratio:.1f}")
print(f" • Difference in spreads: {abs(class1_cov_ratio - class0_cov_ratio):.1f}")
best_model = max([(lda_acc, 'LDA'), (qda_acc, 'QDA'), (logreg_acc, 'Logistic Regression')])
print(f"\n🏆 Best performing model: {best_model[1]} ({best_model[0]:.4f})")
if abs(class1_cov_ratio - class0_cov_ratio) > 1.5:
if qda_acc > max(lda_acc, logreg_acc) + 0.02:
print(f"\n💡 QDA wins! The classes have different covariances, so QDA's curved boundary helps!")
else:
print(f"\n💡 Despite different covariances, the classes may be too separated for flexibility to matter.")
else:
print(f"\n💡 Covariances are similar, so all methods perform comparably.")
# Create interactive widget
interactive_plot3 = interactive(
compare_lda_qda_logreg,
n_samples=widgets.IntSlider(min=50, max=250, step=50, value=150,
description='Samples/class:', style={'description_width': '150px'}),
mean_separation=widgets.FloatSlider(min=1.0, max=5.0, step=0.5, value=3.0,
description='Mean separation:', style={'description_width': '150px'}),
class0_cov_ratio=widgets.FloatSlider(min=0.5, max=4.0, step=0.5, value=1.0,
description='Class 0 cov ratio:', style={'description_width': '150px'}),
class1_cov_ratio=widgets.FloatSlider(min=0.5, max=4.0, step=0.5, value=3.0,
description='Class 1 cov ratio:', style={'description_width': '150px'}),
rotation_angle=widgets.IntSlider(min=0, max=90, step=15, value=45,
description='Rotation angle:', style={'description_width': '150px'}),
random_state=widgets.IntSlider(min=0, max=100, step=1, value=42,
description='Random seed:', style={'description_width': '150px'})
)
display(interactive_plot3)🎯 Key Observations from Demo 3:
- QDA’s Curved Boundary: Notice how QDA creates a curved decision boundary!
- When QDA Wins: QDA performs best when classes have very different spreads
- LDA’s Limitation: LDA forces a straight line even when a curve would be better
- Trade-off: QDA needs more data (more parameters to estimate)
Try this: - Set both “Class 0 cov ratio” = 1.0 and “Class 1 cov ratio” = 1.0 → All methods agree - Set “Class 1 cov ratio” = 4.0 (very different from Class 0) → QDA should win! - Reduce “Mean separation” → See when flexibility matters most - Change “Rotation angle” → See covariances oriented differently
🎓 Summary and Key Takeaways
What We Learned:
- LDA finds optimal projection:
- Maximizes between-class separation
- Minimizes within-class spread
- Reduces dimensionality automatically
- LDA vs Logistic Regression:
- Both create linear boundaries
- LDA assumes Gaussian classes
- Usually similar performance, but can differ
- QDA for flexibility:
- Allows different covariances per class
- Creates curved (quadratic) boundaries
- Needs more training data
- Best when classes have different shapes
Decision Guide:
Are classes well-separated and roughly Gaussian?
└─ Yes → Try LDA first (simple, efficient)
└─ No → Try Logistic Regression
Do classes have very different spreads/shapes?
└─ Yes, and you have lots of data → Try QDA
└─ No or limited data → Stick with LDA
Not sure?
└─ Try all three and compare using cross-validation!For Lab This Friday:
You’ll apply these methods to real datasets and practice: - Using sklearn’s LDA and QDA - Visualizing decision boundaries - Comparing model performance - Choosing the right method for your data
📚 Additional Exercises (Optional)
Try modifying the code to: 1. Add a third class and see how LDA handles multi-class problems 2. Create 3D data and project to 2D 3. Compare computational time: LDA vs QDA vs Logistic Regression 4. Implement a simple LDA from scratch using numpy 5. Visualize the confidence/probability contours instead of just boundaries
Questions? Ask in lab or email Dr. Xing!
End of Interactive Demonstrations