# Install required packages
!pip install -q ipywidgets
# Import libraries
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
import ipywidgets as widgets
from IPython.display import display, clear_output
import warnings
warnings.filterwarnings('ignore')
# For neural networks
from sklearn.datasets import make_circles, make_moons, make_blobs
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
# Keras/TensorFlow
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
from keras.regularizers import l2
from keras.optimizers import SGD, Adam, RMSprop
# Set random seeds for reproducibility
np.random.seed(42)
tf.random.set_seed(42)
print("β All libraries imported successfully!")
print(f"β TensorFlow version: {tf.__version__}")Week 10: Neural Networks - Interactive Demonstrations
MPS311/439 Machine Learning - Dr. Wei Xing
This notebook contains interactive demonstrations to help you understand neural networks.
Setup
Run this cell first to install and import required libraries.
Demo 1: XOR Decision Boundary Evolution π―
Purpose: Watch a neural network learn to solve the XOR problem in real-time!
What to observe: - How the decision boundary starts random and becomes non-linear - How loss decreases over epochs - Effect of different architectures (number of hidden units) - Effect of learning rate on convergence speed
Instructions: 1. Adjust the sliders to change network architecture and training parameters 2. Click βTrain Networkβ to watch it learn 3. Try different settings and observe the differences!
# XOR Decision Boundary Evolution
class XORVisualizer:
def __init__(self):
# XOR dataset
self.X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
self.y = np.array([0, 1, 1, 0])
self.model = None
self.history = None
def create_model(self, hidden_units, learning_rate):
"""Create a simple neural network for XOR"""
model = Sequential([
Dense(hidden_units, input_dim=2, activation='relu'),
Dense(1, activation='sigmoid')
])
optimizer = Adam(learning_rate=learning_rate)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
return model
def plot_decision_boundary(self, ax, model, epoch, loss):
"""Plot current decision boundary"""
# Create mesh
h = 0.02
x_min, x_max = -0.5, 1.5
y_min, y_max = -0.5, 1.5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
# Predict on mesh
Z = model.predict(np.c_[xx.ravel(), yy.ravel()], verbose=0)
Z = Z.reshape(xx.shape)
# Plot
ax.contourf(xx, yy, Z, levels=20, cmap='RdBu', alpha=0.6)
ax.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=2, linestyles='--')
# Plot XOR points
colors = ['blue' if label == 0 else 'red' for label in self.y]
markers = ['o' if label == 0 else 'x' for label in self.y]
for i, (x, y, c, m) in enumerate(zip(self.X[:, 0], self.X[:, 1], colors, markers)):
ax.scatter(x, y, c=c, marker=m, s=200, edgecolors='black', linewidths=2, zorder=3)
ax.set_xlim(x_min, x_max)
ax.set_ylim(y_min, y_max)
ax.set_xlabel('$x_1$', fontsize=12)
ax.set_ylabel('$x_2$', fontsize=12)
ax.set_title(f'Epoch {epoch} | Loss: {loss:.4f}', fontsize=14, fontweight='bold')
ax.grid(True, alpha=0.3)
def train_and_visualize(self, hidden_units, learning_rate, epochs, update_freq):
"""Train model and show evolution"""
self.model = self.create_model(hidden_units, learning_rate)
# Prepare figure
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
losses = []
# Training loop with visualization
for epoch in range(0, epochs + 1, update_freq):
if epoch > 0:
self.model.fit(self.X, self.y, epochs=update_freq, verbose=0)
# Get current loss
loss = self.model.evaluate(self.X, self.y, verbose=0)[0]
losses.append(loss)
# Clear and update plots
ax1.clear()
ax2.clear()
# Plot decision boundary
self.plot_decision_boundary(ax1, self.model, epoch, loss)
# Plot loss curve
ax2.plot(range(0, epoch + 1, update_freq), losses, 'b-', linewidth=2)
ax2.set_xlabel('Epoch', fontsize=12)
ax2.set_ylabel('Loss', fontsize=12)
ax2.set_title('Training Loss', fontsize=14, fontweight='bold')
ax2.grid(True, alpha=0.3)
ax2.set_xlim(0, epochs)
plt.tight_layout()
display(fig)
clear_output(wait=True)
plt.close()
# Final predictions
predictions = self.model.predict(self.X, verbose=0)
print("\n" + "="*60)
print("FINAL RESULTS")
print("="*60)
print(f"Final Loss: {losses[-1]:.4f}\n")
print("Predictions:")
for i, (x, true_y, pred_y) in enumerate(zip(self.X, self.y, predictions)):
print(f" Input {x} β Predicted: {pred_y[0]:.4f}, True: {true_y}")
# Create visualizer
xor_viz = XORVisualizer()
# Interactive widgets
hidden_units_slider = widgets.IntSlider(
value=4, min=2, max=16, step=1,
description='Hidden Units:',
style={'description_width': '150px'},
layout=widgets.Layout(width='500px')
)
learning_rate_slider = widgets.FloatLogSlider(
value=0.1, base=10, min=-3, max=0, step=0.1,
description='Learning Rate:',
style={'description_width': '150px'},
layout=widgets.Layout(width='500px'),
readout_format='.4f'
)
epochs_slider = widgets.IntSlider(
value=500, min=100, max=2000, step=100,
description='Epochs:',
style={'description_width': '150px'},
layout=widgets.Layout(width='500px')
)
update_freq_slider = widgets.IntSlider(
value=50, min=10, max=200, step=10,
description='Update Every:',
style={'description_width': '150px'},
layout=widgets.Layout(width='500px')
)
train_button = widgets.Button(
description='π Train Network',
button_style='success',
layout=widgets.Layout(width='200px', height='40px')
)
def on_train_clicked(b):
print("Training started... Watch the decision boundary evolve!\n")
xor_viz.train_and_visualize(
hidden_units=hidden_units_slider.value,
learning_rate=learning_rate_slider.value,
epochs=epochs_slider.value,
update_freq=update_freq_slider.value
)
train_button.on_click(on_train_clicked)
# Display widgets
print("π― XOR DECISION BOUNDARY EVOLUTION")
print("="*60)
print("Adjust parameters and click 'Train Network' to see learning in action!\n")
display(widgets.VBox([
widgets.HTML("<h3>Network Architecture</h3>"),
hidden_units_slider,
widgets.HTML("<h3>Training Parameters</h3>"),
learning_rate_slider,
epochs_slider,
update_freq_slider,
widgets.HTML("<br>"),
train_button
]))Demo 2: Architecture Explorer π
Purpose: Experiment with different network architectures and see their effects!
What to observe: - How network depth (number of layers) affects performance - How network width (units per layer) affects capacity - Effect on training time and accuracy - Overfitting indicators (train-test gap)
Instructions: 1. Choose a dataset (different complexity levels) 2. Adjust architecture parameters 3. Click βTrain Modelβ to see performance 4. Click βCompareβ to add to comparison plot 5. Try multiple configurations and compare!
# Architecture Explorer
class ArchitectureExplorer:
def __init__(self):
self.comparison_results = []
self.comparison_colors = ['blue', 'red', 'green', 'orange', 'purple', 'brown']
def generate_dataset(self, dataset_name, n_samples=1000):
"""Generate different datasets"""
if dataset_name == 'XOR':
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([0, 1, 1, 0])
# Repeat to have more samples
X = np.tile(X, (250, 1)) + np.random.normal(0, 0.1, (1000, 2))
y = np.tile(y, 250)
elif dataset_name == 'Circles':
X, y = make_circles(n_samples=n_samples, noise=0.1, factor=0.5, random_state=42)
elif dataset_name == 'Moons':
X, y = make_moons(n_samples=n_samples, noise=0.15, random_state=42)
else: # Blobs
X, y = make_blobs(n_samples=n_samples, centers=2, n_features=2, random_state=42)
return X, y
def build_model(self, n_layers, units_per_layer, activation, input_dim=2):
"""Build model with specified architecture"""
model = Sequential()
# First hidden layer
model.add(Dense(units_per_layer, input_dim=input_dim, activation=activation))
# Additional hidden layers
for _ in range(n_layers - 1):
model.add(Dense(units_per_layer, activation=activation))
# Output layer
model.add(Dense(1, activation='sigmoid'))
return model
def train_and_evaluate(self, dataset_name, n_layers, units_per_layer,
activation, epochs, optimizer_name):
"""Train model and return results"""
# Generate dataset
X, y = self.generate_dataset(dataset_name)
# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Scale features
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
# Build model
model = self.build_model(n_layers, units_per_layer, activation)
# Select optimizer
if optimizer_name == 'SGD':
optimizer = SGD(learning_rate=0.01)
elif optimizer_name == 'Adam':
optimizer = Adam(learning_rate=0.001)
else: # RMSprop
optimizer = RMSprop(learning_rate=0.001)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
# Train
import time
start_time = time.time()
history = model.fit(X_train, y_train, epochs=epochs, batch_size=32,
validation_data=(X_test, y_test), verbose=0)
train_time = time.time() - start_time
# Evaluate
train_loss, train_acc = model.evaluate(X_train, y_train, verbose=0)
test_loss, test_acc = model.evaluate(X_test, y_test, verbose=0)
# Count parameters
n_params = model.count_params()
return {
'model': model,
'history': history,
'train_acc': train_acc,
'test_acc': test_acc,
'train_loss': train_loss,
'test_loss': test_loss,
'train_time': train_time,
'n_params': n_params,
'config': f"{n_layers}L-{units_per_layer}U-{activation}",
'scaler': scaler,
'X_test': X_test,
'y_test': y_test
}
def visualize_results(self, results):
"""Visualize training results"""
fig = plt.figure(figsize=(16, 10))
# Create grid
gs = fig.add_gridspec(3, 3, hspace=0.3, wspace=0.3)
# Training curves
ax1 = fig.add_subplot(gs[0, :])
ax1.plot(results['history'].history['loss'], label='Training Loss', linewidth=2)
ax1.plot(results['history'].history['val_loss'], label='Validation Loss', linewidth=2)
ax1.set_xlabel('Epoch', fontsize=11)
ax1.set_ylabel('Loss', fontsize=11)
ax1.set_title('Training and Validation Loss', fontsize=13, fontweight='bold')
ax1.legend()
ax1.grid(True, alpha=0.3)
# Accuracy curves
ax2 = fig.add_subplot(gs[1, :])
ax2.plot(results['history'].history['accuracy'], label='Training Accuracy', linewidth=2)
ax2.plot(results['history'].history['val_accuracy'], label='Validation Accuracy', linewidth=2)
ax2.set_xlabel('Epoch', fontsize=11)
ax2.set_ylabel('Accuracy', fontsize=11)
ax2.set_title('Training and Validation Accuracy', fontsize=13, fontweight='bold')
ax2.legend()
ax2.grid(True, alpha=0.3)
# Metrics
ax3 = fig.add_subplot(gs[2, 0])
metrics = ['Train Acc', 'Test Acc']
values = [results['train_acc'] * 100, results['test_acc'] * 100]
colors = ['lightblue', 'lightcoral']
bars = ax3.bar(metrics, values, color=colors, edgecolor='black', linewidth=1.5)
ax3.set_ylabel('Accuracy (%)', fontsize=11)
ax3.set_title('Final Accuracy', fontsize=12, fontweight='bold')
ax3.set_ylim(0, 105)
for bar, val in zip(bars, values):
height = bar.get_height()
ax3.text(bar.get_x() + bar.get_width()/2., height + 1,
f'{val:.1f}%', ha='center', va='bottom', fontweight='bold')
# Training time and parameters
ax4 = fig.add_subplot(gs[2, 1])
ax4.axis('off')
info_text = f"""
Configuration: {results['config']}
Training Time: {results['train_time']:.2f}s
Parameters: {results['n_params']:,}
Final Train Loss: {results['train_loss']:.4f}
Final Test Loss: {results['test_loss']:.4f}
"""
ax4.text(0.1, 0.5, info_text, fontsize=11, verticalalignment='center',
bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))
# Overfitting indicator
ax5 = fig.add_subplot(gs[2, 2])
ax5.axis('off')
gap = results['train_acc'] - results['test_acc']
if gap < 0.05:
status = "π’ Good Generalization"
color = 'lightgreen'
elif gap < 0.15:
status = "π‘ Slight Overfitting"
color = 'lightyellow'
else:
status = "π΄ Overfitting!"
color = 'lightcoral'
ax5.text(0.5, 0.5, f"{status}\n\nTrain-Test Gap:\n{gap*100:.1f}%",
fontsize=13, ha='center', va='center', fontweight='bold',
bbox=dict(boxstyle='round', facecolor=color, alpha=0.8, pad=1))
plt.tight_layout()
plt.show()
print("\n" + "="*70)
print("SUMMARY")
print("="*70)
print(f"Configuration: {results['config']}")
print(f"Test Accuracy: {results['test_acc']*100:.2f}%")
print(f"Training Time: {results['train_time']:.2f}s")
print(f"Parameters: {results['n_params']:,}")
if gap >= 0.15:
print("\nβ οΈ WARNING: Model is overfitting! Consider:")
print(" - Reducing model complexity")
print(" - Adding regularization")
print(" - Getting more training data")
print("="*70)
def add_to_comparison(self, results):
"""Add results to comparison list"""
self.comparison_results.append(results)
print(f"\nβ Added to comparison ({len(self.comparison_results)} models total)")
def plot_comparison(self):
"""Plot comparison of multiple models"""
if len(self.comparison_results) == 0:
print("No models to compare yet! Train and add some models first.")
return
fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(16, 5))
# Loss comparison
for i, res in enumerate(self.comparison_results):
color = self.comparison_colors[i % len(self.comparison_colors)]
ax1.plot(res['history'].history['val_loss'],
label=res['config'], color=color, linewidth=2)
ax1.set_xlabel('Epoch', fontsize=11)
ax1.set_ylabel('Validation Loss', fontsize=11)
ax1.set_title('Validation Loss Comparison', fontsize=13, fontweight='bold')
ax1.legend()
ax1.grid(True, alpha=0.3)
# Accuracy comparison
configs = [res['config'] for res in self.comparison_results]
test_accs = [res['test_acc'] * 100 for res in self.comparison_results]
colors = [self.comparison_colors[i % len(self.comparison_colors)]
for i in range(len(configs))]
bars = ax2.bar(range(len(configs)), test_accs, color=colors,
edgecolor='black', linewidth=1.5)
ax2.set_xticks(range(len(configs)))
ax2.set_xticklabels(configs, rotation=45, ha='right')
ax2.set_ylabel('Test Accuracy (%)', fontsize=11)
ax2.set_title('Test Accuracy Comparison', fontsize=13, fontweight='bold')
ax2.set_ylim(0, 105)
for bar, val in zip(bars, test_accs):
height = bar.get_height()
ax2.text(bar.get_x() + bar.get_width()/2., height + 1,
f'{val:.1f}%', ha='center', va='bottom', fontsize=9)
# Parameters vs Accuracy
params = [res['n_params'] for res in self.comparison_results]
ax3.scatter(params, test_accs, c=colors, s=200, edgecolors='black', linewidths=2)
for i, config in enumerate(configs):
ax3.annotate(config, (params[i], test_accs[i]),
xytext=(5, 5), textcoords='offset points', fontsize=8)
ax3.set_xlabel('Number of Parameters', fontsize=11)
ax3.set_ylabel('Test Accuracy (%)', fontsize=11)
ax3.set_title('Parameters vs Accuracy', fontsize=13, fontweight='bold')
ax3.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print("\n" + "="*70)
print("COMPARISON SUMMARY")
print("="*70)
for i, res in enumerate(self.comparison_results):
print(f"{i+1}. {res['config']}: {res['test_acc']*100:.2f}% accuracy, "
f"{res['n_params']:,} params, {res['train_time']:.2f}s")
print("="*70)
def clear_comparison(self):
"""Clear comparison results"""
self.comparison_results = []
print("β Comparison cleared!")
# Create explorer
explorer = ArchitectureExplorer()
# Widgets
dataset_dropdown = widgets.Dropdown(
options=['XOR', 'Circles', 'Moons', 'Blobs'],
value='Moons',
description='Dataset:',
style={'description_width': '120px'},
layout=widgets.Layout(width='350px')
)
layers_buttons = widgets.ToggleButtons(
options=[1, 2, 3],
value=1,
description='Hidden Layers:',
style={'description_width': '120px'},
button_style='info'
)
units_slider = widgets.IntSlider(
value=8, min=2, max=64, step=2,
description='Units/Layer:',
style={'description_width': '120px'},
layout=widgets.Layout(width='450px')
)
activation_dropdown = widgets.Dropdown(
options=['relu', 'sigmoid', 'tanh'],
value='relu',
description='Activation:',
style={'description_width': '120px'},
layout=widgets.Layout(width='350px')
)
epochs_slider_arch = widgets.IntSlider(
value=100, min=50, max=500, step=50,
description='Epochs:',
style={'description_width': '120px'},
layout=widgets.Layout(width='450px')
)
optimizer_dropdown = widgets.Dropdown(
options=['Adam', 'SGD', 'RMSprop'],
value='Adam',
description='Optimizer:',
style={'description_width': '120px'},
layout=widgets.Layout(width='350px')
)
train_button_arch = widgets.Button(
description='π Train Model',
button_style='success',
layout=widgets.Layout(width='150px', height='35px')
)
compare_button = widgets.Button(
description='π Add to Compare',
button_style='info',
layout=widgets.Layout(width='150px', height='35px')
)
show_comparison_button = widgets.Button(
description='π Show Comparison',
button_style='warning',
layout=widgets.Layout(width='150px', height='35px')
)
clear_button = widgets.Button(
description='ποΈ Clear',
button_style='danger',
layout=widgets.Layout(width='100px', height='35px')
)
current_results = {'data': None}
def on_train_arch_clicked(b):
print("Training model...\n")
results = explorer.train_and_evaluate(
dataset_name=dataset_dropdown.value,
n_layers=layers_buttons.value,
units_per_layer=units_slider.value,
activation=activation_dropdown.value,
epochs=epochs_slider_arch.value,
optimizer_name=optimizer_dropdown.value
)
current_results['data'] = results
explorer.visualize_results(results)
def on_compare_clicked(b):
if current_results['data'] is not None:
explorer.add_to_comparison(current_results['data'])
else:
print("Train a model first!")
def on_show_comparison_clicked(b):
explorer.plot_comparison()
def on_clear_clicked(b):
explorer.clear_comparison()
train_button_arch.on_click(on_train_arch_clicked)
compare_button.on_click(on_compare_clicked)
show_comparison_button.on_click(on_show_comparison_clicked)
clear_button.on_click(on_clear_clicked)
# Display
print("π ARCHITECTURE EXPLORER")
print("="*70)
print("Experiment with different architectures and compare their performance!\n")
display(widgets.VBox([
widgets.HTML("<h3>Dataset Selection</h3>"),
dataset_dropdown,
widgets.HTML("<h3>Network Architecture</h3>"),
layers_buttons,
units_slider,
activation_dropdown,
widgets.HTML("<h3>Training Parameters</h3>"),
epochs_slider_arch,
optimizer_dropdown,
widgets.HTML("<br>"),
widgets.HBox([train_button_arch, compare_button, show_comparison_button, clear_button])
]))Demo 3: Activation Function Comparison π§
Purpose: Compare different activation functions and understand their characteristics!
What to observe: - Shape of different activation functions (sigmoid, ReLU, tanh) - Their derivatives (important for gradient flow) - How they affect training speed and final performance - Which works best for different datasets
Instructions: 1. Select a dataset 2. Configure the network 3. Click βTrain Allβ to train with all three activations 4. Compare the results!
# Activation Function Comparison
class ActivationComparison:
def __init__(self):
self.activations = ['sigmoid', 'relu', 'tanh']
self.colors = {'sigmoid': 'blue', 'relu': 'red', 'tanh': 'green'}
def plot_activation_functions(self):
"""Plot activation functions and their derivatives"""
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
x = np.linspace(-5, 5, 200)
# Sigmoid
sigmoid = 1 / (1 + np.exp(-x))
sigmoid_deriv = sigmoid * (1 - sigmoid)
axes[0].plot(x, sigmoid, 'b-', linewidth=2.5, label='Sigmoid')
axes[0].plot(x, sigmoid_deriv, 'b--', linewidth=2, alpha=0.7, label="Sigmoid'")
axes[0].set_title('Sigmoid', fontsize=13, fontweight='bold')
axes[0].legend()
axes[0].grid(True, alpha=0.3)
axes[0].axhline(y=0, color='k', linewidth=0.5)
axes[0].axvline(x=0, color='k', linewidth=0.5)
# ReLU
relu = np.maximum(0, x)
relu_deriv = (x > 0).astype(float)
axes[1].plot(x, relu, 'r-', linewidth=2.5, label='ReLU')
axes[1].plot(x, relu_deriv, 'r--', linewidth=2, alpha=0.7, label="ReLU'")
axes[1].set_title('ReLU', fontsize=13, fontweight='bold')
axes[1].legend()
axes[1].grid(True, alpha=0.3)
axes[1].axhline(y=0, color='k', linewidth=0.5)
axes[1].axvline(x=0, color='k', linewidth=0.5)
# Tanh
tanh = np.tanh(x)
tanh_deriv = 1 - tanh**2
axes[2].plot(x, tanh, 'g-', linewidth=2.5, label='Tanh')
axes[2].plot(x, tanh_deriv, 'g--', linewidth=2, alpha=0.7, label="Tanh'")
axes[2].set_title('Tanh', fontsize=13, fontweight='bold')
axes[2].legend()
axes[2].grid(True, alpha=0.3)
axes[2].axhline(y=0, color='k', linewidth=0.5)
axes[2].axvline(x=0, color='k', linewidth=0.5)
for ax in axes:
ax.set_xlabel('Input (z)', fontsize=11)
ax.set_ylabel('Output', fontsize=11)
plt.tight_layout()
plt.show()
def train_all_activations(self, dataset_name, n_layers, units, epochs):
"""Train networks with all three activations"""
results = {}
# Generate dataset
if dataset_name == 'Circles':
X, y = make_circles(n_samples=1000, noise=0.1, factor=0.5, random_state=42)
elif dataset_name == 'Moons':
X, y = make_moons(n_samples=1000, noise=0.15, random_state=42)
elif dataset_name == 'Spirals':
# Create spiral dataset
n = 500
theta = np.sqrt(np.random.rand(n)) * 2 * np.pi
r_a = 2 * theta + np.pi
data_a = np.array([np.cos(theta) * r_a, np.sin(theta) * r_a]).T
x_a = data_a + np.random.randn(n, 2) * 0.5
r_b = -2 * theta - np.pi
data_b = np.array([np.cos(theta) * r_b, np.sin(theta) * r_b]).T
x_b = data_b + np.random.randn(n, 2) * 0.5
X = np.vstack([x_a, x_b])
y = np.hstack([np.zeros(n), np.ones(n)])
else: # XOR
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([0, 1, 1, 0])
X = np.tile(X, (250, 1)) + np.random.normal(0, 0.1, (1000, 2))
y = np.tile(y, 250)
# Split and scale
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
print("Training networks with different activations...\n")
for activation in self.activations:
print(f"Training with {activation}...")
# Build model
model = Sequential()
model.add(Dense(units, input_dim=2, activation=activation))
for _ in range(n_layers - 1):
model.add(Dense(units, activation=activation))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
# Train
history = model.fit(X_train, y_train, epochs=epochs, batch_size=32,
validation_data=(X_test, y_test), verbose=0)
# Evaluate
_, test_acc = model.evaluate(X_test, y_test, verbose=0)
results[activation] = {
'model': model,
'history': history,
'test_acc': test_acc,
'X_test': X_test,
'y_test': y_test
}
return results, scaler
def visualize_comparison(self, results):
"""Visualize comparison of all three activations"""
fig = plt.figure(figsize=(16, 10))
gs = fig.add_gridspec(2, 3, hspace=0.3, wspace=0.3)
# Training loss comparison
ax1 = fig.add_subplot(gs[0, :])
for activation in self.activations:
ax1.plot(results[activation]['history'].history['loss'],
label=f'{activation.capitalize()}',
color=self.colors[activation], linewidth=2)
ax1.set_xlabel('Epoch', fontsize=12)
ax1.set_ylabel('Training Loss', fontsize=12)
ax1.set_title('Training Loss Comparison', fontsize=14, fontweight='bold')
ax1.legend(fontsize=11)
ax1.grid(True, alpha=0.3)
# Decision boundaries
for idx, activation in enumerate(self.activations):
ax = fig.add_subplot(gs[1, idx])
# Create mesh
X_test = results[activation]['X_test']
x_min, x_max = X_test[:, 0].min() - 0.5, X_test[:, 0].max() + 0.5
y_min, y_max = X_test[:, 1].min() - 0.5, X_test[:, 1].max() + 0.5
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100),
np.linspace(y_min, y_max, 100))
# Predict
Z = results[activation]['model'].predict(np.c_[xx.ravel(), yy.ravel()], verbose=0)
Z = Z.reshape(xx.shape)
# Plot
ax.contourf(xx, yy, Z, levels=20, cmap='RdBu', alpha=0.6)
ax.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=2)
# Plot points
y_test = results[activation]['y_test']
ax.scatter(X_test[y_test == 0, 0], X_test[y_test == 0, 1],
c='blue', marker='o', s=50, edgecolors='black', linewidths=1)
ax.scatter(X_test[y_test == 1, 0], X_test[y_test == 1, 1],
c='red', marker='x', s=50, linewidths=2)
test_acc = results[activation]['test_acc']
ax.set_title(f'{activation.capitalize()}\nAccuracy: {test_acc*100:.1f}%',
fontsize=12, fontweight='bold')
ax.set_xlabel('Feature 1', fontsize=10)
ax.set_ylabel('Feature 2', fontsize=10)
plt.tight_layout()
plt.show()
print("\n" + "="*70)
print("COMPARISON SUMMARY")
print("="*70)
for activation in self.activations:
acc = results[activation]['test_acc']
final_loss = results[activation]['history'].history['loss'][-1]
print(f"{activation.capitalize():8s}: Test Acc = {acc*100:5.2f}%, Final Loss = {final_loss:.4f}")
print("="*70)
print("\nπ‘ Observations:")
print(" - ReLU often trains fastest (no saturation)")
print(" - Sigmoid can suffer from vanishing gradients")
print(" - Tanh is zero-centered (sometimes helps)")
print("="*70)
# Create comparison object
activation_comp = ActivationComparison()
# First, show activation functions
print("π§ ACTIVATION FUNCTION COMPARISON")
print("="*70)
print("First, let's visualize the activation functions and their derivatives:\n")
activation_comp.plot_activation_functions()
# Widgets
dataset_act = widgets.Dropdown(
options=['XOR', 'Circles', 'Moons', 'Spirals'],
value='Moons',
description='Dataset:',
style={'description_width': '120px'},
layout=widgets.Layout(width='350px')
)
layers_act = widgets.IntSlider(
value=2, min=1, max=3, step=1,
description='Hidden Layers:',
style={'description_width': '120px'},
layout=widgets.Layout(width='400px')
)
units_act = widgets.IntSlider(
value=8, min=4, max=32, step=4,
description='Units/Layer:',
style={'description_width': '120px'},
layout=widgets.Layout(width='400px')
)
epochs_act = widgets.IntSlider(
value=100, min=50, max=300, step=50,
description='Epochs:',
style={'description_width': '120px'},
layout=widgets.Layout(width='400px')
)
train_all_button = widgets.Button(
description='π Train All Activations',
button_style='success',
layout=widgets.Layout(width='200px', height='40px')
)
def on_train_all_clicked(b):
results, scaler = activation_comp.train_all_activations(
dataset_name=dataset_act.value,
n_layers=layers_act.value,
units=units_act.value,
epochs=epochs_act.value
)
activation_comp.visualize_comparison(results)
train_all_button.on_click(on_train_all_clicked)
print("\n" + "="*70)
print("Now configure and train networks with all three activations:")
print("="*70 + "\n")
display(widgets.VBox([
widgets.HTML("<h3>Configuration</h3>"),
dataset_act,
layers_act,
units_act,
epochs_act,
widgets.HTML("<br>"),
train_all_button
]))Demo 4: Training Dynamics Visualizer π
Purpose: Monitor training in real-time and understand overfitting!
What to observe: - How training and validation loss evolve - When overfitting begins (validation loss increases) - Effect of regularization (L2, Dropout) on overfitting - Impact of learning rate on convergence - Weight distribution during training
Instructions: 1. Adjust dataset size and noise 2. Configure regularization (try with and without) 3. Click βStart Trainingβ and watch in real-time 4. Observe overfitting indicators!
# Training Dynamics Visualizer
class TrainingDynamicsVisualizer:
def __init__(self):
self.training_active = False
def create_model(self, use_l2, l2_lambda, use_dropout, dropout_rate, use_batchnorm):
"""Create model with optional regularization"""
model = Sequential()
# First layer
if use_l2:
model.add(Dense(32, input_dim=2, activation='relu', kernel_regularizer=l2(l2_lambda)))
else:
model.add(Dense(32, input_dim=2, activation='relu'))
if use_batchnorm:
model.add(BatchNormalization())
if use_dropout:
model.add(Dropout(dropout_rate))
# Second layer
if use_l2:
model.add(Dense(16, activation='relu', kernel_regularizer=l2(l2_lambda)))
else:
model.add(Dense(16, activation='relu'))
if use_batchnorm:
model.add(BatchNormalization())
if use_dropout:
model.add(Dropout(dropout_rate))
# Output layer
model.add(Dense(1, activation='sigmoid'))
return model
def train_with_visualization(self, train_size, noise_level, learning_rate,
use_l2, l2_lambda, use_dropout, dropout_rate,
use_batchnorm, max_epochs, update_freq):
"""Train model with real-time visualization"""
# Generate dataset
X, y = make_moons(n_samples=train_size, noise=noise_level, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42)
# Scale
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_val = scaler.transform(X_val)
# Create model
model = self.create_model(use_l2, l2_lambda, use_dropout, dropout_rate, use_batchnorm)
optimizer = Adam(learning_rate=learning_rate)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
# Prepare figure
fig = plt.figure(figsize=(16, 10))
gs = fig.add_gridspec(3, 2, hspace=0.3, wspace=0.3)
ax1 = fig.add_subplot(gs[0, :])
ax2 = fig.add_subplot(gs[1, :])
ax3 = fig.add_subplot(gs[2, 0])
ax4 = fig.add_subplot(gs[2, 1])
train_losses = []
val_losses = []
train_accs = []
val_accs = []
epochs_list = []
overfitting_detected = False
best_val_loss = float('inf')
patience_counter = 0
print("Training started... Watch the dynamics!\n")
for epoch in range(0, max_epochs + 1, update_freq):
if epoch > 0:
model.fit(X_train, y_train, epochs=update_freq, batch_size=32, verbose=0)
# Evaluate
train_loss, train_acc = model.evaluate(X_train, y_train, verbose=0)
val_loss, val_acc = model.evaluate(X_val, y_val, verbose=0)
train_losses.append(train_loss)
val_losses.append(val_loss)
train_accs.append(train_acc)
val_accs.append(val_acc)
epochs_list.append(epoch)
# Check for overfitting
if val_loss > best_val_loss:
patience_counter += 1
if patience_counter >= 3 and not overfitting_detected:
overfitting_detected = True
else:
best_val_loss = val_loss
patience_counter = 0
# Clear axes
ax1.clear()
ax2.clear()
ax3.clear()
ax4.clear()
# Plot 1: Loss curves
ax1.plot(epochs_list, train_losses, 'b-', linewidth=2, label='Training Loss')
ax1.plot(epochs_list, val_losses, 'orange', linewidth=2, label='Validation Loss')
if overfitting_detected and len(epochs_list) > 3:
overfit_idx = len(epochs_list) - patience_counter - 1
ax1.axvspan(epochs_list[overfit_idx], epochs_list[-1],
alpha=0.2, color='red', label='Overfitting Zone')
ax1.set_xlabel('Epoch', fontsize=11)
ax1.set_ylabel('Loss', fontsize=11)
ax1.set_title('Training and Validation Loss', fontsize=13, fontweight='bold')
ax1.legend()
ax1.grid(True, alpha=0.3)
# Plot 2: Accuracy curves
ax2.plot(epochs_list, train_accs, 'b-', linewidth=2, label='Training Accuracy')
ax2.plot(epochs_list, val_accs, 'orange', linewidth=2, label='Validation Accuracy')
ax2.set_xlabel('Epoch', fontsize=11)
ax2.set_ylabel('Accuracy', fontsize=11)
ax2.set_title('Training and Validation Accuracy', fontsize=13, fontweight='bold')
ax2.legend()
ax2.grid(True, alpha=0.3)
# Plot 3: Weight distribution
all_weights = []
for layer in model.layers:
if isinstance(layer, Dense):
all_weights.extend(layer.get_weights()[0].flatten())
ax3.hist(all_weights, bins=50, color='steelblue', edgecolor='black', alpha=0.7)
ax3.set_xlabel('Weight Value', fontsize=11)
ax3.set_ylabel('Count', fontsize=11)
ax3.set_title('Weight Distribution', fontsize=13, fontweight='bold')
ax3.axvline(x=0, color='red', linestyle='--', linewidth=2)
ax3.grid(True, alpha=0.3, axis='y')
# Plot 4: Status panel
ax4.axis('off')
# Determine status
if overfitting_detected:
status = "π΄ Overfitting Detected!"
status_color = 'lightcoral'
elif val_loss < train_loss * 1.1:
status = "π’ Training Normally"
status_color = 'lightgreen'
else:
status = "π‘ Possible Overfitting"
status_color = 'lightyellow'
# Weight health
max_weight = np.max(np.abs(all_weights))
if max_weight > 10:
weight_status = "π΄ Weights Exploding!"
elif max_weight < 0.01:
weight_status = "π΄ Weights Vanishing!"
else:
weight_status = "π’ Weights Healthy"
status_text = f"""
EPOCH: {epoch} / {max_epochs}
Current Learning Rate: {learning_rate:.4f}
{status}
{weight_status}
Train Loss: {train_loss:.4f}
Val Loss: {val_loss:.4f}
Gap: {abs(train_loss - val_loss):.4f}
Train Acc: {train_acc*100:.1f}%
Val Acc: {val_acc*100:.1f}%
"""
ax4.text(0.5, 0.5, status_text, fontsize=11, ha='center', va='center',
bbox=dict(boxstyle='round', facecolor=status_color, alpha=0.8, pad=1.5))
plt.tight_layout()
display(fig)
clear_output(wait=True)
plt.close()
# Final summary
print("\n" + "="*70)
print("TRAINING COMPLETE")
print("="*70)
print(f"Final Training Loss: {train_losses[-1]:.4f}")
print(f"Final Validation Loss: {val_losses[-1]:.4f}")
print(f"Final Training Accuracy: {train_accs[-1]*100:.2f}%")
print(f"Final Validation Accuracy: {val_accs[-1]*100:.2f}%")
print(f"\nTrain-Val Gap: {abs(train_accs[-1] - val_accs[-1])*100:.2f}%")
if overfitting_detected:
print("\nβ οΈ OVERFITTING DETECTED!")
print("\nSuggestions:")
if not use_l2:
print(" - Try enabling L2 regularization")
if not use_dropout:
print(" - Try enabling dropout")
if train_size < 1000:
print(" - Consider increasing training data size")
else:
print("\nβ Model is generalizing well!")
print("="*70)
# Create visualizer
dynamics_viz = TrainingDynamicsVisualizer()
# Widgets
train_size_slider = widgets.IntSlider(
value=500, min=100, max=2000, step=100,
description='Train Size:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
noise_slider = widgets.FloatSlider(
value=0.2, min=0.0, max=0.5, step=0.05,
description='Noise Level:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
lr_slider_dyn = widgets.FloatLogSlider(
value=0.001, base=10, min=-4, max=-1, step=0.1,
description='Learning Rate:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px'),
readout_format='.4f'
)
use_l2_check = widgets.Checkbox(
value=False,
description='Use L2 Regularization',
style={'description_width': '200px'}
)
l2_slider = widgets.FloatSlider(
value=0.01, min=0.0, max=0.1, step=0.01,
description='L2 Lambda:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
use_dropout_check = widgets.Checkbox(
value=False,
description='Use Dropout',
style={'description_width': '200px'}
)
dropout_slider = widgets.FloatSlider(
value=0.3, min=0.0, max=0.7, step=0.1,
description='Dropout Rate:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
use_batchnorm_check = widgets.Checkbox(
value=False,
description='Use Batch Normalization',
style={'description_width': '200px'}
)
epochs_slider_dyn = widgets.IntSlider(
value=200, min=50, max=500, step=50,
description='Max Epochs:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
update_freq_dyn = widgets.IntSlider(
value=10, min=5, max=50, step=5,
description='Update Every:',
style={'description_width': '150px'},
layout=widgets.Layout(width='450px')
)
start_button = widgets.Button(
description='π Start Training',
button_style='success',
layout=widgets.Layout(width='200px', height='40px')
)
def on_start_clicked(b):
dynamics_viz.train_with_visualization(
train_size=train_size_slider.value,
noise_level=noise_slider.value,
learning_rate=lr_slider_dyn.value,
use_l2=use_l2_check.value,
l2_lambda=l2_slider.value,
use_dropout=use_dropout_check.value,
dropout_rate=dropout_slider.value,
use_batchnorm=use_batchnorm_check.value,
max_epochs=epochs_slider_dyn.value,
update_freq=update_freq_dyn.value
)
start_button.on_click(on_start_clicked)
print("π TRAINING DYNAMICS VISUALIZER")
print("="*70)
print("Watch training in real-time and see overfitting happen!\n")
display(widgets.VBox([
widgets.HTML("<h3>Dataset Configuration</h3>"),
train_size_slider,
noise_slider,
widgets.HTML("<h3>Regularization (Try with and without!)</h3>"),
use_l2_check,
l2_slider,
use_dropout_check,
dropout_slider,
use_batchnorm_check,
widgets.HTML("<h3>Training Parameters</h3>"),
lr_slider_dyn,
epochs_slider_dyn,
update_freq_dyn,
widgets.HTML("<br>"),
start_button
]))π Conclusion
What Youβve Learned Today:
- XOR Problem - Neural networks can solve non-linearly separable problems
- Architecture - How layers, units, and activations affect performance
- Activation Functions - ReLU vs Sigmoid vs Tanh and their characteristics
- Training Dynamics - How to detect and prevent overfitting
Key Takeaways:
- β Start simple: 1 layer, few units, ReLU + Adam
- β Monitor curves: Training and validation loss tell you everything
- β Regularize when needed: L2, Dropout, BatchNorm help prevent overfitting
- β Experiment: Try different configurations and learn from results
Practice Suggestions:
- Go back to Demo 1 and try to find the minimal architecture that solves XOR
- Use Demo 2 to compare different architectures on the same dataset
- In Demo 3, see which activation works best for different datasets
- Use Demo 4 to understand when regularization is necessary
Resources:
- π Lecture notes with full mathematical derivations
- π» This Colab notebook (save a copy!)
- π Keras documentation: https://keras.io/
Questions? Observations? Share them with the class!