MPS311/439 Machine Learning
Dr. Wei Xing
University of Sheffield
This notebook contains interactive demonstrations for understanding clustering algorithms.
# Install required packages (run this first in Google Colab)
import sys
if 'google.colab' in sys.modules:
print("Running in Google Colab - installing packages...")
!pip install -q ipywidgets
from google.colab import output
output.enable_custom_widget_manager()
print("✓ Setup complete!")✓ Setup complete!# Import all necessary libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs, make_moons, make_circles
from sklearn.cluster import KMeans, AgglomerativeClustering
from sklearn.preprocessing import StandardScaler
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.spatial import Voronoi, voronoi_plot_2d
import ipywidgets as widgets
from ipywidgets import interact, interactive, fixed, interact_manual
from IPython.display import display, HTML
import warnings
warnings.filterwarnings('ignore')
# Set random seed for reproducibility
np.random.seed(42)
# Set style
plt.style.use('default')
plt.rcParams['figure.figsize'] = (10, 6)
plt.rcParams['font.size'] = 11
print("✓ All libraries imported successfully!")✓ All libraries imported successfully!Learning Objective: Understand how K-means iteratively converges through the assignment and update steps.
What to observe: - How centroids move at each iteration - How cluster assignments change - How inertia (total distance) decreases - Algorithm always makes progress!
Try this: 1. Move the iteration slider slowly to see step-by-step progress 2. Click “Reset with New Initialization” to see different starting points 3. Try “Auto-play” to watch the full convergence
# Generate data for Demo 1
X_demo1, y_true_demo1 = make_blobs(n_samples=300, centers=3, n_features=2,
cluster_std=0.7, random_state=42)
def kmeans_step_by_step(X, K, max_iters=20, random_state=None):
"""Run K-means and track all iterations"""
if random_state is not None:
np.random.seed(random_state)
n, d = X.shape
# Initialize
idx = np.random.choice(n, K, replace=False)
centroids = X[idx].copy()
history = {'centroids': [centroids.copy()], 'labels': [], 'inertia': []}
for i in range(max_iters):
# Assignment
distances = np.sqrt(((X[:, np.newaxis, :] - centroids) ** 2).sum(axis=2))
labels = np.argmin(distances, axis=1)
# Compute inertia
inertia = sum([distances[labels == k, k].sum() ** 2 for k in range(K)])
history['labels'].append(labels.copy())
history['inertia'].append(inertia)
# Update
new_centroids = np.array([X[labels == k].mean(axis=0) if (labels == k).any()
else X[np.random.randint(n)] for k in range(K)])
history['centroids'].append(new_centroids.copy())
# Check convergence
if np.allclose(centroids, new_centroids, atol=1e-4):
break
centroids = new_centroids
return history
# Initial run
history_demo1 = kmeans_step_by_step(X_demo1, K=3, random_state=42)
max_iter_demo1 = len(history_demo1['centroids']) - 1
def plot_kmeans_iteration(iteration, show_lines=False):
"""Plot K-means at a specific iteration"""
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
# Left plot: Clustering visualization
centroids = history_demo1['centroids'][iteration]
if iteration > 0:
labels = history_demo1['labels'][iteration - 1]
colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
for k in range(3):
mask = labels == k
ax1.scatter(X_demo1[mask, 0], X_demo1[mask, 1],
c=colors[k], s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
# Optionally show lines to centroids
if show_lines and mask.any():
for point in X_demo1[mask][:5]: # Show only first 5 to avoid clutter
ax1.plot([point[0], centroids[k, 0]],
[point[1], centroids[k, 1]],
'k-', alpha=0.2, linewidth=0.5)
else:
ax1.scatter(X_demo1[:, 0], X_demo1[:, 1],
c='lightgray', s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
# Plot centroids
ax1.scatter(centroids[:, 0], centroids[:, 1],
marker='X', s=500, c='red', edgecolors='black', linewidth=3, zorder=5)
ax1.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax1.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax1.set_title(f'Iteration {iteration} - Clustering Result', fontsize=14, fontweight='bold')
ax1.grid(True, alpha=0.3)
# Add iteration info
if iteration > 0:
inertia = history_demo1['inertia'][iteration - 1]
ax1.text(0.02, 0.98, f'Inertia: {inertia:.2f}',
transform=ax1.transAxes, fontsize=12, verticalalignment='top',
bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.8))
# Right plot: Inertia over iterations
if len(history_demo1['inertia']) > 0:
iters = range(1, len(history_demo1['inertia']) + 1)
ax2.plot(iters, history_demo1['inertia'], 'o-', linewidth=2, markersize=8)
if iteration > 0:
ax2.plot(iteration, history_demo1['inertia'][iteration-1],
'ro', markersize=15, zorder=5)
ax2.set_xlabel('Iteration', fontsize=12, fontweight='bold')
ax2.set_ylabel('Inertia (Within-Cluster Sum of Squares)', fontsize=12, fontweight='bold')
ax2.set_title('Convergence: Inertia Decreases', fontsize=14, fontweight='bold')
ax2.grid(True, alpha=0.3)
ax2.set_xticks(range(1, len(history_demo1['inertia']) + 1))
plt.tight_layout()
plt.show()
# Create interactive widget
iteration_slider = widgets.IntSlider(
value=0, min=0, max=max_iter_demo1, step=1,
description='Iteration:', continuous_update=False,
style={'description_width': '100px'},
layout=widgets.Layout(width='600px')
)
show_lines_checkbox = widgets.Checkbox(
value=False, description='Show connections to centroids',
style={'description_width': 'initial'}
)
def reset_kmeans(b):
global history_demo1, max_iter_demo1
random_state = np.random.randint(0, 1000)
history_demo1 = kmeans_step_by_step(X_demo1, K=3, random_state=random_state)
max_iter_demo1 = len(history_demo1['centroids']) - 1
iteration_slider.max = max_iter_demo1
iteration_slider.value = 0
print(f"✓ Reset with new initialization (converged in {max_iter_demo1} iterations)")
reset_button = widgets.Button(
description='Reset with New Initialization',
button_style='warning',
icon='refresh'
)
reset_button.on_click(reset_kmeans)
def autoplay(b):
import time
for i in range(max_iter_demo1 + 1):
iteration_slider.value = i
time.sleep(0.5)
autoplay_button = widgets.Button(
description='Auto-play Convergence',
button_style='success',
icon='play'
)
autoplay_button.on_click(autoplay)
controls = widgets.VBox([
iteration_slider,
show_lines_checkbox,
widgets.HBox([reset_button, autoplay_button])
])
interactive_plot = interactive(plot_kmeans_iteration,
iteration=iteration_slider,
show_lines=show_lines_checkbox)
display(controls)
display(interactive_plot.children[-1])# second example
from IPython.display import display, clear_output
from sklearn.metrics import pairwise_distances_argmin
# --- 1. Generate Data ---
X, y_true = make_blobs(n_samples=300, centers=4, cluster_std=0.8, random_state=42)
# --- 2. The Interactive Class ---
class KMeansStepper:
def __init__(self, X):
self.X = X
self.k = 3
self.centroids = None
self.labels = None
self.iteration = 0
self.history = [] # To store centroid paths
# Setup UI
self.out = widgets.Output()
# Widgets
self.slider_k = widgets.IntSlider(value=3, min=2, max=6, description='K (Clusters):')
self.btn_init = widgets.Button(description='Initialize / Reset', button_style='info')
self.btn_step = widgets.Button(description='Take 1 Step', button_style='success')
# Event listeners
self.btn_init.on_click(self.reset)
self.btn_step.on_click(self.step)
self.slider_k.observe(self.on_k_change, names='value')
self.reset(None)
def on_k_change(self, change):
self.k = change['new']
self.reset(None)
def reset(self, b):
# Pick random points as initial centroids
rng = np.random.RandomState(np.random.randint(1000))
i = rng.permutation(self.X.shape[0])[:self.k]
self.centroids = self.X[i]
self.labels = np.zeros(self.X.shape[0])
self.iteration = 0
self.history = [self.centroids]
self.plot()
def step(self, b):
# 1. Expectation: Assign labels
self.labels = pairwise_distances_argmin(self.X, self.centroids)
# 2. Maximization: Update centroids
new_centroids = np.array([self.X[self.labels == i].mean(0)
if np.sum(self.labels == i) > 0
else self.centroids[i]
for i in range(self.k)])
self.centroids = new_centroids
self.history.append(self.centroids)
self.iteration += 1
self.plot()
def plot(self):
with self.out:
clear_output(wait=True)
fig, ax = plt.subplots(figsize=(8, 6))
# Plot data points colored by label
# If iteration 0 (just init), make them all gray
c_map = 'viridis' if self.iteration > 0 else None
color = self.labels if self.iteration > 0 else 'gray'
ax.scatter(self.X[:, 0], self.X[:, 1], c=color, cmap=c_map,
s=30, alpha=0.6, edgecolor='k', linewidth=0.5)
# Plot Centroids
ax.scatter(self.centroids[:, 0], self.centroids[:, 1],
c='red', marker='X', s=200, edgecolor='white', linewidth=2, label='Current Centroids')
# Plot History (Movement trails)
if len(self.history) > 1:
hist_arr = np.array(self.history)
# Draw lines for each centroid
for k_idx in range(self.k):
# Extract path for centroid k
path = hist_arr[:, k_idx, :]
ax.plot(path[:, 0], path[:, 1], 'k--', alpha=0.5)
ax.set_title(f"Iteration: {self.iteration}", fontsize=16)
ax.legend()
plt.show()
def display(self):
controls = widgets.HBox([self.slider_k, self.btn_init, self.btn_step])
display(widgets.VBox([controls, self.out]))
# Run Demo 1
viz = KMeansStepper(X)
viz.display()Learning Objective: Learn how to choose the optimal number of clusters K using the elbow method.
What to observe: - How inertia decreases as K increases - The “elbow point” where the decrease slows down - How the elbow point corresponds to the natural number of clusters
Try this: 1. Change K and see how the clustering looks (left plot) 2. Watch how your current K is highlighted on the elbow curve (right plot) 3. Change “True Clusters” to see how the elbow shifts with different data 4. Does the elbow always match the true number of clusters?
def generate_elbow_data(n_clusters_true=3, random_state=42):
"""Generate data with specified number of true clusters"""
X, y = make_blobs(n_samples=400, centers=n_clusters_true, n_features=2,
cluster_std=0.9, random_state=random_state)
return X, y
def compute_inertias(X, K_range):
"""Compute inertia for different K values"""
inertias = []
for k in K_range:
kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
kmeans.fit(X)
inertias.append(kmeans.inertia_)
return inertias
# Initial data
X_elbow, y_elbow = generate_elbow_data(n_clusters_true=3)
K_range = range(1, 11)
inertias_elbow = compute_inertias(X_elbow, K_range)
def plot_elbow_interactive(K, n_clusters_true, show_elbow_annotation):
"""Interactive elbow method visualization"""
global X_elbow, inertias_elbow
# Regenerate data if n_clusters_true changed
X_elbow, _ = generate_elbow_data(n_clusters_true=n_clusters_true)
inertias_elbow = compute_inertias(X_elbow, K_range)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
# Left plot: Clustering result for current K
kmeans = KMeans(n_clusters=K, random_state=42, n_init=10)
labels = kmeans.fit_predict(X_elbow)
scatter = ax1.scatter(X_elbow[:, 0], X_elbow[:, 1], c=labels,
cmap='viridis', s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
ax1.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],
marker='X', s=500, c='red', edgecolors='black', linewidth=3, zorder=5)
ax1.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax1.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax1.set_title(f'Clustering with K={K}', fontsize=14, fontweight='bold')
ax1.grid(True, alpha=0.3)
# Add info box
info_text = f'K = {K}\nInertia = {kmeans.inertia_:.2f}\nTrue clusters = {n_clusters_true}'
ax1.text(0.02, 0.98, info_text, transform=ax1.transAxes,
fontsize=11, verticalalignment='top',
bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.8))
# Right plot: Elbow curve
ax2.plot(K_range, inertias_elbow, 'o-', linewidth=2, markersize=8,
color='steelblue', markerfacecolor='orange', markeredgewidth=2,
markeredgecolor='steelblue')
# Highlight current K
ax2.plot(K, inertias_elbow[K-1], 'o', markersize=20,
color='red', markeredgewidth=3, markeredgecolor='darkred', zorder=5)
# Show elbow annotation if requested
if show_elbow_annotation:
# Simple heuristic: find elbow using second derivative
inertias_array = np.array(inertias_elbow)
# Compute rate of change
diffs = np.diff(inertias_array)
second_diffs = np.diff(diffs)
elbow_k = np.argmax(second_diffs) + 2 # +2 due to double diff
ax2.axvline(x=elbow_k, color='green', linestyle='--', linewidth=2, alpha=0.7)
ax2.annotate(f'Suggested Elbow\nK = {elbow_k}',
xy=(elbow_k, inertias_elbow[elbow_k-1]),
xytext=(elbow_k + 1.5, inertias_elbow[elbow_k-1] + max(inertias_elbow)*0.15),
arrowprops=dict(arrowstyle='->', color='green', lw=2),
fontsize=11, fontweight='bold', color='green',
bbox=dict(boxstyle='round', facecolor='lightgreen', alpha=0.7))
ax2.set_xlabel('Number of Clusters (K)', fontsize=12, fontweight='bold')
ax2.set_ylabel('Inertia (Within-Cluster Sum of Squares)', fontsize=12, fontweight='bold')
ax2.set_title('Elbow Method: Find the "Bend"', fontsize=14, fontweight='bold')
ax2.set_xticks(K_range)
ax2.grid(True, alpha=0.3, linestyle='--')
plt.tight_layout()
plt.show()
# Create widgets
K_slider = widgets.IntSlider(
value=3, min=1, max=10, step=1,
description='K (clusters):',
continuous_update=False,
style={'description_width': '120px'},
layout=widgets.Layout(width='600px')
)
true_clusters_slider = widgets.IntSlider(
value=3, min=2, max=6, step=1,
description='True Clusters:',
continuous_update=False,
style={'description_width': '120px'},
layout=widgets.Layout(width='600px')
)
show_elbow_checkbox = widgets.Checkbox(
value=True, description='Show elbow annotation',
style={'description_width': 'initial'}
)
interact(plot_elbow_interactive,
K=K_slider,
n_clusters_true=true_clusters_slider,
show_elbow_annotation=show_elbow_checkbox);Learning Objective: Understand when K-means struggles and why.
What to observe: - K-means works perfectly on well-separated spherical clusters - Fails on non-convex shapes (half-moons, circles) - Struggles with elongated or different-density clusters - Voronoi boundaries show why: straight-line decisions only!
Try this: 1. Select different dataset types from dropdown 2. Toggle “Show Voronoi boundaries” to see decision regions 3. Change K and see if any value helps the bad cases 4. Click “Regenerate Data” to try different random configurations
def generate_dataset(dataset_type, random_state=42):
"""Generate different types of datasets"""
if dataset_type == "Well-separated spherical":
X, y = make_blobs(n_samples=300, centers=3, n_features=2,
cluster_std=0.6, random_state=random_state)
K_true = 3
elif dataset_type == "Non-convex (half-moons)":
X, y = make_moons(n_samples=300, noise=0.08, random_state=random_state)
K_true = 2
elif dataset_type == "Concentric circles":
X, y = make_circles(n_samples=300, noise=0.05, factor=0.5, random_state=random_state)
K_true = 2
elif dataset_type == "Different densities":
np.random.seed(random_state)
X1 = np.random.randn(200, 2) * 0.3 + np.array([0, 0])
X2 = np.random.randn(50, 2) * 0.3 + np.array([3, 3])
X3 = np.random.randn(300, 2) * 1.5 + np.array([5, -2])
X = np.vstack([X1, X2, X3])
y = np.array([0]*200 + [1]*50 + [2]*300)
K_true = 3
elif dataset_type == "Elongated clusters":
np.random.seed(random_state)
X1 = np.random.randn(150, 2) @ np.array([[3, 0], [0, 0.5]]) + np.array([0, 0])
X2 = np.random.randn(150, 2) @ np.array([[0.5, 0], [0, 3]]) + np.array([5, 5])
X = np.vstack([X1, X2])
y = np.array([0]*150 + [1]*150)
K_true = 2
return X, y, K_true
def plot_voronoi_kmeans(kmeans, X, ax):
"""Plot Voronoi diagram for K-means clustering"""
try:
from scipy.spatial import Voronoi
vor = Voronoi(kmeans.cluster_centers_)
# Plot Voronoi edges
for simplex in vor.ridge_vertices:
simplex = np.asarray(simplex)
if np.all(simplex >= 0):
ax.plot(vor.vertices[simplex, 0], vor.vertices[simplex, 1],
'k--', alpha=0.4, linewidth=1.5)
except:
pass # Skip if Voronoi fails
current_random_state = 42
def plot_failure_cases(dataset_type, K, show_voronoi):
"""Interactive failure cases visualization"""
global current_random_state
X, y_true, K_true = generate_dataset(dataset_type, current_random_state)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
# Left: True structure (if available)
ax1.scatter(X[:, 0], X[:, 1], c=y_true, cmap='tab10',
s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
ax1.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax1.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax1.set_title(f'True Structure (K={K_true})', fontsize=14, fontweight='bold')
ax1.grid(True, alpha=0.3)
# Right: K-means result
kmeans = KMeans(n_clusters=K, random_state=42, n_init=10)
labels_kmeans = kmeans.fit_predict(X)
ax2.scatter(X[:, 0], X[:, 1], c=labels_kmeans, cmap='viridis',
s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
ax2.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],
marker='X', s=500, c='red', edgecolors='black', linewidth=3, zorder=5)
if show_voronoi:
plot_voronoi_kmeans(kmeans, X, ax2)
ax2.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax2.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax2.set_title(f'K-means Result (K={K})', fontsize=14, fontweight='bold')
ax2.grid(True, alpha=0.3)
# Add assessment
if dataset_type == "Well-separated spherical":
assessment = "✅ K-means works great here!"
color = 'lightgreen'
else:
assessment = "❌ K-means struggles with this structure"
color = 'lightcoral'
ax2.text(0.5, 0.02, assessment, transform=ax2.transAxes,
fontsize=12, fontweight='bold', ha='center',
bbox=dict(boxstyle='round', facecolor=color, alpha=0.8))
plt.tight_layout()
plt.show()
def regenerate_data(b):
global current_random_state
current_random_state = np.random.randint(0, 1000)
print(f"✓ Data regenerated with random_state={current_random_state}")
# Create widgets
dataset_dropdown = widgets.Dropdown(
options=["Well-separated spherical", "Non-convex (half-moons)",
"Concentric circles", "Different densities", "Elongated clusters"],
value="Well-separated spherical",
description='Dataset:',
style={'description_width': '100px'},
layout=widgets.Layout(width='500px')
)
K_slider_failures = widgets.IntSlider(
value=2, min=2, max=5, step=1,
description='K:',
continuous_update=False,
style={'description_width': '100px'},
layout=widgets.Layout(width='400px')
)
voronoi_checkbox = widgets.Checkbox(
value=False, description='Show Voronoi boundaries',
style={'description_width': 'initial'}
)
regenerate_button = widgets.Button(
description='Regenerate Data',
button_style='info',
icon='refresh'
)
regenerate_button.on_click(regenerate_data)
controls = widgets.VBox([
dataset_dropdown,
K_slider_failures,
voronoi_checkbox,
regenerate_button
])
interactive_failures = interactive(plot_failure_cases,
dataset_type=dataset_dropdown,
K=K_slider_failures,
show_voronoi=voronoi_checkbox)
display(controls)
display(interactive_failures.children[-1])Learning Objective: Understand dendrograms and how cutting at different heights gives different numbers of clusters.
What to observe: - The tree structure shows merge hierarchy - Cutting at different heights gives different K values - Different linkage methods create different trees - Large vertical gaps suggest natural separations
Try this: 1. Move the “Cutting Height” slider and watch clusters form/merge 2. Count how many vertical lines the red line crosses = K 3. Change linkage method - see how tree structure changes 4. Find the best cutting height by looking for large vertical gaps
# Generate hierarchical data
X_hier, y_hier = make_blobs(n_samples=50, centers=3, n_features=2,
cluster_std=0.6, random_state=42)
def plot_dendrogram_interactive(cutting_height, linkage_method):
"""Interactive dendrogram visualization"""
fig = plt.figure(figsize=(16, 6))
gs = fig.add_gridspec(1, 2, width_ratios=[1.2, 1])
ax1 = fig.add_subplot(gs[0])
ax2 = fig.add_subplot(gs[1])
# Compute linkage
Z = linkage(X_hier, method=linkage_method)
# Plot dendrogram
dendro = dendrogram(Z, ax=ax1, color_threshold=cutting_height,
above_threshold_color='gray')
# Add cutting line
ax1.axhline(y=cutting_height, color='red', linestyle='--', linewidth=3,
label=f'Cut at height {cutting_height:.2f}')
ax1.set_xlabel('Data Point Index', fontsize=12, fontweight='bold')
ax1.set_ylabel('Distance (Dissimilarity)', fontsize=12, fontweight='bold')
ax1.set_title(f'Dendrogram ({linkage_method.capitalize()} Linkage)',
fontsize=14, fontweight='bold')
ax1.legend(fontsize=11)
ax1.grid(True, axis='y', alpha=0.3)
# Get clusters at this cutting height
hc = AgglomerativeClustering(n_clusters=None, distance_threshold=cutting_height,
linkage=linkage_method)
labels_hier = hc.fit_predict(X_hier)
n_clusters = len(np.unique(labels_hier))
# Plot scatter with clusters
scatter = ax2.scatter(X_hier[:, 0], X_hier[:, 1], c=labels_hier,
cmap='tab10', s=100, alpha=0.7, edgecolors='k', linewidth=1)
ax2.set_xlabel('Feature 1', fontsize=12, fontweight='bold')
ax2.set_ylabel('Feature 2', fontsize=12, fontweight='bold')
ax2.set_title(f'Resulting Clusters (K={n_clusters})', fontsize=14, fontweight='bold')
ax2.grid(True, alpha=0.3)
# Add info box
info_text = f'Cutting Height: {cutting_height:.2f}\nNumber of Clusters: {n_clusters}\nLinkage: {linkage_method}'
ax2.text(0.02, 0.98, info_text, transform=ax2.transAxes,
fontsize=11, verticalalignment='top',
bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.8))
plt.tight_layout()
plt.show()
# Create widgets
height_slider = widgets.FloatSlider(
value=3.0, min=0.5, max=8.0, step=0.1,
description='Cutting Height:',
continuous_update=False,
readout_format='.2f',
style={'description_width': '120px'},
layout=widgets.Layout(width='600px')
)
linkage_dropdown = widgets.Dropdown(
options=['single', 'complete', 'average', 'ward'],
value='average',
description='Linkage:',
style={'description_width': '120px'},
layout=widgets.Layout(width='400px')
)
interact(plot_dendrogram_interactive,
cutting_height=height_slider,
linkage_method=linkage_dropdown);Learning Objective: Understand why feature scaling is critical for K-means.
What to observe: - Without scaling: Feature 2 (large range) dominates the clustering - With scaling: Both features contribute equally - Cluster centers move dramatically when scaling is applied - Results can be completely different!
Try this: 1. Toggle “Apply Scaling” on and off 2. Look at the centroid coordinates - see how different? 3. Observe how cluster boundaries change 4. Change K to see if scaling matters more for some K values
# Generate data with different scales
np.random.seed(42)
X_scale_1 = np.random.randn(150, 2) * np.array([0.5, 100]) + np.array([2, 500])
X_scale_2 = np.random.randn(150, 2) * np.array([0.5, 100]) + np.array([5, 800])
X_scale_3 = np.random.randn(150, 2) * np.array([0.5, 100]) + np.array([8, 300])
X_scaling = np.vstack([X_scale_1, X_scale_2, X_scale_3])
def plot_scaling_impact(K, apply_scaling):
"""Demonstrate impact of feature scaling"""
fig, axes = plt.subplots(1, 2, figsize=(16, 6))
# Prepare data
if apply_scaling:
scaler = StandardScaler()
X_plot = scaler.fit_transform(X_scaling)
title_suffix = "(WITH Scaling)"
color = 'lightgreen'
else:
X_plot = X_scaling.copy()
title_suffix = "(NO Scaling)"
color = 'lightcoral'
# Fit K-means
kmeans = KMeans(n_clusters=K, random_state=42, n_init=10)
labels = kmeans.fit_predict(X_plot)
centers = kmeans.cluster_centers_
# Plot clustering result
axes[0].scatter(X_plot[:, 0], X_plot[:, 1], c=labels, cmap='viridis',
s=50, alpha=0.6, edgecolors='k', linewidth=0.5)
axes[0].scatter(centers[:, 0], centers[:, 1], marker='X', s=500,
c='red', edgecolors='black', linewidth=3, zorder=5)
axes[0].set_xlabel('Feature 1 (Small range)', fontsize=12, fontweight='bold')
axes[0].set_ylabel('Feature 2 (Large range)', fontsize=12, fontweight='bold')
axes[0].set_title(f'K-means Clustering {title_suffix}', fontsize=14, fontweight='bold')
axes[0].grid(True, alpha=0.3)
# Add centroid coordinates
coord_text = "Centroids:\n"
for i, center in enumerate(centers):
coord_text += f"C{i+1}: ({center[0]:.2f}, {center[1]:.2f})\n"
axes[0].text(0.02, 0.98, coord_text, transform=axes[0].transAxes,
fontsize=10, verticalalignment='top',
bbox=dict(boxstyle='round', facecolor=color, alpha=0.8))
# Plot feature ranges
feature_ranges = [
['Feature 1', X_plot[:, 0].min(), X_plot[:, 0].max(), X_plot[:, 0].std()],
['Feature 2', X_plot[:, 1].min(), X_plot[:, 1].max(), X_plot[:, 1].std()]
]
axes[1].axis('off')
# Create text display
info_text = f"**Feature Statistics {title_suffix}**\n\n"
info_text += f"Feature 1 (e.g., normalized spending): \n"
info_text += f" Range: [{X_plot[:, 0].min():.2f}, {X_plot[:, 0].max():.2f}]\n"
info_text += f" Std Dev: {X_plot[:, 0].std():.2f}\n\n"
info_text += f"Feature 2 (e.g., raw visit count): \n"
info_text += f" Range: [{X_plot[:, 1].min():.2f}, {X_plot[:, 1].max():.2f}]\n"
info_text += f" Std Dev: {X_plot[:, 1].std():.2f}\n\n"
if apply_scaling:
explanation = "✅ Both features contribute equally\n"
explanation += "✅ Balanced influence on clustering\n"
explanation += "✅ Both features have similar ranges"
box_color = 'lightgreen'
else:
explanation = "⚠️ Feature 2 DOMINATES clustering\n"
explanation += "⚠️ Feature 1 barely affects results\n"
explanation += "⚠️ Large range difference = imbalanced"
box_color = 'lightcoral'
axes[1].text(0.1, 0.7, info_text, fontsize=12, verticalalignment='top',
family='monospace',
bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.8))
axes[1].text(0.1, 0.3, explanation, fontsize=13, verticalalignment='top',
fontweight='bold',
bbox=dict(boxstyle='round', facecolor=box_color, alpha=0.9))
axes[1].set_title('Impact Analysis', fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()
# Create widgets
K_slider_scaling = widgets.IntSlider(
value=3, min=2, max=5, step=1,
description='K:',
continuous_update=False,
style={'description_width': '100px'},
layout=widgets.Layout(width='400px')
)
scaling_checkbox = widgets.Checkbox(
value=False, description='Apply StandardScaler (IMPORTANT!)',
style={'description_width': 'initial'}
)
interact(plot_scaling_impact,
K=K_slider_scaling,
apply_scaling=scaling_checkbox);Learning Objective: Compare K-means and hierarchical clustering side-by-side.
What to observe: - Both methods can give different results on same data - K-means is faster (check computation time!) - Hierarchical provides the dendrogram (extra information) - Different linkage methods affect hierarchical results
Try this: 1. Change K - both methods update 2. Look at computation times displayed 3. Try different linkage methods for hierarchical 4. Click “Randomize Data” to test on different patterns 5. For which datasets do they agree? Disagree?
import time
# Generate comparison data
X_compare, y_compare = make_blobs(n_samples=200, centers=3, n_features=2,
cluster_std=0.8, random_state=42)
comparison_random_state = 42
def plot_comparison(K, linkage_method, show_centers):
"""Compare K-means and hierarchical clustering"""
global X_compare, comparison_random_state
X_compare, _ = make_blobs(n_samples=200, centers=3, n_features=2,
cluster_std=0.8, random_state=comparison_random_state)
fig = plt.figure(figsize=(18, 6))
gs = fig.add_gridspec(1, 3, width_ratios=[1, 1, 1])
ax1 = fig.add_subplot(gs[0])
ax2 = fig.add_subplot(gs[1])
ax3 = fig.add_subplot(gs[2])
# K-means
start_time = time.time()
kmeans = KMeans(n_clusters=K, random_state=42, n_init=10)
labels_kmeans = kmeans.fit_predict(X_compare)
kmeans_time = time.time() - start_time
ax1.scatter(X_compare[:, 0], X_compare[:, 1], c=labels_kmeans, cmap='viridis',
s=60, alpha=0.6, edgecolors='k', linewidth=0.5)
if show_centers:
ax1.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],
marker='X', s=500, c='red', edgecolors='black', linewidth=3, zorder=5)
ax1.set_xlabel('Feature 1', fontsize=11, fontweight='bold')
ax1.set_ylabel('Feature 2', fontsize=11, fontweight='bold')
ax1.set_title(f'K-means (K={K})\nTime: {kmeans_time*1000:.2f} ms',
fontsize=13, fontweight='bold')
ax1.grid(True, alpha=0.3)
# Hierarchical
start_time = time.time()
hc = AgglomerativeClustering(n_clusters=K, linkage=linkage_method)
labels_hier = hc.fit_predict(X_compare)
hier_time = time.time() - start_time
ax2.scatter(X_compare[:, 0], X_compare[:, 1], c=labels_hier, cmap='viridis',
s=60, alpha=0.6, edgecolors='k', linewidth=0.5)
ax2.set_xlabel('Feature 1', fontsize=11, fontweight='bold')
ax2.set_ylabel('Feature 2', fontsize=11, fontweight='bold')
ax2.set_title(f'Hierarchical ({linkage_method})\nTime: {hier_time*1000:.2f} ms',
fontsize=13, fontweight='bold')
ax2.grid(True, alpha=0.3)
# Dendrogram
Z = linkage(X_compare, method=linkage_method)
dendrogram(Z, ax=ax3, no_labels=True, color_threshold=0,
above_threshold_color='steelblue')
ax3.set_xlabel('Data Points', fontsize=11, fontweight='bold')
ax3.set_ylabel('Distance', fontsize=11, fontweight='bold')
ax3.set_title('Dendrogram\n(Hierarchical Only)', fontsize=13, fontweight='bold')
ax3.grid(True, axis='y', alpha=0.3)
# Speed comparison
speedup = hier_time / kmeans_time
fig.text(0.5, 0.02,
f'Speed Comparison: K-means is {speedup:.1f}x faster on this dataset (n=200 points)',
ha='center', fontsize=12, fontweight='bold',
bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.8))
plt.tight_layout(rect=[0, 0.05, 1, 1])
plt.show()
def randomize_comparison_data(b):
global comparison_random_state
comparison_random_state = np.random.randint(0, 1000)
print(f"✓ New dataset generated (random_state={comparison_random_state})")
# Create widgets
K_slider_compare = widgets.IntSlider(
value=3, min=2, max=7, step=1,
description='K:',
continuous_update=False,
style={'description_width': '100px'},
layout=widgets.Layout(width='400px')
)
linkage_dropdown_compare = widgets.Dropdown(
options=['single', 'complete', 'average', 'ward'],
value='average',
description='Linkage:',
style={'description_width': '100px'},
layout=widgets.Layout(width='400px')
)
centers_checkbox = widgets.Checkbox(
value=True, description='Show K-means centroids',
style={'description_width': 'initial'}
)
randomize_compare_button = widgets.Button(
description='Randomize Data',
button_style='warning',
icon='random'
)
randomize_compare_button.on_click(randomize_comparison_data)
controls = widgets.VBox([
K_slider_compare,
linkage_dropdown_compare,
centers_checkbox,
randomize_compare_button
])
interactive_compare = interactive(plot_comparison,
K=K_slider_compare,
linkage_method=linkage_dropdown_compare,
show_centers=centers_checkbox)
display(controls)
display(interactive_compare.children[-1])K-means is iterative: It always makes progress and converges by alternating assignment and update steps
The elbow method works: Look for the bend in the inertia curve to choose K
K-means has limitations: Fails on non-convex shapes, different densities, and elongated clusters
Dendrograms are powerful: Cut at different heights to explore multiple clustering scales
Scaling is CRITICAL: Always standardize features before clustering!
Choose the right tool: K-means for speed and known K; hierarchical for exploration
Now that you’ve explored these concepts interactively, try:
make_blobs() and apply both algorithmsRemember: Clustering is exploratory - there’s often no single “right” answer!
Questions? Come to office hours or post on the discussion forum!
Dr. Wei Xing
Office: Hicks Building I22
Office Hours: Tuesday 12:00-1:00 pm