Data Cleaning — Customer Churn Analysis

Setup

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

df = pd.read_csv("WA_Fn-UseC_-Telco-Customer-Churn.csv")
print(f"Shape: {df.shape}")
print(f"\nMissing values:\n{df.isnull().sum()[df.isnull().sum() > 0]}")
print(f"\nDuplicate rows: {df.duplicated().sum()}")

---

Step 1 — Fix TotalCharges Type

TotalCharges is stored as a string and has blank values for new customers.

# Inspect the problem
print(df["TotalCharges"].dtype)        # object
print(df[df["TotalCharges"] == " "])   # 11 rows with spaces

# Fix: convert to numeric, blanks become NaN
df["TotalCharges"] = pd.to_numeric(df["TotalCharges"], errors="coerce")

# For customers with tenure=0, TotalCharges should be 0
df.loc[(df["tenure"] == 0) & df["TotalCharges"].isna(), "TotalCharges"] = 0
print(f"Remaining NaN in TotalCharges: {df['TotalCharges'].isna().sum()}")

---

Step 2 — Standardise SeniorCitizen

SeniorCitizen is 0/1 while all other binary fields are Yes/No — inconsistent.

df["SeniorCitizen"] = df["SeniorCitizen"].map({0: "No", 1: "Yes"})
print(df["SeniorCitizen"].value_counts())

---

Step 3 — Simplify Internet Add-On Columns

Five columns use "No internet service" as a third value — effectively the same as "No" for our analysis.

internet_cols = ["OnlineSecurity", "OnlineBackup", "DeviceProtection",
                 "TechSupport", "StreamingTV", "StreamingMovies", "MultipleLines"]

for col in internet_cols:
    df[col] = df[col].replace("No internet service", "No")
    df[col] = df[col].replace("No phone service", "No")

print(df[internet_cols].apply(lambda x: x.unique()))

---

Step 4 — Convert Target to Binary

df["Churn"] = df["Churn"].map({"Yes": 1, "No": 0})
print(f"Churn rate: {df['Churn'].mean():.1%}")  # 26.5%

---

Step 5 — Feature Engineering

Create derived features that will improve our analysis.

# Tenure buckets
df["tenure_bucket"] = pd.cut(
    df["tenure"],
    bins=[-1, 6, 24, 48, 72],
    labels=["New (< 6mo)", "Growing (6-24mo)", "Established (2-4yr)", "Loyal (4+ yr)"]
)

# Add-on count — how many services does the customer have?
addons = ["OnlineSecurity", "OnlineBackup", "DeviceProtection",
          "TechSupport", "StreamingTV", "StreamingMovies"]
df["addon_count"] = (df[addons] == "Yes").sum(axis=1)

# Revenue per month per add-on (proxy for engagement)
df["monthly_charge_per_addon"] = df["MonthlyCharges"] / (df["addon_count"] + 1)

# Has automatic payment?
df["auto_payment"] = df["PaymentMethod"].isin(
    ["Bank transfer (automatic)", "Credit card (automatic)"]
).astype(int)

# Is a multi-year contract?
df["long_term_contract"] = (df["Contract"] != "Month-to-month").astype(int)

---

Step 6 — Audit After Cleaning

def audit(df, label=""):
    print(f"\n{'='*50}")
    print(f"AUDIT: {label}")
    print(f"Shape: {df.shape}")
    print(f"Missing: {df.isnull().sum().sum()} cells")
    print(f"Duplicates: {df.duplicated().sum()}")
    print(f"Churn rate: {df['Churn'].mean():.1%}")
    print(f"Avg tenure: {df['tenure'].mean():.1f} months")
    print(f"Avg monthly charge: £{df['MonthlyCharges'].mean():.2f}")

audit(df, "After Cleaning")

# Save clean dataset
df.to_csv("churn_clean.csv", index=False)
print("\nSaved to churn_clean.csv")

---

Distribution Checks

fig, axes = plt.subplots(2, 3, figsize=(15, 8))

# Tenure by churn status
df.groupby("Churn")["tenure"].hist(ax=axes[0, 0], alpha=0.7, bins=24)
axes[0, 0].set_title("Tenure Distribution by Churn Status")
axes[0, 0].legend(["Not Churned (0)", "Churned (1)"])

# Monthly charges by churn
df.boxplot(column="MonthlyCharges", by="Churn", ax=axes[0, 1])
axes[0, 1].set_title("Monthly Charges by Churn Status")

# Contract type distribution
df["Contract"].value_counts().plot(kind="bar", ax=axes[0, 2], color="#0D9488")
axes[0, 2].set_title("Contract Type Distribution")

# Add-on count vs churn
df.groupby("addon_count")["Churn"].mean().plot(kind="bar", ax=axes[1, 0], color="#F59E0B")
axes[1, 0].set_title("Churn Rate by Add-on Count")
axes[1, 0].set_ylabel("Churn Rate")

# Tenure bucket churn rate
churn_by_tenure = df.groupby("tenure_bucket", observed=True)["Churn"].mean()
churn_by_tenure.plot(kind="bar", ax=axes[1, 1], color="#EF4444")
axes[1, 1].set_title("Churn Rate by Tenure Bucket")

# Internet service churn
churn_by_internet = df.groupby("InternetService")["Churn"].mean()
churn_by_internet.plot(kind="bar", ax=axes[1, 2], color="#0D9488")
axes[1, 2].set_title("Churn Rate by Internet Service")

plt.suptitle("Churn Analysis — Data Distribution", fontsize=14)
plt.tight_layout()
plt.savefig("churn_distributions.png", dpi=150, bbox_inches="tight")
plt.show()

---

← Dataset Guide · Next: SQL Analysis →