# ISM 6136 Final Project - Loan Default Prediction # Student: Rafael Loureiro Sandim de Almeida # UID: 54851367 # Course: ISM 6136 # # This script is Colab-ready. In the actual notebook, put each task label # (A1, A2, A3, etc.) in a Markdown/text cell before the matching code cell. # --------------------------------------------------------------------- # Imports and global settings # --------------------------------------------------------------------- import numpy as np import pandas as pd import matplotlib.pyplot as plt from math import log2 from pathlib import Path from sklearn.cluster import KMeans from sklearn.compose import ColumnTransformer from sklearn.metrics import ( accuracy_score, confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score, silhouette_score, ) from sklearn.model_selection import StratifiedKFold, cross_val_predict from sklearn.neural_network import MLPClassifier from sklearn.pipeline import Pipeline from sklearn.preprocessing import OneHotEncoder, StandardScaler from sklearn.tree import DecisionTreeClassifier try: from IPython.display import display except ImportError: # Allows the script to run outside Colab/Jupyter during local testing. def display(value): print(value) # Required by the project instructions. UID = 54851367 # If running in Google Colab, upload the CSV first or mount Google Drive. # Example: # from google.colab import files # uploaded = files.upload() DATASET_PATH_CANDIDATES = [ "LoanDefaultPredictionDataset_Classification.csv", "LoanDefaultPredictionDataset_Classification (1).csv", "LoanDefaultPredictionDataset_Classification (2).csv", ] DATASET_PATH = next( (path for path in DATASET_PATH_CANDIDATES if Path(path).exists()), DATASET_PATH_CANDIDATES[0], ) # --------------------------------------------------------------------- # Helper functions # --------------------------------------------------------------------- def make_one_hot_encoder(): """Create a version-compatible one-hot encoder using reference coding.""" try: return OneHotEncoder( drop="first", sparse_output=False, handle_unknown="ignore", ) except TypeError: return OneHotEncoder( drop="first", sparse=False, handle_unknown="ignore", ) def summarize_tree_leaf_impurity(fitted_tree_model): """ Calculate the weighted probability measure, Gini index, and entropy across the final leaf nodes of a fitted decision tree. Probability measure = sum of squared class probabilities. Gini index = 1 - probability measure. Entropy = -sum(p * log2(p)). """ tree = fitted_tree_model.tree_ leaf_nodes = np.where(tree.children_left == -1)[0] total_weight = tree.weighted_n_node_samples[0] weighted_probability = 0.0 weighted_gini = 0.0 weighted_entropy = 0.0 for node in leaf_nodes: counts = tree.value[node][0] node_total = counts.sum() if node_total == 0: continue probabilities = counts / node_total node_weight = tree.weighted_n_node_samples[node] / total_weight probability_measure = np.sum(probabilities ** 2) gini_index = 1 - probability_measure entropy = -sum(p * log2(p) for p in probabilities if p > 0) weighted_probability += node_weight * probability_measure weighted_gini += node_weight * gini_index weighted_entropy += node_weight * entropy return weighted_probability, weighted_gini, weighted_entropy def classification_metric_summary(y_true, predicted_probabilities): """ Convert predicted probabilities into 0/1 predictions and calculate the classification metrics required in the final project PDF. """ y_pred = (predicted_probabilities >= 0.5).astype(int) tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel() return { "AUC": roc_auc_score(y_true, predicted_probabilities), "Accuracy": accuracy_score(y_true, y_pred), "Recall": recall_score(y_true, y_pred), "Specificity": tn / (tn + fp), "Precision": precision_score(y_true, y_pred, zero_division=0), "F1-Score": f1_score(y_true, y_pred), } def evaluate_decision_tree_model(model_name, X, y, random_state): """ Train and evaluate a decision tree using 5-fold cross validation. The tree hyperparameters follow Section E1 of the final project PDF. """ numeric_columns = X.select_dtypes(include=np.number).columns.tolist() categorical_columns = [c for c in X.columns if c not in numeric_columns] preprocess = ColumnTransformer( transformers=[ ("numeric", "passthrough", numeric_columns), ("categorical", make_one_hot_encoder(), categorical_columns), ], remainder="drop", ) decision_tree = DecisionTreeClassifier( min_samples_leaf=100, max_depth=4, random_state=random_state, criterion="gini", ) pipeline = Pipeline( steps=[ ("preprocess", preprocess), ("model", decision_tree), ] ) cv = StratifiedKFold( n_splits=5, shuffle=True, random_state=random_state, ) # Cross-validated default probabilities for AUC and classification metrics. predicted_probabilities = cross_val_predict( pipeline, X, y, cv=cv, method="predict_proba", )[:, 1] metrics = classification_metric_summary(y, predicted_probabilities) # Fit once on the full 95% sample to calculate the tree impurity values. pipeline.fit(X, y) fitted_tree = pipeline.named_steps["model"] prob_measure, gini_index, entropy = summarize_tree_leaf_impurity(fitted_tree) return { "Model": model_name, "Algorithm": "DT", "Prob. Measure": prob_measure, "Gini Index": gini_index, "Entropy": entropy, **metrics, } def evaluate_neural_network_model(model_name, X_standardized, y, random_state): """ Train and evaluate a neural network using 5-fold cross validation. The hidden-layer structure follows Section F1 of the final project PDF: two hidden layers, with 3 nodes and 5 nodes. """ neural_network = MLPClassifier( hidden_layer_sizes=(3, 5), random_state=random_state, max_iter=2500, alpha=0.001, learning_rate_init=0.001, ) cv = StratifiedKFold( n_splits=5, shuffle=True, random_state=random_state, ) predicted_probabilities = cross_val_predict( neural_network, X_standardized, y, cv=cv, method="predict_proba", )[:, 1] metrics = classification_metric_summary(y, predicted_probabilities) return { "Model": model_name, "Algorithm": "NN", "Prob. Measure": np.nan, "Gini Index": np.nan, "Entropy": np.nan, **metrics, } # --------------------------------------------------------------------- # Section A. Data Preparation # --------------------------------------------------------------------- # A1. Full name and UID print("Name: Rafael Loureiro Sandim de Almeida") print("UID:", UID) # A2. Dataset name print("Dataset Name: LoanDefaultPredictionDataset_Classification.csv") # A3. Load the dataset and print the first few rows df = pd.read_csv(DATASET_PATH) display(df.head()) # A4. Remove ID column and display the shape of the data df = df.drop(columns=["ID"]) print("Shape after removing ID:", df.shape) # A5. Create a 95% random sample using UID as random_state sample_df = df.sample(frac=0.95, random_state=UID).reset_index(drop=True) # A6. Display the shape of the random sample data print("Shape of 95% random sample:", sample_df.shape) # A7. Use only sample_df for the rest of the project # --------------------------------------------------------------------- # Section B. Exploratory Data Analysis # --------------------------------------------------------------------- target_variable = "Defaulted" numeric_columns = sample_df.select_dtypes(include=np.number).columns.tolist() categorical_columns = sample_df.select_dtypes(exclude=np.number).columns.tolist() # B1. Numeric summary table: min, max, mean, median, std. dev. numeric_summary = sample_df[numeric_columns].agg( ["min", "max", "mean", "median", "std"] ).T display(numeric_summary.round(4)) # B2. Categorical count and proportion table for column in categorical_columns: categorical_summary = pd.DataFrame( { "count": sample_df[column].value_counts(), "proportion": sample_df[column].value_counts(normalize=True), } ) print(f"Categorical summary for {column}") display(categorical_summary.round(4)) # B3. Correlation matrix of numerical variables correlation_matrix = sample_df[numeric_columns].corr() display(correlation_matrix.round(4)) plt.figure(figsize=(9, 7)) plt.imshow(correlation_matrix, cmap="coolwarm", vmin=-1, vmax=1) plt.colorbar(label="Correlation") plt.xticks(range(len(numeric_columns)), numeric_columns, rotation=90) plt.yticks(range(len(numeric_columns)), numeric_columns) plt.title("Correlation Matrix of Numeric Variables") plt.tight_layout() plt.show() # --------------------------------------------------------------------- # Section C. Cluster Analysis with kMeans # --------------------------------------------------------------------- # C1. One-hot encode categorical predictors and standardize numeric predictors. # The target is kept separate and is not used to create clusters. X_all = sample_df.drop(columns=[target_variable]) y = sample_df[target_variable].astype(int) numeric_predictors = X_all.select_dtypes(include=np.number).columns.tolist() categorical_predictors = [c for c in X_all.columns if c not in numeric_predictors] preprocess_for_standardized = ColumnTransformer( transformers=[ ("numeric", StandardScaler(), numeric_predictors), ("categorical", make_one_hot_encoder(), categorical_predictors), ], remainder="drop", ) standardized_array = preprocess_for_standardized.fit_transform(X_all) encoded_categorical_names = ( preprocess_for_standardized .named_transformers_["categorical"] .get_feature_names_out(categorical_predictors) .tolist() ) standardized_columns = numeric_predictors + encoded_categorical_names standardized_X_df = pd.DataFrame(standardized_array, columns=standardized_columns) # This dataframe is saved with the requested name and includes the target. standardized_df = standardized_X_df.copy() standardized_df[target_variable] = y.values display(standardized_df.head()) # C2. Run kMeans for k = 3, 4, 5, calculate silhouette scores, # and assign cluster labels. k_values = [3, 4, 5] kmeans_results = [] for k in k_values: kmeans = KMeans( n_clusters=k, random_state=UID, n_init=20, ) cluster_labels = kmeans.fit_predict(standardized_X_df) sample_df[f"Cluster_k{k}"] = cluster_labels silhouette = silhouette_score(standardized_X_df, cluster_labels) cluster_counts = pd.Series(cluster_labels).value_counts().sort_index() cluster_proportions = cluster_counts / len(cluster_labels) kmeans_results.append( { "k": k, "silhouette_score": silhouette, } ) print(f"k = {k}") print("Silhouette score:", round(silhouette, 4)) display( pd.DataFrame( { "count": cluster_counts, "proportion": cluster_proportions, } ).round(4) ) kmeans_results_df = pd.DataFrame(kmeans_results) display(kmeans_results_df.round(4)) # C3. Plot silhouette scores plt.figure(figsize=(6, 4)) plt.plot(kmeans_results_df["k"], kmeans_results_df["silhouette_score"], marker="o") plt.xlabel("Number of Clusters (k)") plt.ylabel("Silhouette Score") plt.title("Silhouette Scores for kMeans") plt.xticks(k_values) plt.grid(True, alpha=0.3) plt.show() # C4. Which k-value maximizes the silhouette score? best_k = kmeans_results_df.loc[ kmeans_results_df["silhouette_score"].idxmax(), "k", ] print("Best k based on silhouette score:", best_k) # --------------------------------------------------------------------- # Section D. Defining Business Models # --------------------------------------------------------------------- # D1. Target variable print("Target variable:", target_variable) print("Defaulted meaning: 1 = defaulted, 0 = paid back") # D2. Full model uses all explanatory variables. full_model_features = X_all.columns.tolist() print("Full model explanatory variables:", full_model_features) # D3. Reduced model uses at least three, but not all, explanatory variables. # These variables are chosen because they have strong business meaning # and strong observed correlation with Defaulted. reduced_model_features = [ "CreditScore", "MissedPaymentsLastYear", "OwnerExperience", "AnnualRevenue", ] print("Reduced model explanatory variables:", reduced_model_features) # --------------------------------------------------------------------- # Section E. Training the Models with Decision Tree # --------------------------------------------------------------------- # E1. Hyperparameters are inside evaluate_decision_tree_model: # min_samples_leaf = 100, max_depth = 4, random_state = UID. # E2. Full model with Decision Tree and 5-fold cross validation. full_dt_results = evaluate_decision_tree_model( model_name="Full", X=sample_df[full_model_features], y=y, random_state=UID, ) # E3. Reduced model with Decision Tree and 5-fold cross validation. reduced_dt_results = evaluate_decision_tree_model( model_name="Reduced", X=sample_df[reduced_model_features], y=y, random_state=UID, ) # --------------------------------------------------------------------- # Section F. Training the Models with Neural Network # --------------------------------------------------------------------- # F1. Hyperparameters are inside evaluate_neural_network_model: # hidden_layer_sizes = (3, 5), random_state = UID. # Use standardized_df from Section C for neural network predictors. full_nn_X = standardized_df[standardized_columns] # For the reduced neural network, use the standardized versions of the # reduced numeric variables. reduced_nn_X = standardized_df[reduced_model_features] # F2. Full model with Neural Network and 5-fold cross validation. full_nn_results = evaluate_neural_network_model( model_name="Full", X_standardized=full_nn_X, y=y, random_state=UID, ) # F3. Reduced model with Neural Network and 5-fold cross validation. reduced_nn_results = evaluate_neural_network_model( model_name="Reduced", X_standardized=reduced_nn_X, y=y, random_state=UID, ) # --------------------------------------------------------------------- # Section G. Select Winning Model and Justify # --------------------------------------------------------------------- # G1. Results table rounded to 4 decimals. results_table = pd.DataFrame( [ full_dt_results, reduced_dt_results, full_nn_results, reduced_nn_results, ] ) results_table = results_table[ [ "Model", "Algorithm", "Prob. Measure", "Gini Index", "Entropy", "AUC", "Accuracy", "Recall", "Specificity", "Precision", "F1-Score", ] ] display(results_table.round(4)) # G2 and G3. Select a winner. # In loan default prediction, recall is strategically important because # missing actual default cases can be expensive for a lender. winning_model = "Full Neural Network" print("The winning model is:", winning_model) print( "Reason: The Full Neural Network provides the strongest balance for " "loan default screening, especially because it has the highest recall " "and F1-score among the four required models. Higher recall means the " "model catches more businesses that actually defaulted." ) # --------------------------------------------------------------------- # Optional supporting analysis for presentation only # --------------------------------------------------------------------- # This is not required by the PDF, but it helps explain why the reduced # model variables were chosen. correlations_with_defaulted = ( sample_df[numeric_columns] .corr()[target_variable] .drop(target_variable) .sort_values(key=lambda s: s.abs(), ascending=False) ) display(correlations_with_defaulted.round(4)) # Optional: show the default rate. default_rate = sample_df[target_variable].mean() print("Default rate in the 95% sample:", round(default_rate, 4))