import pandas as pd # dataframe manipulationimport numpy as np # linear algebra# data visualizationimport matplotlib.pyplot as pltimport matplotlib.cm as cmimport plotly.express as pximport plotly.graph_objects as goimport seaborn as snsimport shap# sklearnfrom sklearn.cluster import KMeansfrom sklearn.preprocessing import PowerTransformer, OrdinalEncoderfrom sklearn.pipeline import Pipelinefrom sklearn.manifold import TSNEfrom sklearn.metrics import silhouette_score, silhouette_samples, accuracy_score, classification_reportfrom pyod.models.ecod import ECODfrom yellowbrick.cluster import KElbowVisualizerimport lightgbm as lgbimport prince# Read filedf = pd.read_csv("train.csv", sep = ";")df = df.iloc[:,0:8]# Preprocessing partcategorical_transformer_onehot = Pipeline(steps=[("encoder", OneHotEncoder(handle_unknown="ignore", drop="first", sparse=False))])categorical_transformer_ordinal = Pipeline(steps=[("encoder", OrdinalEncoder())])num = Pipeline(steps=[("encoder", PowerTransformer())])preprocessor = ColumnTransformer(transformers = [('cat_onehot', categorical_transformer_onehot, ["default", "housing", "loan", "job", "marital"]),('cat_ordinal', categorical_transformer_ordinal, ["education"]),('num', num, ["age", "balance"])])pipeline = Pipeline(steps=[("preprocessor", preprocessor)])pipe_fit = pipeline.fit(df)data = pd.DataFrame(pipe_fit.transform(df), columns = pipe_fit.get_feature_names_out().tolist())print(data.columns.tolist())# OUTPUT['cat_onehot__default_yes','cat_onehot__housing_yes','cat_onehot__loan_yes','cat_onehot__job_blue-collar','cat_onehot__job_entrepreneur','cat_onehot__job_housemaid','cat_onehot__job_management','cat_onehot__job_retired','cat_onehot__job_self-employed','cat_onehot__job_services','cat_onehot__job_student','cat_onehot__job_technician','cat_onehot__job_unemployed','cat_onehot__job_unknown','cat_onehot__marital_married','cat_onehot__marital_single','cat_ordinal__education','num__age','num__balance']

from pyod.models.ecod import ECODclf = ECOD()clf.fit(data)outliers = clf.predict(data)data["outliers"] = outliers# Data without outliersdata_no_outliers = data[data["outliers"] == 0]data_no_outliers = data_no_outliers.drop(["outliers"], axis = 1)# Data with Outliersdata_with_outliers = data.copy()data_with_outliers = data_with_outliers.drop(["outliers"], axis = 1)print(data_no_outliers.shape) -> (40690, 19)print(data_with_outliers.shape) -> (45211, 19)

from yellowbrick.cluster import KElbowVisualizer# Instantiate the clustering model and visualizerkm = KMeans(init="k-means++", random_state=0, n_init="auto")visualizer = KElbowVisualizer(km, k=(2,10))visualizer.fit(data_no_outliers)# Fit the data to the visualizervisualizer.show()

from sklearn.metrics import davies_bouldin_score, silhouette_score, silhouette_samplesimport matplotlib.cm as cm
def make_Silhouette_plot(X, n_clusters):plt.xlim([-0.1, 1])plt.ylim([0, len(X) + (n_clusters + 1) * 10])clusterer = KMeans(n_clusters=n_clusters, max_iter = 1000, n_init = 10, init = 'k-means++', random_state=10)cluster_labels = clusterer.fit_predict(X)silhouette_avg = silhouette_score(X, cluster_labels)print("For n_clusters =", n_clusters,"The average silhouette_score is :", silhouette_avg,)# Compute the silhouette scores for each samplesample_silhouette_values = silhouette_samples(X, cluster_labels)y_lower = 10for i in range(n_clusters):ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels == i]ith_cluster_silhouette_values.sort()size_cluster_i = ith_cluster_silhouette_values.shape[0]y_upper = y_lower + size_cluster_icolor = cm.nipy_spectral(float(i) / n_clusters)plt.fill_betweenx(np.arange(y_lower, y_upper),0,ith_cluster_silhouette_values,facecolor=color,edgecolor=color,alpha=0.7,)plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))y_lower = y_upper + 10plt.title(f"The Silhouette Plot for n_cluster = {n_clusters}", fontsize=26)plt.xlabel("The silhouette coefficient values", fontsize=24)plt.ylabel("Cluster label", fontsize=24)plt.axvline(x=silhouette_avg, color="red", linestyle="--")plt.yticks([]) plt.xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])
range_n_clusters = list(range(2,10))
for n_clusters in range_n_clusters:print(f"N cluster: {n_clusters}")make_Silhouette_plot(data_no_outliers, n_clusters) plt.savefig('Silhouette_plot_{}.png'.format(n_clusters))plt.close()

"""N cluster: 2For n_clusters = 2 The average silhouette_score is : 0.18111287366156115N cluster: 3For n_clusters = 3 The average silhouette_score is : 0.16787543108034586N cluster: 4For n_clusters = 4 The average silhouette_score is : 0.1583411958880734N cluster: 5For n_clusters = 5 The average silhouette_score is : 0.1672987260052535N cluster: 6For n_clusters = 6 The average silhouette_score is : 0.15485098506258177N cluster: 7For n_clusters = 7 The average silhouette_score is : 0.1495307642182009N cluster: 8For n_clusters = 8 The average silhouette_score is : 0.15098396457075294N cluster: 9For n_clusters = 9 The average silhouette_score is : 0.14842917303536465"""

km = KMeans(n_clusters=5,init='k-means++', n_init=10,max_iter=100, random_state=42)clusters_predict = km.fit_predict(data_no_outliers)"""clusters_predict -> array([4, 2, 0, ..., 3, 4, 3])np.unique(clusters_predict) -> array([0, 1, 2, 3, 4])"""

from sklearn.metrics import silhouette_scorefrom sklearn.metrics import calinski_harabasz_scorefrom sklearn.metrics import davies_bouldin_score
"""The Davies Bouldin index is defined as the average similarity measure of each cluster with its most similar cluster, where similarity is the ratio of within-cluster distances to between-cluster distances.
The minimum value of the DB Index is 0, whereas a smaller value (closer to 0) represents a better model that produces better clusters."""print(f"Davies bouldin score: {davies_bouldin_score(data_no_outliers,clusters_predict)}")
"""Calinski Harabaz Index -> Variance Ratio Criterion.
Calinski Harabaz Index is defined as the ratio of the sum of between-cluster dispersion and of within-cluster dispersion.
The higher the index the more separable the clusters."""print(f"Calinski Score: {calinski_harabasz_score(data_no_outliers,clusters_predict)}")

"""The silhouette score is a metric used to calculate the goodness of fit of a clustering algorithm, but can also be used as a method for determining an optimal value of k (see here for more).
Its value ranges from -1 to 1.A value of 0 indicates clusters are overlapping and eitherthe data or the value of k is incorrect.
1 is the ideal value and indicates that clusters are very dense and nicely separated."""print(f"Silhouette Score: {silhouette_score(data_no_outliers,clusters_predict)}")

"""Davies bouldin score: 1.676769775662788Calinski Score: 6914.705500337112Silhouette Score: 0.16729335453305272"""

import princeimport plotly.express as px

def get_pca_2d(df, predict):
pca_2d_object = prince.PCA(n_components=2,n_iter=3,rescale_with_mean=True,rescale_with_std=True,copy=True,check_input=True,engine='sklearn',random_state=42)
pca_2d_object.fit(df)
df_pca_2d = pca_2d_object.transform(df)df_pca_2d.columns = ["comp1", "comp2"]df_pca_2d["cluster"] = predict
return pca_2d_object, df_pca_2d


def get_pca_3d(df, predict):
pca_3d_object = prince.PCA(n_components=3,n_iter=3,rescale_with_mean=True,rescale_with_std=True,copy=True,check_input=True,engine='sklearn',random_state=42)
pca_3d_object.fit(df)
df_pca_3d = pca_3d_object.transform(df)df_pca_3d.columns = ["comp1", "comp2", "comp3"]df_pca_3d["cluster"] = predict
return pca_3d_object, df_pca_3d


def plot_pca_3d(df, title = "PCA Space", opacity=0.8, width_line = 0.1):
df = df.astype({"cluster": "object"})df = df.sort_values("cluster")
fig = px.scatter_3d(df, x='comp1', y='comp2', z='comp3',color='cluster',template="plotly",
# symbol = "cluster",
color_discrete_sequence=px.colors.qualitative.Vivid,title=title).update_traces(# mode = 'markers',marker={"size": 4,"opacity": opacity,# "symbol" : "diamond","line": {"width": width_line,"color": "black",}}).update_layout(width = 800, height = 800, autosize = True, showlegend = True,legend=dict(title_font_family="Times New Roman",font=dict(size= 20)),scene = dict(xaxis=dict(title = 'comp1', titlefont_color = 'black'),yaxis=dict(title = 'comp2', titlefont_color = 'black'),zaxis=dict(title = 'comp3', titlefont_color = 'black')),font = dict(family = "Gilroy", color = 'black', size = 15))

fig.show()

pca_3d_object, df_pca_3d = pca_plot_3d(data_no_outliers, clusters_predict)plot_pca_3d(df_pca_3d, title ="PCA Space", opacity=1, width_line = 0.1)print("The variability is :", pca_3d_object.eigenvalues_summary)

from sklearn.manifold import TSNEsampling_data = data_no_outliers.sample(frac=0.5, replace=True, random_state=1)sampling_clusters = pd.DataFrame(clusters_predict).sample(frac=0.5, replace=True, random_state=1)[0].valuesdf_tsne_3d = TSNE(n_components=3, learning_rate=500, init='random', perplexity=200, n_iter =5000).fit_transform(sampling_data)df_tsne_3d = pd.DataFrame(df_tsne_3d, columns=["comp1", "comp2",'comp3'])df_tsne_3d["cluster"] = sampling_clustersplot_pca_3d(df_tsne_3d, title ="PCA Space", opacity=1, width_line = 0.1)

import lightgbm as lgbimport shap# We create the LGBMClassifier model and train itclf_km = lgb.LGBMClassifier(colsample_by_tree=0.8)clf_km.fit(X=data_no_outliers, y=clusters_predict)#SHAP valuesexplainer_km = shap.TreeExplainer(clf_km)shap_values_km = explainer_km.shap_values(data_no_outliers)shap.summary_plot(shap_values_km, data_no_outliers, plot_type="bar", plot_size=(15, 10))

df_no_outliers = df[df.outliers ==0]df_no_outliers["cluster"] = clusters_predictdf_no_outliers.groupby('cluster').agg({'job': lambda x: x.value_counts().index[0],'marital': lambda x: x.value_counts().index[0],'education': lambda x: x.value_counts().index[0],'housing': lambda x: x.value_counts().index[0],'loan': lambda x: x.value_counts().index[0],'contact': lambda x: x.value_counts().index[0],'age':'mean','balance': 'mean','default': lambda x: x.value_counts().index[0],}).reset_index()