import numpy as np import io import sys import os import pandas as pd import gc #garabage collector from io import BytesIO import panel as pn import holoviews as hv import hvplot.pandas from warnings import filterwarnings ''' development env: panel serve script.py --autoreload prod prep: panel convert script.py --to pyodide-worker --out pyodide ''' filterwarnings("ignore") hv.extension('bokeh') text = """ # Feature Distribution and Stats ## AUTHOR: [`FIRAS ALI OBEID`](https://www.linkedin.com/in/feras-obeid/) ### GNU General Public License v3.0 (GPL-3.0) #### Developed while working at [OppFi Inc.](https://www.oppfi.com/) This tool performs feature binning by equal intervals and by equal pouplations in each interval vs bad rate/target binary variable To get the feature deep dive feature distribution: 1. Upload a CSV (only numerical data) 2. Choose & press on the binary (0 / 1) target column in the `Select Target Variable` section below 3. Press Run Analysis 4. Wait few seconds and analyze the updated charts """ file_input = pn.widgets.FileInput(align='center') selector = pn.widgets.MultiSelect(name='Select Target Variable') button = pn.widgets.Button(name='Run Analysis') widgets = pn.WidgetBox( pn.panel(text, margin=(0, 10)), pn.panel('Upload a CSV containing (X) features and (y) binary variable:', margin=(0, 10)), file_input, selector, button ) def closest(lst, K): try: return lst[min(range(len(lst)), key = lambda i: abs(lst[i]-K))] except: return K control_max = lambda x: x.max() * 1.01 if x.max() > 0 else (x.max() * 0.99 if x.max() < 0 else x.max() + 0.01) control_min = lambda x: x.min() * 0.99 if x.min() > 0 else (x.min() * 1.01 if x.min() < 0 else x.min() - 0.01) def get_data(): global target, New_Refit_routing if file_input.value is None: New_Refit_routing = pd.DataFrame({"Open_accounts": np.random.randint(1,50,100000), "Income": np.random.randint(1000,20000,100000), "Years_of_experience": np.random.randint(0,20,100000), "default": np.random.random_integers(0,1,100000)}) target = "default" else: New_Refit_routing = BytesIO() New_Refit_routing.write(file_input.value) New_Refit_routing.seek(0) try: New_Refit_routing = pd.read_csv(New_Refit_routing, error_bad_lines=False).apply(pd.to_numeric, errors='ignore')#.set_index("id") except: New_Refit_routing = pd.read_csv(New_Refit_routing, error_bad_lines=False) target = None New_Refit_routing = New_Refit_routing.select_dtypes(exclude=['datetime', "category","object"]) New_Refit_routing = New_Refit_routing.replace([np.inf, -np.inf], np.nan) # New_Refit_routing = New_Refit_routing[[cols for cols in New_Refit_routing.columns if New_Refit_routing[cols].nunique() >= 2]] #remove columns with less then 2 unique values return target, New_Refit_routing def update_target(event): _ , New_Refit_routing = get_data() target = list(New_Refit_routing.columns) selector.set_param(options=target, value=target) file_input.param.watch(update_target, 'value') update_target(None) def stats_(): global stats stats = New_Refit_routing.describe().T stats["Missing_Values(%)"] = (New_Refit_routing.isna().sum() / len(New_Refit_routing)) * 100 stats = pd.concat([stats, New_Refit_routing.quantile(q = [.01, .05, .95, .99]).T.rename(columns = {0.01: '1%', 0.05: '5%', 0.95: '95%', 0.99:'99%'})], axis = 1) stats = stats[['count', 'mean', 'std', 'min', '1%', '5%' ,'25%', '50%', '75%', '95%', '99%', 'max','Missing_Values(%)']] stats = stats.round(4).astype(str) def cuts_(target): global test, test2, final_df , outlier_removed_stats df = New_Refit_routing.copy() neglect = [target] + [cols for cols in df.columns if df[cols].nunique() <= 2] #remove binary and target variable cols = df.columns.difference(neglect) # Getting all columns except the ones in [] #REMOVE OUTIERS# df[cols] = df[cols].apply(lambda col: col.clip(lower = col.quantile(.01), upper = closest(col[col < col.quantile(.99)].dropna().values, col.quantile(.99))),axis = 0) outlier_removed_stats = df.describe().T remove_feature = list(outlier_removed_stats[(outlier_removed_stats["mean"]==outlier_removed_stats["max"]) & (outlier_removed_stats["mean"]==outlier_removed_stats["min"])].index) outlier_removed_stats = pd.concat([outlier_removed_stats, df.quantile(q = [.01, .05, .95, .99]).T.rename(columns = {0.01: '1%', 0.05: '5%', 0.95: '95%', 0.99:'99%'})], axis = 1) outlier_removed_stats = outlier_removed_stats[['count', 'mean', 'std', 'min', '1%', '5%' ,'25%', '50%', '75%', '95%', '99%', 'max']] outlier_removed_stats = outlier_removed_stats.round(4).astype(str) neglect += remove_feature cols = df.columns.difference(neglect) # Getting all columns except the ones in [] df[cols] = df[cols].apply(lambda col: pd.cut(col.fillna(np.nan), bins = pd.interval_range(start=float(np.apply_along_axis(control_min , 0,col.dropna())), end = float(np.apply_along_axis(control_max , 0,col.dropna())), periods = 10), include_lowest=True).cat.add_categories(pd.Categorical(f"Missing_{col.name}")).fillna(f"Missing_{col.name}"), axis=0) test = pd.concat([df[cols].value_counts(normalize = True) for cols in df[cols]], axis = 1) cols = test.columns test = test.reset_index().melt(id_vars="index", var_name='column', value_name='value').dropna().reset_index(drop = True) test = test.rename(columns={"index":"IntervalCuts", "column":"feature", "value":"Count_Pct"}) test.Count_Pct = test.Count_Pct.round(4) test.IntervalCuts = test.IntervalCuts.astype(str) test.IntervalCuts = test.IntervalCuts.apply(lambda x: "("+str(round(float(x.split(",")[0].strip("(")),4)) +', ' + str(round(float(x.split(",")[-1].strip("]")),4)) +"]" if (x.split(",")[0].strip("(").strip("-")[0]).isdigit() else x) test2 = pd.concat([df.groupby(col)[target].mean().fillna(0) for col in df[cols]], axis = 1) test2.columns = cols test2 = test2.reset_index().melt(id_vars="index", var_name='column', value_name='value').dropna().reset_index(drop = True) test2 = test2.rename(columns={"index":"IntervalCuts", "column":"feature", "value":"Bad_Rate_Pct"}) test2.Bad_Rate_Pct = test2.Bad_Rate_Pct.round(4) test2.IntervalCuts = test2.IntervalCuts.astype(str) test2.IntervalCuts = test2.IntervalCuts.apply(lambda x: "("+str(round(float(x.split(",")[0].strip("(")),4)) +', ' + str(round(float(x.split(",")[-1].strip("]")),4)) +"]" if (x.split(",")[0].strip("(").strip("-")[0]).isdigit() else x) test["index"] = test["feature"] + "_" + test["IntervalCuts"] test = test.set_index("index").sort_index() test2["index"] = test2["feature"] + "_" + test2["IntervalCuts"] test2 = test2.set_index("index").sort_index() final_df = pd.merge(test2, test[test.columns.difference(test2.columns)], on = "index") ## QCUT ## def qcuts_(target): global test_q, test2_q, final_df_q df2 = New_Refit_routing.copy() neglect = [target] + [cols for cols in df2.columns if df2[cols].nunique() <= 2] #remove binary and target variable cols = df2.columns.difference(neglect) # Getting all columns except the ones in [] #DEBUGGING CODE##################################################################################### # for i in df2[cols].columns: # print(i) # print(df2[i][df2[i] < df2[i].quantile(.99)].dropna().values) # print(df2[i].quantile(.99)) # print(closest(df2[i][df2[i] < df2[i].quantile(.99)].dropna().values, df2[i].quantile(.99))) # df2.apply(lambda col: col.clip(lower = col.quantile(.01), # upper = closest(col[col < col.quantile(.99)].dropna().values, # col.quantile(.99))),axis = 0) #################################################################################################### #REMOVE OUTIERS# df2[cols] = df2[cols].apply(lambda col: col.clip(lower = col.quantile(.01), upper = closest(col[col < col.quantile(.99)].dropna().values, col.quantile(.99))),axis = 0) temp = df2.describe().T remove_feature = list(temp[(temp["mean"]==temp["max"]) & (temp["mean"]==temp["min"])].index) neglect+= remove_feature cols = df2.columns.difference(neglect) # Getting all columns except the ones in [] # rank(method='first') is a must in qcut # df2[cols] = df2[cols].apply(lambda col: pd.qcut(col.fillna(np.nan).rank(method='first'), # q = 10, duplicates = "drop").cat.add_categories(pd.Categorical(f"Qcut_Missing_{col.name}")).fillna(f"Qcut_Missing_{col.name}"), axis=0) df2[cols] = df2[cols].apply(lambda col: pd.qcut(col.fillna(np.nan).rank(method='first'),q = 10, labels=range(1,11)).cat.rename_categories({10:"Last"}).astype(str).replace(dict(dict(pd.concat([col, pd.qcut(col.fillna(np.nan).rank(method='first'),q = 10, labels=range(1,11)).cat.rename_categories({10:"Last"}) .apply(str)], axis = 1, keys= ["feature", "qcuts"]).groupby("qcuts").agg([min, max]).reset_index().astype(str).set_index("qcuts",drop = False) .apply(lambda x :x[0]+"_"+"("+str(round(float(x[1]),2))+","+str(round(float(x[2]),2))+"]",axis = 1)),**{"nan":f"Qcut_Missing_{col.name}"})), axis=0) test_q = pd.concat([df2[cols].value_counts(normalize = True) for cols in df2[cols]], axis = 1) cols = test_q.columns test_q = test_q.reset_index().melt(id_vars="index", var_name='column', value_name='value').dropna().reset_index(drop = True) test_q = test_q.rename(columns={"index":"IntervalCuts", "column":"feature", "value":"Count_Pct"}) test_q.Count_Pct = test_q.Count_Pct.round(4) test_q.IntervalCuts = test_q.IntervalCuts.astype(str) # test_q.IntervalCuts = test_q.IntervalCuts.apply(lambda x: "("+str(round(float(x.split(",")[0].strip("(")),4)) +', ' + str(round(float(x.split(",")[-1].strip("]")),4)) +"]" if (x.split(",")[0].strip("(")[0]).isdigit() else x) test2_q = pd.concat([df2.groupby(col)[target].mean().fillna(0) for col in df2[cols]], axis = 1) test2_q.columns = cols test2_q = test2_q.reset_index().melt(id_vars="index", var_name='column', value_name='value').dropna().reset_index(drop = True) test2_q = test2_q.rename(columns={"index":"IntervalCuts", "column":"feature", "value":"Bad_Rate_Pct"}) test2_q.Bad_Rate_Pct = test2_q.Bad_Rate_Pct.round(4) test2_q.IntervalCuts = test2_q.IntervalCuts.astype(str) # test2_q.IntervalCuts = test2_q.IntervalCuts.apply(lambda x: "("+str(round(float(x.split(",")[0].strip("(")),4)) +', ' + str(round(float(x.split(",")[-1].strip("]")),4)) +"]" if (x.split(",")[0].strip("(")[0]).isdigit() else x) test_q["index"] = test_q["feature"] + "_" + test_q["IntervalCuts"] test_q = test_q.set_index("index").sort_index() test2_q["index"] = test2_q["feature"] + "_" + test2_q["IntervalCuts"] test2_q = test2_q.set_index("index").sort_index() final_df_q = pd.merge(test2_q, test_q[test_q.columns.difference(test2_q.columns)], on = "index") @pn.depends(button.param.clicks) def run(_): target, New_Refit_routing = get_data() if target == None: target = str(selector.value[0]) else: target = "default" print(str(selector.value[0])) print(target) # print(type(file_input.value)) # print(type(New_Refit_routing)) print(New_Refit_routing.head()) stats_() cuts_(target) qcuts_(target) test2_plot = test2.set_index("IntervalCuts").hvplot.scatter(yaxis = "left", y = "Bad_Rate_Pct", groupby = "feature", xlabel = "Intervals(Bins)", ylabel = "%Count vs %BadRate",height = 500, width = 1000, title = "Features Segments Cuts by Count", legend = True,label = "Bad Rate(%)").opts(xrotation=45, yformatter = "%.04f",show_grid=True, framewise=True, color = "red", legend_position='top_right') test_plot = test.set_index("IntervalCuts").hvplot.bar(y = "Count_Pct", groupby = "feature", xlabel = "Intervals(Bins)", ylabel = "%Count vs %BadRate",height = 500, width = 1000, title = "Features Segments Cuts by Count", legend=True, alpha=0.3, label ="Equal Intervals Data Points(%)").opts(xrotation=45, yformatter = "%.04f",show_grid=True, framewise=True, yaxis='left') final_table = final_df.hvplot.table(groupby = "feature", width=400) test2_plot_q = test2_q.set_index("IntervalCuts").hvplot.scatter(yaxis = "left", y = "Bad_Rate_Pct", groupby = "feature", xlabel = "Intervals(Bins)", ylabel = "%Count vs %BadRate",height = 500, width = 1000, title = "Features Segments Q_Cuts by Count", legend = True).opts(xrotation=45, yformatter = "%.04f",show_grid=True, framewise=True, color = "red") test_plot_q = test_q.set_index("IntervalCuts").hvplot.bar(y = "Count_Pct", groupby = "feature", xlabel = "Intervals(Bins)", ylabel = "%Count vs %BadRate",height = 500, width = 1000, title = "Features Segments Q_Cuts by Count", legend=True, alpha=0.3, label ="Equal Population Data Points(%)").opts(xrotation=45, yformatter = "%.04f",show_grid=True, framewise=True, yaxis='left') final_table_q = final_df_q.hvplot.table(groupby = "feature", width=400) stats_table = stats.reset_index().hvplot.table(width = 1000,title="Summary Statistics of the Data", hover = True, responsive=True, shared_axes= False, fit_columns = True, padding=True, height=500, index_position = 0, fontscale = 1.5) stats_table_no_outliers = outlier_removed_stats.reset_index().hvplot.table(width = 1000,title="Summary Statistics of the Capped Outliers Data", hover = True, responsive=True, shared_axes= False, fit_columns = True, padding=True, height=500, index_position = 0, fontscale = 1.5) #PANEL pn.extension( template="fast") pn.state.template.param.update( # site_url="", site="CreditRisk", title="Feature Distribution & Statistics", # favicon="https://raw.githubusercontent.com/opploans/DS_modelling_tools/main/docs/Resources/favicon.ico?token=GHSAT0AAAAAABYR5F6VDZ2PU33UY6NN7NQEY3C2ASA" # favicon="", ) title = pn.pane.Markdown( """ ### Feature Distribution (Bin Count & Bad Rate) """, width=800, ) return pn.Column( title, (test2_plot * test_plot * test2_plot_q * test_plot_q + (final_table + final_table_q)).cols(3), (stats_table + stats_table_no_outliers).cols(2), ) profiles = ''' ### Other Web Apps: * [Twitter Sentiment Analysis Flask App](https://firobeid.pythonanywhere.com/) * [Personal Lectures @ UCBerkley Using Panel App](https://firobeid.github.io/compose-plots/script.html) ''' pn.Row(pn.Column(widgets, profiles), pn.layout.Spacer(width=20), run).servable(target='main')