allmodels.py

# -*- coding: utf-8 -*-
"""AllModels.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Bi5CewIiaR1CPcEG2D7FdAxziJwbc6mD

**Models included Random Forest Classifier, Adaboost Classifier, Xgboost Classifier, LGBM Classifier, LGBM K-Fold, CatBoost Classifier, Stacking**

# **Preprocessing**
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix
import time

train=pd.read_csv('../input/project/train.csv/train.csv')
test=pd.read_csv('../input/project/test.csv/test.csv')
test1=pd.read_csv('../input/project/test.csv/test.csv')

# deleting the columns with more than 50% Null values
col_drop=['ps_car_03_cat', 'ps_car_05_cat']
train.drop(col_drop, inplace=True, axis=1)
test.drop(col_drop, inplace=True, axis=1)

train = train.replace(-1, np.NaN)
train.isna().sum().sort_values(ascending=False).head(15)

null_values_train=['ps_reg_03','ps_car_14','ps_car_07_cat','ps_ind_05_cat','ps_car_09_cat','ps_ind_02_cat','ps_car_01_cat','ps_ind_04_cat','ps_car_02_cat','ps_car_11']

#Filling null values with the mean for train data
for col in null_values_train:
    train[col] = train[col].fillna((train[col].mean()))

test=test.replace(-1, np.NaN)
test.isna().sum().sort_values(ascending=False).head(15)

null_values_test=['ps_reg_03','ps_car_14','ps_car_07_cat','ps_ind_05_cat','ps_car_09_cat','ps_ind_02_cat','ps_car_01_cat','ps_ind_04_cat','ps_car_02_cat','ps_car_11','ps_car_12']

#Filling null values with the mean for test data
for col in null_values_test:
    test[col] = test[col].fillna((test[col].mean()))

train.info()

categorical_col=['ps_ind_02_cat','ps_ind_04_cat','ps_ind_05_cat','ps_car_01_cat','ps_car_02_cat','ps_car_04_cat','ps_car_06_cat',
 'ps_car_07_cat','ps_car_08_cat','ps_car_09_cat','ps_car_10_cat','ps_car_11_cat']

#Performing Label Encoding
le=LabelEncoder()
for col in categorical_col:
  train[col]=le.fit_transform(train[col])

for col in categorical_col:
  test[col]=le.fit_transform(test[col])

train_new=train
test_new=test

col_drop=['id']
train_new.drop(col_drop, inplace=True, axis=1)
test_new.drop(col_drop, inplace=True, axis=1)

#When using test-train split we will use these datasets
X = train_new.drop(['target'],axis=1)
y= train_new['target']

X_out=test_new

#When training the models on the complete data we will use these datasets
X_train1 = train_new.drop(['target'],axis=1)
Y_train1= train_new['target']

import gc
gc.collect()

#Defined Gini Function to get an idea of the gini score we might get by a model
def ginic(actual,pred):
    actual=np.asarray(actual)
    n=len(actual)
    a_s=actual[np.argsort(pred)]
    a_c=a_s.cumsum()
    ginisum=a_c.sum() / a_s.sum() - (n+1) / 2.0
    return ginisum / n

def gini_normalized(a,p):
    if p.ndim == 2:
        p = p[:,1]
    return ginic(a,p) / ginic(a,a)

# create a 80/20 split of the data 
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.2)

"""# **Random Forest Classifier**"""

from sklearn.ensemble import RandomForestClassifier

#creating object and then fitting on the test_train split data
rf = RandomForestClassifier(n_estimators=3000,max_depth=7)
rf.fit(xtrain, ytrain)

rf_predictions = rf.predict_proba(xvalid)[:, 1]
print(roc_auc_score(yvalid, rf_predictions))
# get accuracy score (predict the class)
rf_predictions_class = rf.predict(xvalid)
print(accuracy_score(yvalid, rf_predictions_class, normalize=True))
# get the gini scoe
score=gini_normalized(yvalid,rf_predictions)
print(score)

print('Confusion matrix\n',confusion_matrix(yvalid,rf_predictions_class))

#After analysis of the gini score obtained and confusion matrix fitting model on complete dataset
rf.fit(X_train1, Y_train1)

result = rf.predict_proba(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result[:,1]})
submit=submit[['id','target']]

#Creating the CSV file
submit.to_csv("Random Forest Classifier.csv", index = False)
submit.head(10)

"""# **AdaBoost Classifier**"""

from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier

#creating object and then fitting on the test_train split data using DecisionTreeClassifier as base
#estimator
abc = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),n_estimators=2000)
abc.fit(xtrain, ytrain)

abc_predictions = abc.predict_proba(xvalid)[:, 1]
print(roc_auc_score(yvalid, abc_predictions))
# get accuracy score (predict the class)
abc_predictions_class = abc.predict(xvalid)
print(accuracy_score(yvalid, abc_predictions_class, normalize=True))
score=gini_normalized(yvalid,rf_predictions)
print(score)

print('Confusion matrix\n',confusion_matrix(y_test,abc_predictions_class))

#After analysis of the gini score obtained and confusion matrix fitting model on complete dataset
abc.fit(X_train1, Y_train1)

result = abc.predict_proba(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result[:,1]})
submit=submit[['id','target']]

#Creating the CSV file
submit.to_csv("Adaboost Classifier.csv", index = False)
submit.head(10)

"""We also tried AdaBoost Classifier with SVC as base using the following code but could not run it due to time contraints of Kaggle."""

from sklearn.svm import SVC
svc=SVC(probability=True, kernel='linear')

abc1 =AdaBoostClassifier(n_estimators=3000, base_estimator=svc,learning_rate=1, random_state=0)
# train adaboost classifer
abc1.fit(xtrain, ytrain)

"""# **XGBoost**"""

import xgboost as xgb

clf_xgb = xgb.XGBClassifier(learning_rate=0.01, 
                            n_estimators=3000, 
                            max_depth=4,                    
                            subsample=0.9,
                            colsample_bytree=0.6,
                            objective= 'binary:logistic',
                            reg_alpha = 0,
                            reg_lambda = 1,
                            seed=42)

#creating object and then fitting on the test_train split data
clf_xgb.fit(xtrain, ytrain, eval_set=[(xtrain, ytrain), (xvalid, yvalid)], 
            early_stopping_rounds=100, eval_metric='auc', verbose=100)

predictions = clf_xgb.predict(xvalid)
print("accuracy_score", accuracy_score(yvalid, predictions))
predictions_probas = clf_xgb.predict_proba(xvalid)
print("roc-auc score", roc_auc_score(yvalid, predictions_probas[:,1]))
#print gini score
score=gini_normalized(yvalid,predictions_probas)
print(score)
gc.collect()

print('Confusion matrix\n',confusion_matrix(yvalid,predictions))

#After analysis of the gini score obtained and confusion matrix fitting model on complete dataset
clf_xgb.fit(X_train1, Y_train1)
print()
gc.collect()

result = clf_xgb.predict_proba(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result[:,1]})
submit=submit[['id','target']]

#Creating .CSV file
submit.to_csv("Xgboost Classifier.csv", index = False)
submit.head(10)

"""# **LGBM Classifier**"""

import lightgbm as lgb

clf_lgb=lgb.LGBMClassifier(objective= 'binary',
                           max_depth= 4,
                           n_estimators=2000,
                           learning_rate= 0.01,         
                           feature_fraction= 0.8,
                           bagging_freq= 1,
                           bagging_fraction= 0.8 ,
                           bagging_seed= 1,         
                           lambda_l1= 0.1,        
                           verbosity= -1)

#creating object and then fitting on the test_train split data
clf_lgb.fit(xtrain, ytrain, eval_set=[(xtrain, ytrain), (xvalid, yvalid)], 
            early_stopping_rounds=100, eval_metric='auc', verbose=100)

predictions = clf_lgb.predict(xvalid)
print("accuracy_score", accuracy_score(yvalid, predictions))
predictions_probas = clf_lgb.predict_proba(xvalid)
print("roc-auc score", roc_auc_score(yvalid, predictions_probas[:,1]))
#print gini score
score=gini_normalized(yvalid,predictions_probas)
print(score)
gc.collect()

print('Confusion matrix\n',confusion_matrix(yvalid,predictions))

#After analysis of the gini score obtained and confusion matrix fitting model on complete dataset
clf_lgb.fit(X_train1, Y_train1)
print()
gc.collect()

result = clf_lgb.predict_proba(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result[:,1]})
submit=submit[['id','target']]

#Creating .CSV file
submit.to_csv("LGBM Classifier.csv", index = False)
submit.head(10)

"""# **LGBM with K-fold**"""

from sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit
import lightgbm as lgb

n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=15)

params = { 'objective': 'binary',
         'max_depth': 4,
         'learning_rate': 0.01,         
         "feature_fraction": 0.8,
         "bagging_freq": 1,
         "bagging_fraction": 0.8 ,
         "bagging_seed": 1,         
         "lambda_l1": 0.1,         
         "verbosity": -1 }

scores = []
for fold_n, (train_index, valid_index) in enumerate(folds.split(X, y)):
    gc.collect()
    print('Fold', fold_n + 1, 'started at', time.ctime())
    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]        
    
    train_data = lgb.Dataset(X_train, label=y_train)
    valid_data = lgb.Dataset(X_valid, label=y_valid)
            
    model = lgb.train(params,
    train_data,
    num_boost_round=2000,
    valid_sets = [train_data, valid_data],
    verbose_eval=300,
    early_stopping_rounds = 100)
    del train_data, valid_data
            
    y_pred_valid = model.predict(X_valid)

print('Fold roc_auc:', roc_auc_score(y_valid, y_pred_valid))
score=gini_normalized(y_valid,y_pred_valid)
print(score)

#After analysis of the gini score obtained and confusion matrix fitting model on complete dataset
train_data = lgb.Dataset(X, label=y)
model = lgb.train(params,train_data,
    num_boost_round=2000,
    verbose_eval=300)

result = model.predict(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result})
submit=submit[['id','target']]

#Creating CSV file
submit.to_csv("LGBM with K-fold.csv", index = False)
submit.head(10)

"""# **CatBoost Classifier**"""

cat_feature=['ps_ind_02_cat','ps_ind_04_cat','ps_ind_05_cat','ps_car_01_cat','ps_car_02_cat','ps_car_04_cat','ps_car_06_cat',
 'ps_car_07_cat','ps_car_08_cat','ps_car_09_cat','ps_car_10_cat','ps_car_11_cat']

from catboost import CatBoostClassifier

clf_cat = CatBoostClassifier(
    iterations=2000,
    learning_rate=0.01,
    
)

#fitting model on test-train split data
clf_cat.fit(xtrain, ytrain, 
        cat_features=cat_feature, 
        eval_set=(xvalid, yvalid),
        early_stopping_rounds=100,
        verbose=False
)

print('CatBoost model is fitted: ' + str(clf_cat.is_fitted()))
print('CatBoost model parameters:')
print(clf_cat.get_params())

predictions = clf_cat.predict(xvalid)
print("accuracy_score", accuracy_score(yvalid predictions))
predictions_probas = clf_cat.predict_proba(xvalid)
score=gini_normalized(yvalid,predictions_probas)
print(score)

print('Confusion matrix\n',confusion_matrix(yvalid,predictions))

#completely training the whole train dataset after analysing the gini index value
clf_cat.fit(X_train1, Y_train1,cat_features=cat_feature,verbose=False)

result = clf_cat.predict_proba(X_out)[:,1]
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result})
submit=submit[['id','target']]

#Creating CSV file
submit.to_csv("Catboost Classifier.csv", index = False)
submit.head(10)

"""# **Stacking**"""

from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression

#defining the models that will be used later
clf_xgb = xgb.XGBClassifier(learning_rate=0.01, 
                            n_estimators=3000, 
                            max_depth=4,                    
                            subsample=0.9,
                            colsample_bytree=0.6,
                            objective= 'binary:logistic',
                            reg_alpha = 0,
                            reg_lambda = 1,
                            seed=42,
                            tree_method='gpu_hist')

clf_lgb=lgb.LGBMClassifier(objective= 'binary',
                           max_depth= 4,
                           n_estimators=2000,
                           learning_rate= 0.01,         
                           feature_fraction= 0.8,
                           bagging_freq= 1,
                           bagging_fraction= 0.8 ,
                           bagging_seed= 1,         
                           lambda_l1= 0.1,        
                           verbosity= -1)

clf_cat = CatBoostClassifier(iterations=2000, 
                             learning_rate=0.01,
                             task_type="GPU",
)

"""***Used Three Variants of Stacking by changing the base estimators and final estimators ***"""

#Base estimator XGBoost and LGBM Classifier, final estimator Logistic Regression  
estimators = [('xgb',clf_xgb),('lgb',clf_lgb)]
clf_stacking1 = StackingClassifier(estimators=estimators,final_estimator=LogisticRegression())

#Base estimator XGBoost, LGBM Classifier and CatBoost, final estimator Logistic Regression
estimators = [('xgb',clf_xgb),('lgb',clf_lgb),('catboost',clf_cat)]
clf_stacking2 = StackingClassifier(estimators=estimators,final_estimator=LogisticRegression())

#Base estimator Catboost and LGBM Classifier, final estimatorXgboost
estimators = [('catboost',clf_cat),('lgb',clf_lgb)]
clf_stacking3 = StackingClassifier(estimators=estimators,final_estimator=clf_xgb)

#for all the above three stacking models trained the data individually
clf_stacking1.fit(xtrain,ytrain)

predictions_probas = clf_stacking1.predict_proba(xvalid)
score=gini_normalized(yvalid,predictions_probas)
print(score)

print('Confusion matrix\n',confusion_matrix(yvalid,predictions))

#After analysing gini score fit the model on complete train data
clf_stacking1.fit(X_train1,Y_train1)

result = clf_stacking.predict_proba(X_out)
result

id=test1['id']
submit=pd.DataFrame({'id':id,'target':result1[:,1]})
submit=submit[['id','target']]

#We created three seperate Stacking.CSV files for each variant of stacking mentioned above
submit.to_csv("Stacking.csv", index = False)
submit.head(10)

"""**These are all the models that we tried**"""