week5_assi3_sol1.r

# -*- coding: utf-8 -*-
# """Week5 Assi3 Sol1.ipynb
# 
# Automatically generated by Colaboratory.
# 
# Original file is located at
#     https://colab.research.google.com/drive/1lgkdooOU5Vg6kMYuoyzdjfHvtZb3jkOo
# """


###########################################################################
## Week-6, Homework-3, Sol-1
## Sreya Dhar 
## Created: Oct 02, 2020
## Edited: Oct 14, 2020
###########################################################################

# rm(list = ls()) ## clearing working environment

## installing all the libaries in R kernel

# install.packages("corrplot")
# install.packages("forecast")
# install.packages("zoo")
# install.packages("rsample")
# install.packages("leaps")
# install.packages("car")
# install.packages("caret")
# install.packages("ROCR")
# install.packages("PerformanceAnalytics")
# install.packages("funModeling")
# install.packages("hrbrthemes")
# install.packages("ggthemes")
# install.packages("GGally")
# install.packages("glmnet")

## importing the libraries in R kernel

library(ggplot2)
library(dplyr)
library(tidyverse)
library(tidyr)
library(corrplot)
library(repr)
library(reshape2)
library(forecast)
library(zoo)
library(rsample)
library(gplots)
library(ROCR)
library(class)
library(readr)
library(leaps)
library(car)
library(PerformanceAnalytics)
library(funModeling)
library(caret)
library(MASS)
library(Hmisc)
library(hrbrthemes)
library(GGally)
library(glmnet)
library(pROC)


# Set working directory to where data file is located
setwd("C:/File E/EAS 506 Statistical Mining I/Week 5/Assignment-3")

# uploading the data
train_data <-read.table('ticdata2000.txt')
test_data_X <-read.table('ticeval2000.txt')
test_data_Y <- read.table('tictgts2000.txt')
test_data_X["V86"] <- test_data_Y
test_data <- test_data_X

head(train_data)

names(train_data)

glimpse(train_data)

status(train_data)

profiling_num(train_data)

status(train_data)

# describe(train_data)
# summary(train_data)

data_lm = lm(V86~., data = train_data)
summary(data_lm)

options(repr.plot.width=6, repr.plot.height=6, repr.plot.res = 180)
par(mfrow=c(2,2))
plot(data_lm)

# # Other useful functions
# coefficients(data_lm) # model coefficients
# confint(data_lm, level=0.95) # CIs for predictors
# fitted(data_lm) # predicted values
# residuals(data_lm) # residuals
# anova(data_lm) # anova table
# vcov(data_lm) # covariance matrix for variables
# influence(data_lm) # linear regression diagnostics

anova(data_lm)['Residuals', 'Mean Sq'] # MSE calculation from anova table
sigma(data_lm) # residual standard deviation

head(test_data)

## predict the model on train set
fitted_model_tr <- predict(data_lm, train_data, type='response')
fitted_results_tr <- ifelse(fitted_model_tr > 0.5,1,0)
error_tr_lm <- mean(fitted_results_tr != train_data$V86) ## error on train set
print(1-error_tr_lm) ## Accuracy in train set
print('Error of train set from linear regression')
error_tr_lm # TRAINING ERROR

print('accuracy of train set from linear regression')
1-error_tr_lm # train accuracy

## predict the model on test set
fitted_model_1 <- predict(data_lm, test_data, type='response')
fitted_results <- ifelse(fitted_model_1 > 0.5,1,0)
error_te_lm <- mean(fitted_results != test_data$V86) ## error on test set
print('Error of test set from linear regression')
error_te_lm # test error
print('accuracy of test set from linear regression')
1-error_te_lm

## creating confusion matrix
cfmat <- tibble("target test" = test_data$V86,
                     "prediction test" = fitted_results)
cfm_tab <-table(cfmat)
cfm_tab # test confu. matrix
confusionMatrix(as.factor(fitted_results),as.factor(test_data$V86))

cfmat_tr <- tibble("target train" = train_data$V86,
                     "prediction train" = fitted_results_tr)
cfm_tab_tr <-table(cfmat_tr)
cfm_tab_tr # train confu. matrix
confusionMatrix(as.factor(fitted_results_tr),as.factor(train_data$V86))

## ROC AUC plot
options(repr.plot.width=3, repr.plot.height=3, repr.plot.res = 180)
pred_1 <- prediction(fitted_model_1, test_data$V86)
perform <- performance(pred_1, measure = "tpr", x.measure = "fpr")

options(repr.plot.width=4, repr.plot.height=4, repr.plot.res = 180)
plot(perform, ylim=c(0,1)) ## plot ROC
abline(a=0, b=1, type='l', lwd=1, lty=2)

### ROC AUC plot for lm 
roc_cur <- roc(test_data$V86, fitted_model_1) ## AUC ROC
auc(roc_cur) ## roc auc

auc_1 <- performance(pred_1, measure = "auc")
auc_lm <- auc_1@y.values[[1]]
auc_lm ## accuracy from linear reg on test set

################################ Backward subsets selection ####################################
data_back <- regsubsets(V86~., data= train_data, nvmax = 85, method = "backward")
back_sum <- summary(data_back)

# # names of the 85 selected variables
# back_sum$outmat[85,]

# # Structure of the best 85 variable model
# back_sum$outmat

# # Look at the regression models determined by the different methods
data.frame(coef(data_back,85))

#How many variables are needed for the best model fit.
data.frame(
  Adj.R2 = which.max(back_sum$adjr2),
  CP = which.min(back_sum$cp),
  BIC = which.min(back_sum$bic),
  RSS = which.min(back_sum$rss)
  )

## comparison for statistical parameters from backward selection
options(repr.plot.width=6, repr.plot.height=6, repr.plot.res = 200)
## Adjusted R2
par(mfrow = c(2,2))
plot(back_sum$cp, xlab = "Number of Variables", ylab = "Mallow's Cp", type = "l")
points(x= 1:85, y=back_sum$cp, col="red",cex=1,pch=20)
abline(v=which.min(back_sum$cp), y=min(back_sum$cp),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(back_sum$cp), h=min(back_sum$cp),  type = "l", col = "blue", lty = 3) 

plot(back_sum$bic, xlab = "Number of Variables", ylab = "BIC", type = "l")
points(x= 1:85, y=back_sum$bic, col="red",cex=1,pch=20)
abline(v=which.min(back_sum$bic), y=min(back_sum$bic),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(back_sum$bic), h=min(back_sum$bic),  type = "l", col = "blue", lty = 3) 

plot(back_sum$rss, xlab = "Number of Variables", ylab = "RSS", type = "l")
points(x= 1:85, y=back_sum$rss, col="red",cex=1,pch=20)
abline(v=which.min(back_sum$rss), y=min(back_sum$rss),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(back_sum$rss), h=min(back_sum$rss),  type = "l", col = "blue", lty = 3) 

plot(back_sum$adjr2, xlab = "Number of Variables", ylab = "Adjusted R^2", type = "l")
points(x= 1:85, y=back_sum$adjr2, col="red",cex=1,pch=20)
abline(v=which.max(back_sum$adjr2), y=max(back_sum$adjr2),  type = "l", col = "blue", lty = 3) 
abline(x=which.max(back_sum$adjr2), h=max(back_sum$adjr2),  type = "l", col = "blue", lty = 3)

options(repr.plot.width=10, repr.plot.height=10, repr.plot.res = 200)
par(mfrow = c(2,2))
plot(data_back, scale = "r2", main = "R^2")
plot(data_back, scale = "adjr2", main = "Adjusted R^2")
plot(data_back, scale = "Cp",main = "Cp" )
plot(data_back, scale = "bic", main = "BIC")

## prediction on train and test set for backward selection for model selection
test_mse_back = rep(NA, 85)
train_mse_back = rep(NA, 85)

new_test_back = model.matrix(V86 ~., data=test_data)
new_train_back = model.matrix(V86 ~., data=train_data)

for (i in 1:85){
        coeffs_back = coef(data_back, id=i)
        pred_te_back = new_test_back[,names(coeffs_back)]%*%coeffs_back
        pred_tr_back = new_train_back[,names(coeffs_back)]%*%coeffs_back
        test_mse_back[i] = mean((test_data$V86-pred_te_back)^2) # predict on test 
        train_mse_back[i] = mean((train_data$V86-pred_tr_back)^2) # predict on train 
}

## mse plot from train and test prediction 

options(repr.plot.width=6, repr.plot.height=6, repr.plot.res = 250)
plot(test_mse_back, ylim= c(0.05,0.06), col='red', type = "l", xlab="subset size", ylab= "MSE from backward selection")
abline(v = which.min(test_mse_back),y = min(test_mse_back), type = "l", col = "red", lwd = 2, lty=2)
lines(train_mse_back, col= "blue", type = "l")
abline(v = which.min(train_mse_back),y = min(train_mse_back), type = "l", col = "blue", lwd = 2, lty=2)


legend("topright",inset=.02, c("Test Set", "Train Set"), lty= c(1,1), lwd=c(2.5,2.5),col= c("red", "blue"))

print(which.min(test_mse_back))
print(which.min(train_mse_back))

## getting confusion matrix for best subset model from backward selection on train set

k_back_tr= which.min(train_mse_back)
test_error_back = rep(NA, 85)
train_error_back = rep(NA, 85)
for (i in k_back_tr){
        coeffs_back = coef(data_back, id=i)
        pred_te_back = new_test_back[,names(coeffs_back)]%*%coeffs_back
        pred_tr_back = new_train_back[,names(coeffs_back)]%*%coeffs_back
        back_results_tr <- ifelse(pred_tr_back > 0.5,1,0)
        train_error_back[i] <- mean(back_results_tr != train_data$V86)*100
        back_results_te <- ifelse(pred_te_back > 0.5,1,0)
        test_error_back[i] <- mean(back_results_te != test_data$V86)*100
}

## creating confusion matrix 
cfmat_back_tr <- tibble("target train back" = train_data$V86,
                     "prediction train back" = back_results_tr)
cfm_tab_back_tr  <-table(cfmat_back_tr )
cfm_tab_back_tr  # test confu. matrix

## getting confusion matrix for best subset model from backward selection on test set
confusionMatrix(as.factor(back_results_tr),as.factor(train_data$V86))

## creating confusion matrix 
cfmat_back <- tibble("target test back" = test_data$V86,
                     "prediction test back" = back_results_te)
cfm_tab_back <-table(cfmat_back)
cfm_tab_back # test confu. matrix
confusionMatrix(as.factor(back_results_te),as.factor(test_data$V86))



################################ Forward subsets selection ####################################
data_for <- regsubsets(V86~., data= train_data, nvmax = 85, method = "forward")
for_sum <- summary(data_for)

# # names of the 14 selected variables
# for_sum$outmat[85,]

# # Structure of the best 9 variable model
# for_sum$outmat

# # Look at the regression models determined by the different methods
# data.frame(coef(data_for,85))

#How many variables are needed for the best model fit.
data.frame(
  Adj.R2 = which.max(for_sum$adjr2),
  CP = which.min(for_sum$cp),
  BIC = which.min(for_sum$bic),
  RSS = which.min(for_sum$rss)
  )






## comparison for statistical parameters from forward selection
options(repr.plot.width=6, repr.plot.height=6, repr.plot.res = 200)
## Adjusted R2
par(mfrow = c(2,2))
plot(for_sum$cp, xlab = "Number of Variables", ylab = "Mallow's Cp", type = "l")
points(x= 1:85, y=for_sum$cp, col="red",cex=1,pch=20)
abline(v=which.min(for_sum$cp), y=min(for_sum$cp),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(for_sum$cp), h=min(for_sum$cp),  type = "l", col = "blue", lty = 3) 

plot(for_sum$bic, xlab = "Number of Variables", ylab = "BIC", type = "l")
points(x= 1:85, y=for_sum$bic, col="red",cex=1,pch=20)
abline(v=which.min(for_sum$bic), y=min(for_sum$bic),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(for_sum$bic), h=min(for_sum$bic),  type = "l", col = "blue", lty = 3) 

plot(for_sum$rss, xlab = "Number of Variables", ylab = "RSS", type = "l")
points(x= 1:85, y=for_sum$rss, col="red",cex=1,pch=20)
abline(v=which.min(for_sum$rss), y=min(for_sum$rss),  type = "l", col = "blue", lty = 3) 
abline(x=which.min(for_sum$rss), h=min(for_sum$rss),  type = "l", col = "blue", lty = 3) 

plot(for_sum$adjr2, xlab = "Number of Variables", ylab = "Adjusted R^2", type = "l")
points(x= 1:85, y=for_sum$adjr2, col="red",cex=1,pch=20)
abline(v=which.max(for_sum$adjr2), y=max(for_sum$adjr2),  type = "l", col = "blue", lty = 3) 
abline(x=which.max(for_sum$adjr2), h=max(for_sum$adjr2),  type = "l", col = "blue", lty = 3)

options(repr.plot.width=10, repr.plot.height=10, repr.plot.res = 200)
par(mfrow = c(2,2))
plot(data_for, scale = "r2", main = "R^2")
plot(data_for, scale = "adjr2", main = "Adjusted R^2")
plot(data_for, scale = "Cp",main = "Cp" )
plot(data_for, scale = "bic", main = "BIC")

## prediction on train and test set from Forward selection for model selection
test_mse_for = rep(NA, 85)
train_mse_for = rep(NA, 85)

new_test_for = model.matrix(V86 ~., data=test_data)
new_train_for = model.matrix(V86 ~., data=train_data)

for (i in 1:85){
        coeffs_for = coef(data_for, id=i)
        pred_te_for = new_test_for[,names(coeffs_for)]%*%coeffs_for
        pred_tr_for = new_train_for[,names(coeffs_for)]%*%coeffs_for
        test_mse_for[i] = mean((test_data$V86-pred_te_for)^2) # predict on test 
        train_mse_for[i] = mean((train_data$V86-pred_tr_for)^2) # predict on train 
}

## mse plot from train and test prediction in forward selection

options(repr.plot.width=6, repr.plot.height=6, repr.plot.res = 250)
plot(test_mse_for, ylim= c(0.05,0.06), col='red', type = "l", xlab="subset size", ylab= "MSE from forward selection")
abline(v = which.min(test_mse_for),y = min(test_mse_for)*100, type = "l", col = "red", lwd = 2, lty=2)
lines(train_mse_for, col= "blue", type = "l")
abline(v = which.min(train_mse_for),y = min(train_mse_for)*100, type = "l", col = "blue", lwd = 2, lty=2)


legend("topright",inset=.02, c("Test Set", "Train Set"), lty= c(1,1), lwd=c(2.5,2.5),col= c("red", "blue"))

print(which.min(test_mse_for))
print(which.min(train_mse_for))

# test_error_for[which.min(test_error_for)]
test_mse_for[which.min(test_mse_for)]

## getting confusion matrix for best subset model from forward selection train set

k= which.min(test_mse_for)
test_error_for = rep(NA, 85)
train_error_for = rep(NA, 85)

for (i in k){
        coeffs_for = coef(data_for, id=i)
        pred_te_for = new_test_for[,names(coeffs_for)]%*%coeffs_for
        pred_tr_for = new_train_for[,names(coeffs_for)]%*%coeffs_for
        for_results_tr <- ifelse(pred_tr_for > 0.5,1,0)
        train_error_for[i] <- mean(for_results_tr != train_data$V86)*100
        for_results_te <- ifelse(pred_te_for > 0.5,1,0)
        test_error_for[i] <- mean(for_results_te != test_data$V86)*100
}

## creating confusion matrix 
cfmat_for <- tibble("target test for" = test_data$V86,
                     "prediction test for" = for_results_te)
cfm_tab_for <-table(cfmat_for)
cfm_tab_for # test confu. matrix

confusionMatrix(as.factor(for_results_te),as.factor(test_data$V86))

## getting confusion matrix for best subset model from forward selection on train set

## creating confusion matrix 
cfmat_for_tr <- tibble("target test for" = train_data$V86,
                     "prediction test for" = for_results_tr)
cfm_tab_for_tr <-table(cfmat_for_tr)
cfm_tab_for_tr # test confu. matrix

## confusion matrix from "caret" library
confusionMatrix(as.factor(for_results_tr),as.factor(train_data$V86))

###############################################################################################
################################ Ridge Regression #############################################
###############################################################################################

## converting the dataframe to matrix
X_train <- as.matrix(train_data[,-86])
Y_train <- as.matrix(train_data[,86])
X_test <- as.matrix(test_data[,-86])
Y_test <- as.matrix(test_data[,86])

## defining a range of lambda
lam_ridge <- 10^seq(3, -3, by = -.1)
ridge_mod = glmnet(X_train, Y_train, nlambda = 25, alpha = 0, family = 'gaussian', lambda = lam_ridge)

# summary(ridge_mod)

# ridge_mod$dev.ratio

# ridge_mod$lambda

options(repr.plot.width=12, repr.plot.height=6, repr.plot.res = 200)
par(mfrow=c(1,2))
plot(ridge_mod, xvar="lambda",  ylab="Standardised coefficients", label=TRUE)
plot(ridge_mod,  ylab="Standardised coefficients", xlab= "L2 norm", label=TRUE)

# finding the optimal lambda value
cvglm_ridge <- cv.glmnet(X_train, Y_train, alpha = 0, lambda = lam_ridge)

options(repr.plot.width=8, repr.plot.height=4, repr.plot.res = 200)

# plot(cvglm_ridge, xvar="lambda",  ylab="Standardised coefficients", label=TRUE)
opt_lam <- cvglm_ridge$lambda.min
opt_lam

par(mfrow=c(1,2))
plot(cvglm_ridge, ylab="MSE from CV in Ridge")
abline(v=log(opt_lam), col="green", lty=2, ldw=3)

#Creating training model using ridge regression
ridge_best =glmnet(X_train, Y_train,alpha=0,lambda=opt_lam)


# Computing R^2 from original and predicted values
eval_results <- function(original, predicted) {
  SSE <- sum((predicted - original)^2)
  SST <- sum((original - mean(original))^2)
  R_square <- (1 - SSE / SST)*100 ## in percentage
  MSE = SSE/nrow(original) ## calculating mse
 
  # Model performance metrics
  data.frame(
    MSE = MSE,
    Rsquare_percent = R_square)
}

#Retrieving the ridge coefficients
ridge_coef=predict(ridge_best,type="coefficients",s=opt_lam)[0:length(ridge_best$beta)+1,]
#Printing non zero coefficients (all coeff.)
# as.data.frame(ridge_coef[ridge_coef !=0])

# Prediction and evaluation on train data
ridge_pred_train <- predict(ridge_best, s = opt_lam, newx = X_train)

fitted_res_ridge_tr <- ifelse(ridge_pred_train > 0.5,1,0)
error_tr_ridge <- mean(fitted_res_ridge_tr != train_data$V86) ## error on train set
print(1-error_tr_ridge) ## Accuracy in train set
print('Error of train set from linear regression')
error_tr_ridge # TRAINING ERROR

print('accuracy of train set from linear regression')
1-error_tr_ridge # train accuracy

# Prediction and evaluation on test data
ridge_pred_test <- predict(ridge_best, s = opt_lam, newx = X_test)

fitted_res_ridge_te <- ifelse(ridge_pred_test > 0.5,1,0)
error_te_ridge <- mean(fitted_res_ridge_te != test_data$V86) ## error on train set
print(1-error_te_ridge) ## Accuracy in train set
print('Error of test set from linear regression')
error_te_ridge # Test ERROR

print('accuracy of test set from linear regression')
1-error_te_ridge # test accuracy

# Prediction and evaluation on train data
ridge_pred_train <- predict(ridge_best, s = opt_lam, newx = X_train)
# Calculate MSE and R2 on test data
eval_results(Y_train, ridge_pred_train)

# Prediction and evaluation on test data
ridge_pred_test <- predict(ridge_best, s = opt_lam, newx = X_test)
eval_results(Y_test, ridge_pred_test)

## creating confusion matrix from Ridge Regression on train set 
cfmat_ridge_tr <- tibble("target train ridge" = train_data$V86,
                     "prediction train ridge" = fitted_res_ridge_tr)
cfm_tab_ridge_tr <-table(cfmat_ridge_tr)
cfm_tab_ridge_tr # test confu. matrix

## creating confusion matrix from Ridge Regression on test set 
cfmat_ridge <- tibble("target test ridge" = test_data$V86,
                     "prediction test ridge" = fitted_res_ridge_te)
cfm_tab_ridge <-table(cfmat_ridge)
cfm_tab_ridge # test confu. matrix

## confusion matrix from "caret" library
confusionMatrix(as.factor(fitted_res_ridge_te),as.factor(test_data$V86))



###################################################################################################
################################ Lasso Regression #############################################
######################################################################################################

lam_lasso <- 10^seq(3, -4, by = -.1)

# Setting alpha = 1 implements lasso regression
lasso_mod <- glmnet(X_train, Y_train, alpha = 1, lambda = lam_lasso)
sum_lasso <- summary(lasso_mod)

# sum_lasso

options(repr.plot.width=14, repr.plot.height=6, repr.plot.res = 200)
par(mfrow=c(1,2))
plot(lasso_mod, xvar="lambda", label=TRUE, cex=5)
plot(lasso_mod,   label=TRUE)

cvglm_lasso <- cv.glmnet(X_train, Y_train, alpha = 1, lambda = lam_lasso)
options(repr.plot.width=4, repr.plot.height=4, repr.plot.res = 200)

# Best lambda selection
opt_las <- cvglm_lasso$lambda.min 
opt_las

par(mfrow=c(1,1))
plot(cvglm_lasso, ylab= "MSE from CV in Lasso")
abline(v=log(opt_las), col="green", lty=2, ldw=3)

#Creating training model using lasso regression with best lambda
lasso_best =glmnet(X_train, Y_train,alpha=1,lambda=opt_las)
#Printing out the logistic model
# lasso_best$beta

#Retrieving the lasso coefficients
lasso_coef=predict(lasso_best,type="coefficients",s=opt_las)[1:length(lasso_best$beta)+1,]
#Printing non zero coefficients
as.data.frame(lasso_coef[lasso_coef!=0])

# Prediction and evaluation on train data
lasso_pred_train <- predict(lasso_best, s = opt_las, newx = X_train)
fitted_res_lasso_tr <- ifelse(lasso_pred_train > 0.5,1,0)
error_tr_lasso <- mean(fitted_res_lasso_tr != train_data$V86) ## error on train set
print(1-error_tr_lasso) ## Accuracy in train set
print('Error of train set from linear regression')
error_tr_lasso # TRAINING ERROR

print('accuracy of train set from linear regression')
1-error_tr_lasso # train accuracy

# Prediction and evaluation on test data
lasso_pred_test <- predict(lasso_best, s = opt_las, newx = X_test)
fitted_res_lasso_te <- ifelse(lasso_pred_test > 0.5,1,0)
error_te_lasso <- mean(fitted_res_lasso_te != test_data$V86) ## error on train set
print(1-error_te_lasso) ## Accuracy in train set
print('Error of test set from linear regression')
error_te_lasso # Test ERROR

print('accuracy of test set from linear regression')
1-error_te_lasso # test accuracy

# evaluation on train data
eval_results(Y_train, lasso_pred_train)

# Calculate MSE and R2 on test data
eval_results(Y_test, lasso_pred_test)

## creating confusion matrix from Lasso Regression on train set
cfmat_las_tr <- tibble("target train lasso" = train_data$V86,
                     "prediction train lasso" = fitted_res_lasso_tr)
cfm_tab_las_tr <-table(cfmat_las_tr)
cfm_tab_las_tr # train confu. matrix

## creating confusion matrix from Lasso Regression on test set
cfmat_las <- tibble("target test lasso" = test_data$V86,
                     "prediction test lasso" = fitted_res_lasso_te)
cfm_tab_las <-table(cfmat_las)
cfm_tab_las # test confu. matrix

## confusion matrix from "caret" library
confusionMatrix(as.factor(fitted_res_lasso_te),as.factor(test_data$V86))

## ROC AUC plot for comparison
options(repr.plot.width=3, repr.plot.height=3, repr.plot.res = 180)
par(mfrow=c(1,1))

pred_back <- prediction(pred_te_back, test_data$V86)
perform_back <- performance(pred_back, measure = "tpr", x.measure = "fpr")

pred_for <- prediction(pred_te_for, test_data$V86)
perform_for <- performance(pred_for, measure = "tpr", x.measure = "fpr")

pred_ridge <- prediction(ridge_pred_test, test_data$V86)
perform_ridge <- performance(pred_ridge, measure = "tpr", x.measure = "fpr")

pred_lasso <- prediction(lasso_pred_test, test_data$V86)
perform_lasso <- performance(pred_lasso, measure = "tpr", x.measure = "fpr")



options(repr.plot.width=8, repr.plot.height=8, repr.plot.res = 250)
plot(perform,  ylim=c(0,1), col="red", type = "l",)
plot(perform_lasso,  add=TRUE, col="blue", type = "l",) ## plot ROC
plot(perform_ridge,  add=TRUE, col="green", type = "l",) ## plot ROC
plot(perform_for,  add=TRUE, col="pink", type = "l",) ## plot ROC
plot(perform_back,  add=TRUE, col="yellow", type = "l",) ## plot ROC
abline(a=0, b=1, type='l', lwd=1, lty=2, col="black")
legend("bottomright", 95, legend=c("lm", "lasso", "ridge", "forward", "backward", "diagonal"), col=c("red", "blue", "green","pink", "yellow", "black" ), lty=1:2, cex=0.8,box.lty=2, box.lwd=2, box.col="green")


### end ###