
R version 3.0.1 (2013-05-16) -- "Good Sport"
Copyright (C) 2013 The R Foundation for Statistical Computing
Platform: x86_64-apple-darwin10.8.0 (64-bit)

R is free software and comes with ABSOLUTELY NO WARRANTY.
You are welcome to redistribute it under certain conditions.
Type 'license()' or 'licence()' for distribution details.

  Natural language support but running in an English locale

R is a collaborative project with many contributors.
Type 'contributors()' for more information and
'citation()' on how to cite R or R packages in publications.

Type 'demo()' for some demos, 'help()' for on-line help, or
'help.start()' for an HTML browser interface to help.
Type 'q()' to quit R.

> ################################################################################
> ### R code from Applied Predictive Modeling (2013) by Kuhn and Johnson.
> ### Copyright 2013 Kuhn and Johnson
> ### Web Page: http://www.appliedpredictivemodeling.com
> ### Contact: Max Kuhn (mxkuhn@gmail.com) 
> ###
> ### Chapter 16: Remedies for Severe Class Imbalance
> ###
> ### Required packages: AppliedPredictiveModeling, caret, C50, earth, DMwR, 
> ###                    DVD, kernlab,  mda, pROC, randomForest, rpart
> ###
> ### Data used: The insurance data from the DWD package. 
> ###
> ### Notes: 
> ### 1) This code is provided without warranty.
> ###
> ### 2) This code should help the user reproduce the results in the
> ### text. There will be differences between this code and what is is
> ### the computing section. For example, the computing sections show
> ### how the source functions work (e.g. randomForest() or plsr()),
> ### which were not directly used when creating the book. Also, there may be 
> ### syntax differences that occur over time as packages evolve. These files 
> ### will reflect those changes.
> ###
> ### 3) In some cases, the calculations in the book were run in 
> ### parallel. The sub-processes may reset the random number seed.
> ### Your results may slightly vary.
> ###
> ################################################################################
> 
> ################################################################################
> ### Section 16.1 Case Study: Predicting Caravan Policy Ownership
> 
> library(DWD)
Loading required package: Matrix
Loading required package: lattice
> data(ticdata)
> 
> ### Some of the predictor names and levels have characters that would results in
> ### illegal variable names. We convert then to more generic names and treat the
> ### ordered factors as nominal (i.e. unordered) factors. 
> 
> isOrdered <- unlist(lapply(ticdata, function(x) any(class(x) == "ordered")))
> 
> recodeLevels <- function(x)
+   {
+     x <- gsub("f ", "", as.character(x))
+     x <- gsub(" - ", "_to_", x)
+     x <- gsub("-", "_to_", x)
+     x <- gsub("%", "", x)
+     x <- gsub("?", "Unk", x, fixed = TRUE)
+     x <- gsub("[,'\\(\\)]", "", x)
+     x <- gsub(" ", "_", x)
+     factor(paste("_", x, sep = ""))
+   }
> 
> convertCols <- c("STYPE", "MGEMLEEF", "MOSHOOFD",
+                  names(isOrdered)[isOrdered])
> 
> for(i in convertCols) ticdata[,i] <- factor(gsub(" ", "0",format(as.numeric(ticdata[,i]))))
> 
> ticdata$CARAVAN <- factor(as.character(ticdata$CARAVAN),
+                           levels = rev(levels(ticdata$CARAVAN)))
> 
> ### Split the data into three sets: training, test and evaluation. 
> library(caret)
Loading required package: ggplot2
> 
> set.seed(156)
> 
> split1 <- createDataPartition(ticdata$CARAVAN, p = .7)[[1]]
> 
> other     <- ticdata[-split1,]
> training  <- ticdata[ split1,]
> 
> set.seed(934)
> 
> split2 <- createDataPartition(other$CARAVAN, p = 1/3)[[1]]
> 
> evaluation  <- other[ split2,]
> testing     <- other[-split2,]
> 
> predictors <- names(training)[names(training) != "CARAVAN"]
> 
> testResults <- data.frame(CARAVAN = testing$CARAVAN)
> evalResults <- data.frame(CARAVAN = evaluation$CARAVAN)
> 
> trainingInd <- data.frame(model.matrix(CARAVAN ~ ., data = training))[,-1]
> evaluationInd <- data.frame(model.matrix(CARAVAN ~ ., data = evaluation))[,-1]
> testingInd <- data.frame(model.matrix(CARAVAN ~ ., data = testing))[,-1]
> 
> trainingInd$CARAVAN <- training$CARAVAN
> evaluationInd$CARAVAN <- evaluation$CARAVAN
> testingInd$CARAVAN <- testing$CARAVAN
> 
> isNZV <- nearZeroVar(trainingInd)
> noNZVSet <- names(trainingInd)[-isNZV]
> 
> testResults <- data.frame(CARAVAN = testing$CARAVAN)
> evalResults <- data.frame(CARAVAN = evaluation$CARAVAN)
> 
> ################################################################################
> ### Section 16.2 The Effect of Class Imbalance
> 
> ### These functions are used to measure performance
> 
> fiveStats <- function(...) c(twoClassSummary(...), defaultSummary(...))
> fourStats <- function (data, lev = levels(data$obs), model = NULL)
+ {
+ 
+   accKapp <- postResample(data[, "pred"], data[, "obs"])
+   out <- c(accKapp,
+            sensitivity(data[, "pred"], data[, "obs"], lev[1]),
+            specificity(data[, "pred"], data[, "obs"], lev[2]))
+   names(out)[3:4] <- c("Sens", "Spec")
+   out
+ }
> 
> ctrl <- trainControl(method = "cv",
+                      classProbs = TRUE,
+                      summaryFunction = fiveStats)
> 
> ctrlNoProb <- ctrl
> ctrlNoProb$summaryFunction <- fourStats
> ctrlNoProb$classProbs <- FALSE
> 
> 
> set.seed(1410)
> rfFit <- train(CARAVAN ~ ., data = trainingInd,
+                method = "rf",
+                trControl = ctrl,
+                ntree = 1500,
+                tuneLength = 5,
+                metric = "ROC")
Loading required package: randomForest
randomForest 4.6-7
Type rfNews() to see new features/changes/bug fixes.
Loading required package: pROC
Loading required package: plyr
Type 'citation("pROC")' for a citation.

Attaching package: ‘pROC’

The following object is masked from ‘package:stats’:

    cov, smooth, var

Loading required package: class
> rfFit
Random Forest 

6877 samples
 503 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6190, 6190, 6188, 6189, 6189, 6190, ... 

Resampling results across tuning parameters:

  mtry  ROC    Sens    Spec   Accuracy  Kappa      ROC SD  Sens SD  Spec SD
  2     0.608  0       1      0.94      0          0.0863  0        0      
  7     0.669  0       1      0.94      -0.000285  0.0335  0        0.00049
  31    0.689  0.0146  0.993  0.934     0.0134     0.0376  0.0171   0.00373
  126   0.696  0.0292  0.986  0.928     0.0233     0.0387  0.0193   0.0042 
  502   0.688  0.0365  0.98   0.923     0.0233     0.042   0.0208   0.00392
  Accuracy SD  Kappa SD
  0.000422     0       
  0.000602     0.000901
  0.00447      0.0341  
  0.00475      0.0338  
  0.00445      0.0335  

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 126. 
> 
> evalResults$RF <- predict(rfFit, evaluationInd, type = "prob")[,1]
> testResults$RF <- predict(rfFit, testingInd, type = "prob")[,1]
> rfROC <- roc(evalResults$CARAVAN, evalResults$RF,
+              levels = rev(levels(evalResults$CARAVAN)))
> rfROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RF,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RF in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7596
> 
> rfEvalCM <- confusionMatrix(predict(rfFit, evaluationInd), evalResults$CARAVAN)
> rfEvalCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance           4           9
  noinsurance        55         915
                                          
               Accuracy : 0.9349          
                 95% CI : (0.9176, 0.9495)
    No Information Rate : 0.94            
    P-Value [Acc > NIR] : 0.7727          
                                          
                  Kappa : 0.0914          
 Mcnemar's Test P-Value : 1.855e-08       
                                          
            Sensitivity : 0.067797        
            Specificity : 0.990260        
         Pos Pred Value : 0.307692        
         Neg Pred Value : 0.943299        
             Prevalence : 0.060020        
         Detection Rate : 0.004069        
   Detection Prevalence : 0.013225        
      Balanced Accuracy : 0.529028        
                                          
       'Positive' Class : insurance       
                                          
> 
> set.seed(1410)
> lrFit <- train(CARAVAN ~ .,
+                data = trainingInd[, noNZVSet],
+                method = "glm",
+                trControl = ctrl,
+                metric = "ROC")
There were 20 warnings (use warnings() to see them)
> lrFit
Generalized Linear Model 

6877 samples
 203 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6190, 6190, 6188, 6189, 6189, 6190, ... 

Resampling results

  ROC    Sens    Spec   Accuracy  Kappa   ROC SD  Sens SD  Spec SD  Accuracy SD
  0.702  0.0121  0.998  0.939     0.0179  0.0488  0.0128   0.0032   0.00323    
  Kappa SD
  0.0249  

 
> 
> evalResults$LogReg <- predict(lrFit, evaluationInd[, noNZVSet], type = "prob")[,1]
Warning messages:
1: In predict.lm(object, newdata, se.fit, scale = 1, type = ifelse(type ==  :
  prediction from a rank-deficient fit may be misleading
2: In predict.lm(object, newdata, se.fit, scale = 1, type = ifelse(type ==  :
  prediction from a rank-deficient fit may be misleading
> testResults$LogReg <- predict(lrFit, testingInd[, noNZVSet], type = "prob")[,1]
Warning messages:
1: In predict.lm(object, newdata, se.fit, scale = 1, type = ifelse(type ==  :
  prediction from a rank-deficient fit may be misleading
2: In predict.lm(object, newdata, se.fit, scale = 1, type = ifelse(type ==  :
  prediction from a rank-deficient fit may be misleading
> lrROC <- roc(evalResults$CARAVAN, evalResults$LogReg,
+              levels = rev(levels(evalResults$CARAVAN)))
> lrROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$LogReg,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$LogReg in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7267
> 
> lrEvalCM <- confusionMatrix(predict(lrFit, evaluationInd), evalResults$CARAVAN)
Warning message:
In predict.lm(object, newdata, se.fit, scale = 1, type = ifelse(type ==  :
  prediction from a rank-deficient fit may be misleading
> lrEvalCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance           1           2
  noinsurance        58         922
                                          
               Accuracy : 0.939           
                 95% CI : (0.9221, 0.9531)
    No Information Rate : 0.94            
    P-Value [Acc > NIR] : 0.5872          
                                          
                  Kappa : 0.0266          
 Mcnemar's Test P-Value : 1.243e-12       
                                          
            Sensitivity : 0.016949        
            Specificity : 0.997835        
         Pos Pred Value : 0.333333        
         Neg Pred Value : 0.940816        
             Prevalence : 0.060020        
         Detection Rate : 0.001017        
   Detection Prevalence : 0.003052        
      Balanced Accuracy : 0.507392        
                                          
       'Positive' Class : insurance       
                                          
> 
> set.seed(1401)
> fdaFit <- train(CARAVAN ~ ., data = training,
+                 method = "fda",
+                 tuneGrid = data.frame(degree = 1, nprune = 1:25),
+                 metric = "ROC",
+                 trControl = ctrl)
Loading required package: earth
Loading required package: leaps
Loading required package: plotmo
Loading required package: plotrix
Loading required package: mda
> fdaFit
Flexible Discriminant Analysis 

6877 samples
  85 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6189, 6190, 6190, 6189, 6189, 6189, ... 

Resampling results across tuning parameters:

  nprune  ROC    Sens    Spec   Accuracy  Kappa    ROC SD  Sens SD  Spec SD
  1       0.5    0       1      0.94      0        0       0        0      
  2       0.664  0       1      0.94      0        0.0291  0        0      
  3       0.691  0       0.999  0.94      -0.0011  0.0272  0        0.00149
  4       0.705  0.0146  0.997  0.938     0.0201   0.0333  0.0171   0.00231
  5       0.704  0.0146  0.997  0.938     0.0206   0.0303  0.0171   0.00251
  6       0.723  0.0244  0.997  0.938     0.0358   0.0325  0.0304   0.00204
  7       0.724  0.0268  0.995  0.937     0.035    0.0323  0.0372   0.00292
  8       0.724  0.0268  0.995  0.937     0.0347   0.0316  0.0372   0.00311
  9       0.728  0.0293  0.995  0.937     0.0383   0.0315  0.0378   0.0032 
  10      0.727  0.0317  0.994  0.936     0.0393   0.0339  0.0382   0.00482
  11      0.73   0.0366  0.993  0.936     0.0475   0.0351  0.0368   0.00484
  12      0.73   0.0415  0.992  0.936     0.0531   0.0325  0.0364   0.00452
  13      0.734  0.0488  0.993  0.936     0.0651   0.0385  0.0398   0.00411
  14      0.73   0.0488  0.992  0.935     0.0626   0.034   0.0415   0.004  
  15      0.732  0.0463  0.992  0.935     0.0599   0.0327  0.0422   0.00307
  16      0.728  0.0537  0.991  0.935     0.0707   0.0356  0.0427   0.00311
  17      0.732  0.0512  0.991  0.935     0.0647   0.0353  0.0437   0.00409
  18      0.731  0.0512  0.991  0.935     0.0648   0.0362  0.0466   0.00398
  19      0.729  0.0488  0.991  0.934     0.0597   0.0369  0.0488   0.00425
  20      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  21      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  22      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  23      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  24      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  25      0.727  0.0488  0.991  0.934     0.0599   0.0364  0.0488   0.00399
  Accuracy SD  Kappa SD
  0.000452     0       
  0.000452     0       
  0.0014       0.00265 
  0.00222      0.0286  
  0.00209      0.0281  
  0.00268      0.0509  
  0.0023       0.0541  
  0.0026       0.0544  
  0.0026       0.0553  
  0.00315      0.0509  
  0.00346      0.0495  
  0.00292      0.0481  
  0.00295      0.0557  
  0.00267      0.0575  
  0.00267      0.0614  
  0.00337      0.0637  
  0.00339      0.0624  
  0.00327      0.0652  
  0.00331      0.0679  
  0.00324      0.0685  
  0.00324      0.0685  
  0.00324      0.0685  
  0.00324      0.0685  
  0.00324      0.0685  
  0.00324      0.0685  

Tuning parameter 'degree' was held constant at a value of 1
ROC was used to select the optimal model using  the largest value.
The final values used for the model were degree = 1 and nprune = 13. 
> 
> evalResults$FDA <- predict(fdaFit, evaluation[, predictors], type = "prob")[,1]
> testResults$FDA <- predict(fdaFit, testing[, predictors], type = "prob")[,1]
> fdaROC <- roc(evalResults$CARAVAN, evalResults$FDA,
+               levels = rev(levels(evalResults$CARAVAN)))
> fdaROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$FDA,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$FDA in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.754
> 
> fdaEvalCM <- confusionMatrix(predict(fdaFit, evaluation[, predictors]), evalResults$CARAVAN)
> fdaEvalCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance           1           3
  noinsurance        58         921
                                         
               Accuracy : 0.9379         
                 95% CI : (0.921, 0.9522)
    No Information Rate : 0.94           
    P-Value [Acc > NIR] : 0.638          
                                         
                  Kappa : 0.0243         
 Mcnemar's Test P-Value : 4.712e-12      
                                         
            Sensitivity : 0.016949       
            Specificity : 0.996753       
         Pos Pred Value : 0.250000       
         Neg Pred Value : 0.940756       
             Prevalence : 0.060020       
         Detection Rate : 0.001017       
   Detection Prevalence : 0.004069       
      Balanced Accuracy : 0.506851       
                                         
       'Positive' Class : insurance      
                                         
> 
> 
> labs <- c(RF = "Random Forest", LogReg = "Logistic Regression",
+           FDA = "FDA (MARS)")
> lift1 <- lift(CARAVAN ~ RF + LogReg + FDA, data = evalResults,
+               labels = labs)
> 
> plotTheme <- caretTheme()
> 
> plot(fdaROC, type = "S", col = plotTheme$superpose.line$col[3], legacy.axes = TRUE)

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$FDA,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$FDA in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.754
> plot(rfROC, type = "S", col = plotTheme$superpose.line$col[1], add = TRUE, legacy.axes = TRUE)

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RF,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RF in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7596
> plot(lrROC, type = "S", col = plotTheme$superpose.line$col[2], add = TRUE, legacy.axes = TRUE)

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$LogReg,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$LogReg in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7267
> legend(.7, .25,
+        c("Random Forest", "Logistic Regression", "FDA (MARS)"),
+        cex = .85,
+        col = plotTheme$superpose.line$col[1:3],
+        lwd = rep(2, 3),
+        lty = rep(1, 3))
> 
> xyplot(lift1,
+        ylab = "%Events Found",
+        xlab =  "%Customers Evaluated",
+        lwd = 2,
+        type = "l")
> 
> 
> ################################################################################
> ### Section 16.4 Alternate Cutoffs
> 
> rfThresh <- coords(rfROC, x = "best", ret="threshold",
+                    best.method="closest.topleft")
> rfThreshY <- coords(rfROC, x = "best", ret="threshold",
+                     best.method="youden")
> 
> cutText <- ifelse(rfThresh == rfThreshY,
+                   "is the same as",
+                   "is similar to")
> 
> evalResults$rfAlt <- factor(ifelse(evalResults$RF > rfThresh,
+                                    "insurance", "noinsurance"),
+                             levels = levels(evalResults$CARAVAN))
> testResults$rfAlt <- factor(ifelse(testResults$RF > rfThresh,
+                                    "insurance", "noinsurance"),
+                             levels = levels(testResults$CARAVAN))
> rfAltEvalCM <- confusionMatrix(evalResults$rfAlt, evalResults$CARAVAN)
> rfAltEvalCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          39         257
  noinsurance        20         667
                                         
               Accuracy : 0.7182         
                 95% CI : (0.689, 0.7462)
    No Information Rate : 0.94           
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.1329         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.66102        
            Specificity : 0.72186        
         Pos Pred Value : 0.13176        
         Neg Pred Value : 0.97089        
             Prevalence : 0.06002        
         Detection Rate : 0.03967        
   Detection Prevalence : 0.30112        
      Balanced Accuracy : 0.69144        
                                         
       'Positive' Class : insurance      
                                         
> 
> rfAltTestCM <- confusionMatrix(testResults$rfAlt, testResults$CARAVAN)
> rfAltTestCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          71         467
  noinsurance        45        1379
                                         
               Accuracy : 0.739          
                 95% CI : (0.719, 0.7584)
    No Information Rate : 0.9409         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.1328         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.61207        
            Specificity : 0.74702        
         Pos Pred Value : 0.13197        
         Neg Pred Value : 0.96840        
             Prevalence : 0.05912        
         Detection Rate : 0.03619        
   Detection Prevalence : 0.27421        
      Balanced Accuracy : 0.67954        
                                         
       'Positive' Class : insurance      
                                         
> 
> rfTestCM <- confusionMatrix(predict(rfFit, testingInd), testResults$CARAVAN)
> 
> 
> plot(rfROC, print.thres = c(.5, .3, .10, rfThresh), type = "S",
+      print.thres.pattern = "%.3f (Spec = %.2f, Sens = %.2f)",
+      print.thres.cex = .8, legacy.axes = TRUE)

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RF,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RF in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7596
> 
> ################################################################################
> ### Section 16.5 Adjusting Prior Probabilities
> 
> priors <- table(ticdata$CARAVAN)/nrow(ticdata)*100
> fdaPriors <- fdaFit
> fdaPriors$finalModel$prior <- c(insurance = .6, noinsurance =  .4)
> fdaPriorPred <- predict(fdaPriors, evaluation[,predictors])
> evalResults$FDAprior <-  predict(fdaPriors, evaluation[,predictors], type = "prob")[,1]
> testResults$FDAprior <-  predict(fdaPriors, testing[,predictors], type = "prob")[,1]
> fdaPriorCM <- confusionMatrix(fdaPriorPred, evaluation$CARAVAN)
> fdaPriorCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          42         306
  noinsurance        17         618
                                          
               Accuracy : 0.6714          
                 95% CI : (0.6411, 0.7007)
    No Information Rate : 0.94            
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1156          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.71186         
            Specificity : 0.66883         
         Pos Pred Value : 0.12069         
         Neg Pred Value : 0.97323         
             Prevalence : 0.06002         
         Detection Rate : 0.04273         
   Detection Prevalence : 0.35402         
      Balanced Accuracy : 0.69035         
                                          
       'Positive' Class : insurance       
                                          
> 
> fdaPriorROC <- roc(testResults$CARAVAN, testResults$FDAprior,
+                    levels = rev(levels(testResults$CARAVAN)))
> fdaPriorROC

Call:
roc.default(response = testResults$CARAVAN, predictor = testResults$FDAprior,     levels = rev(levels(testResults$CARAVAN)))

Data: testResults$FDAprior in 1846 controls (testResults$CARAVAN noinsurance) < 116 cases (testResults$CARAVAN insurance).
Area under the curve: 0.7469
> 
> ################################################################################
> ### Section 16.7 Sampling Methods
> 
> set.seed(1237)
> downSampled <- downSample(trainingInd[, -ncol(trainingInd)], training$CARAVAN)
> 
> set.seed(1237)
> upSampled <- upSample(trainingInd[, -ncol(trainingInd)], training$CARAVAN)
> 
> library(DMwR)
Loading required package: xts
Loading required package: zoo

Attaching package: ‘zoo’

The following object is masked from ‘package:base’:

    as.Date, as.Date.numeric

Loading required package: quantmod
Loading required package: Defaults
Loading required package: TTR
Version 0.4-0 included new data defaults. See ?getSymbols.
Loading required package: ROCR
Loading required package: gplots
Loading required package: gtools

Attaching package: ‘gtools’

The following object is masked from ‘package:e1071’:

    permutations

Loading required package: gdata
gdata: read.xls support for 'XLS' (Excel 97-2004) files ENABLED.

gdata: read.xls support for 'XLSX' (Excel 2007+) files ENABLED.

Attaching package: ‘gdata’

The following object is masked from ‘package:randomForest’:

    combine

The following object is masked from ‘package:stats’:

    nobs

The following object is masked from ‘package:utils’:

    object.size

Loading required package: caTools
Loading required package: grid
Loading required package: KernSmooth
KernSmooth 2.23 loaded
Copyright M. P. Wand 1997-2009
Loading required package: MASS

Attaching package: ‘gplots’

The following object is masked from ‘package:plotrix’:

    plotCI

The following object is masked from ‘package:stats’:

    lowess

Loading required package: rpart
Loading required package: abind
Loading required package: cluster

Attaching package: ‘DMwR’

The following object is masked from ‘package:plyr’:

    join

Warning message:
'.path.package' is deprecated.
Use 'path.package' instead.
See help("Deprecated") 
> set.seed(1237)
> smoted <- SMOTE(CARAVAN ~ ., data = trainingInd)
> 
> set.seed(1410)
> rfDown <- train(Class ~ ., data = downSampled,
+                 "rf",
+                 trControl = ctrl,
+                 ntree = 1500,
+                 tuneLength = 5,
+                 metric = "ROC")
> rfDown
Random Forest 

822 samples
503 predictors
  2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 740, 740, 739, 739, 740, 740, ... 

Resampling results across tuning parameters:

  mtry  ROC    Sens   Spec   Accuracy  Kappa  ROC SD  Sens SD  Spec SD
  2     0.698  0.652  0.648  0.65      0.3    0.0724  0.0921   0.12   
  7     0.682  0.608  0.677  0.642     0.285  0.0712  0.0715   0.1    
  31    0.69   0.623  0.662  0.642     0.285  0.0582  0.0719   0.079  
  126   0.698  0.628  0.657  0.642     0.285  0.056   0.0655   0.0886 
  502   0.683  0.618  0.63   0.624     0.248  0.0575  0.0516   0.0818 
  Accuracy SD  Kappa SD
  0.0764       0.152   
  0.064        0.128   
  0.0513       0.103   
  0.0489       0.0979  
  0.0413       0.0827  

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 126. 
> 
> evalResults$RFdown <- predict(rfDown, evaluationInd, type = "prob")[,1]
> testResults$RFdown <- predict(rfDown, testingInd, type = "prob")[,1]
> rfDownROC <- roc(evalResults$CARAVAN, evalResults$RFdown,
+                  levels = rev(levels(evalResults$CARAVAN)))
> rfDownROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RFdown,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RFdown in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7922
> 
> set.seed(1401)
> rfDownInt <- train(CARAVAN ~ ., data = trainingInd,
+                    "rf",
+                    ntree = 1500,
+                    tuneLength = 5,
+                    strata = training$CARAVAN,
+                    sampsize = rep(sum(training$CARAVAN == "insurance"), 2),
+                    metric = "ROC",
+                    trControl = ctrl)
> rfDownInt
Random Forest 

6877 samples
 503 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6189, 6190, 6190, 6189, 6189, 6189, ... 

Resampling results across tuning parameters:

  mtry  ROC    Sens   Spec   Accuracy  Kappa  ROC SD  Sens SD  Spec SD
  2     0.703  0.144  0.97   0.92      0.138  0.0353  0.0409   0.00587
  7     0.704  0.424  0.835  0.81      0.133  0.0284  0.0737   0.0204 
  31    0.72   0.414  0.857  0.831     0.154  0.0286  0.0601   0.0188 
  126   0.722  0.424  0.841  0.816     0.14   0.0306  0.0667   0.0171 
  502   0.718  0.465  0.824  0.802     0.141  0.0356  0.0692   0.021  
  Accuracy SD  Kappa SD
  0.00682      0.0535  
  0.0176       0.0323  
  0.0183       0.0401  
  0.0167       0.0374  
  0.0201       0.0373  

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 126. 
> 
> evalResults$RFdownInt <- predict(rfDownInt, evaluationInd, type = "prob")[,1]
> testResults$RFdownInt <- predict(rfDownInt, testingInd, type = "prob")[,1]
> rfDownIntRoc <- roc(evalResults$CARAVAN,
+                     evalResults$RFdownInt,
+                     levels = rev(levels(training$CARAVAN)))
> rfDownIntRoc

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RFdownInt,     levels = rev(levels(training$CARAVAN)))

Data: evalResults$RFdownInt in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7962
> 
> set.seed(1410)
> rfUp <- train(Class ~ ., data = upSampled,
+               "rf",
+               trControl = ctrl,
+               ntree = 1500,
+               tuneLength = 5,
+               metric = "ROC")
> rfUp
Random Forest 

12932 samples
  503 predictors
    2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 11640, 11638, 11639, 11638, 11638, 11640, ... 

Resampling results across tuning parameters:

  mtry  ROC    Sens   Spec   Accuracy  Kappa  ROC SD   Sens SD  Spec SD
  2     0.865  0.836  0.731  0.783     0.567  0.0115   0.00971  0.0186 
  7     0.987  0.992  0.861  0.927     0.853  0.00354  0.00375  0.0226 
  31    0.993  0.999  0.938  0.968     0.937  0.00309  0.00167  0.0127 
  126   0.992  1      0.95   0.975     0.95   0.00345  0        0.0103 
  502   0.992  1      0.943  0.971     0.943  0.00379  0        0.0136 
  Accuracy SD  Kappa SD
  0.0112       0.0224  
  0.01         0.02    
  0.00668      0.0134  
  0.00515      0.0103  
  0.00681      0.0136  

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 31. 
> 
> evalResults$RFup <- predict(rfUp, evaluationInd, type = "prob")[,1]
> testResults$RFup <- predict(rfUp, testingInd, type = "prob")[,1]
> rfUpROC <- roc(evalResults$CARAVAN, evalResults$RFup,
+                levels = rev(levels(evalResults$CARAVAN)))
> rfUpROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RFup,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RFup in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7336
> 
> set.seed(1410)
> rfSmote <- train(CARAVAN ~ ., data = smoted,
+                  "rf",
+                  trControl = ctrl,
+                  ntree = 1500,
+                  tuneLength = 5,
+                  metric = "ROC")
> rfSmote
Random Forest 

2877 samples
 503 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 2590, 2589, 2589, 2590, 2588, 2590, ... 

Resampling results across tuning parameters:

  mtry  ROC    Sens   Spec   Accuracy  Kappa  ROC SD  Sens SD  Spec SD
  2     0.906  0.666  0.998  0.856     0.693  0.0215  0.0322   0.00409
  7     0.908  0.69   0.973  0.852     0.687  0.0177  0.0299   0.0241 
  31    0.914  0.731  0.947  0.854     0.695  0.0168  0.0243   0.0223 
  126   0.918  0.736  0.942  0.853     0.693  0.0146  0.0231   0.0208 
  502   0.912  0.742  0.923  0.845     0.678  0.0151  0.0201   0.0306 
  Accuracy SD  Kappa SD
  0.0142       0.0314  
  0.0215       0.0451  
  0.0183       0.0378  
  0.0154       0.0319  
  0.0211       0.0428  

ROC was used to select the optimal model using  the largest value.
The final value used for the model was mtry = 126. 
> 
> evalResults$RFsmote <- predict(rfSmote, evaluationInd, type = "prob")[,1]
> testResults$RFsmote <- predict(rfSmote, testingInd, type = "prob")[,1]
> rfSmoteROC <- roc(evalResults$CARAVAN, evalResults$RFsmote,
+                   levels = rev(levels(evalResults$CARAVAN)))
> rfSmoteROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$RFsmote,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$RFsmote in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.7675
> 
> rfSmoteCM <- confusionMatrix(predict(rfSmote, evaluationInd), evalResults$CARAVAN)
> rfSmoteCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          11          50
  noinsurance        48         874
                                          
               Accuracy : 0.9003          
                 95% CI : (0.8799, 0.9183)
    No Information Rate : 0.94            
    P-Value [Acc > NIR] : 1.0000          
                                          
                  Kappa : 0.1303          
 Mcnemar's Test P-Value : 0.9195          
                                          
            Sensitivity : 0.18644         
            Specificity : 0.94589         
         Pos Pred Value : 0.18033         
         Neg Pred Value : 0.94794         
             Prevalence : 0.06002         
         Detection Rate : 0.01119         
   Detection Prevalence : 0.06205         
      Balanced Accuracy : 0.56616         
                                          
       'Positive' Class : insurance       
                                          
> 
> samplingSummary <- function(x, evl, tst)
+   {
+     lvl <- rev(levels(tst$CARAVAN))
+     evlROC <- roc(evl$CARAVAN,
+                   predict(x, evl, type = "prob")[,1],
+                   levels = lvl)
+     rocs <- c(auc(evlROC),
+               auc(roc(tst$CARAVAN,
+                       predict(x, tst, type = "prob")[,1],
+                       levels = lvl)))
+     cut <- coords(evlROC, x = "best", ret="threshold",
+                   best.method="closest.topleft")
+     bestVals <- coords(evlROC, cut, ret=c("sensitivity", "specificity"))
+     out <- c(rocs, bestVals*100)
+     names(out) <- c("evROC", "tsROC", "tsSens", "tsSpec")
+     out
+ 
+   }
> 
> rfResults <- rbind(samplingSummary(rfFit, evaluationInd, testingInd),
+                    samplingSummary(rfDown, evaluationInd, testingInd),
+                    samplingSummary(rfDownInt, evaluationInd, testingInd),
+                    samplingSummary(rfUp, evaluationInd, testingInd),
+                    samplingSummary(rfSmote, evaluationInd, testingInd))
> rownames(rfResults) <- c("Original", "Down--Sampling",  "Down--Sampling (Internal)",
+                          "Up--Sampling", "SMOTE")
> 
> rfResults
                              evROC     tsROC   tsSens   tsSpec
Original                  0.7596119 0.7360673 66.10169 72.18615
Down--Sampling            0.7921894 0.7291301 86.44068 67.74892
Down--Sampling (Internal) 0.7961516 0.7649158 66.10169 80.30303
Up--Sampling              0.7336195 0.7408283 72.88136 63.96104
SMOTE                     0.7675178 0.7318643 81.35593 65.36797
> 
> rocCols <- c("black", rgb(1, 0, 0, .5), rgb(0, 0, 1, .5))
> 
> plot(roc(testResults$CARAVAN, testResults$RF, levels = rev(levels(testResults$CARAVAN))),
+      type = "S", col = rocCols[1], legacy.axes = TRUE)

Call:
roc.default(response = testResults$CARAVAN, predictor = testResults$RF,     levels = rev(levels(testResults$CARAVAN)))

Data: testResults$RF in 1846 controls (testResults$CARAVAN noinsurance) < 116 cases (testResults$CARAVAN insurance).
Area under the curve: 0.7361
> plot(roc(testResults$CARAVAN, testResults$RFdownInt, levels = rev(levels(testResults$CARAVAN))),
+      type = "S", col = rocCols[2],add = TRUE, legacy.axes = TRUE)

Call:
roc.default(response = testResults$CARAVAN, predictor = testResults$RFdownInt,     levels = rev(levels(testResults$CARAVAN)))

Data: testResults$RFdownInt in 1846 controls (testResults$CARAVAN noinsurance) < 116 cases (testResults$CARAVAN insurance).
Area under the curve: 0.7649
> plot(roc(testResults$CARAVAN, testResults$RFsmote, levels = rev(levels(testResults$CARAVAN))),
+      type = "S", col = rocCols[3], add = TRUE, legacy.axes = TRUE)

Call:
roc.default(response = testResults$CARAVAN, predictor = testResults$RFsmote,     levels = rev(levels(testResults$CARAVAN)))

Data: testResults$RFsmote in 1846 controls (testResults$CARAVAN noinsurance) < 116 cases (testResults$CARAVAN insurance).
Area under the curve: 0.7319
> legend(.6, .4,
+        c("Normal", "Down-Sampling (Internal)", "SMOTE"),
+        lty = rep(1, 3),
+        lwd = rep(2, 3),
+        cex = .8,
+        col = rocCols)
> 
> xyplot(lift(CARAVAN ~ RF + RFdownInt + RFsmote,
+             data = testResults),
+        type = "l",
+        ylab = "%Events Found",
+        xlab =  "%Customers Evaluated")
> 
> 
> ################################################################################
> ### Section 16.8 Cost–Sensitive Training
> 
> library(kernlab)
> 
> set.seed(1157)
> sigma <- sigest(CARAVAN ~ ., data = trainingInd[, noNZVSet], frac = .75)
> names(sigma) <- NULL
> 
> svmGrid1 <- data.frame(sigma = sigma[2],
+                        C = 2^c(2:10))
> 
> set.seed(1401)
> svmFit <- train(CARAVAN ~ .,
+                 data = trainingInd[, noNZVSet],
+                 method = "svmRadial",
+                 tuneGrid = svmGrid1,
+                 preProc = c("center", "scale"),
+                 metric = "Kappa",
+                 trControl = ctrl)
> svmFit
Support Vector Machines with Radial Basis Function Kernel 

6877 samples
 203 predictors
   2 classes: 'insurance', 'noinsurance' 

Pre-processing: centered, scaled 
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6189, 6190, 6190, 6189, 6189, 6189, ... 

Resampling results across tuning parameters:

  C     ROC    Sens  Spec  Accuracy  Kappa      ROC SD  Sens SD  Spec SD 
  4     0.665  0     1     0.94      -0.000285  0.0465  0        0.000489
  8     0.671  0     1     0.94      0          0.0476  0        0       
  16    0.678  0     1     0.94      0          0.041   0        0       
  32    0.678  0     1     0.94      0          0.0368  0        0       
  64    0.668  0     1     0.94      0          0.0399  0        0       
  128   0.655  0     1     0.94      0          0.039   0        0       
  256   0.648  0     1     0.94      0          0.0395  0        0       
  512   0.644  0     1     0.94      0          0.0401  0        0       
  1020  0.643  0     1     0.94      0          0.037   0        0       
  Accuracy SD  Kappa SD
  6e-04        9e-04   
  0.000452     0       
  0.000452     0       
  0.000452     0       
  0.000452     0       
  0.000452     0       
  0.000452     0       
  0.000452     0       
  0.000452     0       

Tuning parameter 'sigma' was held constant at a value of 0.002454182
Kappa was used to select the optimal model using  the largest value.
The final values used for the model were sigma = 0.00245 and C = 8. 
> 
> evalResults$SVM <- predict(svmFit, evaluationInd[, noNZVSet], type = "prob")[,1]
> testResults$SVM <- predict(svmFit, testingInd[, noNZVSet], type = "prob")[,1]
> svmROC <- roc(evalResults$CARAVAN, evalResults$SVM,
+               levels = rev(levels(evalResults$CARAVAN)))
> svmROC

Call:
roc.default(response = evalResults$CARAVAN, predictor = evalResults$SVM,     levels = rev(levels(evalResults$CARAVAN)))

Data: evalResults$SVM in 924 controls (evalResults$CARAVAN noinsurance) < 59 cases (evalResults$CARAVAN insurance).
Area under the curve: 0.6952
> 
> svmTestROC <- roc(testResults$CARAVAN, testResults$SVM,
+                   levels = rev(levels(testResults$CARAVAN)))
> svmTestROC

Call:
roc.default(response = testResults$CARAVAN, predictor = testResults$SVM,     levels = rev(levels(testResults$CARAVAN)))

Data: testResults$SVM in 1846 controls (testResults$CARAVAN noinsurance) < 116 cases (testResults$CARAVAN insurance).
Area under the curve: 0.6974
> 
> confusionMatrix(predict(svmFit, evaluationInd[, noNZVSet]), evalResults$CARAVAN)
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance           0           0
  noinsurance        59         924
                                         
               Accuracy : 0.94           
                 95% CI : (0.9233, 0.954)
    No Information Rate : 0.94           
    P-Value [Acc > NIR] : 0.5346         
                                         
                  Kappa : 0              
 Mcnemar's Test P-Value : 4.321e-14      
                                         
            Sensitivity : 0.00000        
            Specificity : 1.00000        
         Pos Pred Value :     NaN        
         Neg Pred Value : 0.93998        
             Prevalence : 0.06002        
         Detection Rate : 0.00000        
   Detection Prevalence : 0.00000        
      Balanced Accuracy : 0.50000        
                                         
       'Positive' Class : insurance      
                                         
> 
> confusionMatrix(predict(svmFit, testingInd[, noNZVSet]), testingInd$CARAVAN)
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance           0           0
  noinsurance       116        1846
                                          
               Accuracy : 0.9409          
                 95% CI : (0.9295, 0.9509)
    No Information Rate : 0.9409          
    P-Value [Acc > NIR] : 0.5247          
                                          
                  Kappa : 0               
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.00000         
            Specificity : 1.00000         
         Pos Pred Value :     NaN         
         Neg Pred Value : 0.94088         
             Prevalence : 0.05912         
         Detection Rate : 0.00000         
   Detection Prevalence : 0.00000         
      Balanced Accuracy : 0.50000         
                                          
       'Positive' Class : insurance       
                                          
> 
> 
> set.seed(1401)
> svmWtFit <- train(CARAVAN ~ .,
+                   data = trainingInd[, noNZVSet],
+                   method = "svmRadial",
+                   tuneGrid = svmGrid1,
+                   preProc = c("center", "scale"),
+                   metric = "Kappa",
+                   class.weights = c(insurance = 18, noinsurance = 1),
+                   trControl = ctrlNoProb)
> svmWtFit
Support Vector Machines with Radial Basis Function Kernel 

6877 samples
 203 predictors
   2 classes: 'insurance', 'noinsurance' 

Pre-processing: centered, scaled 
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6189, 6190, 6190, 6189, 6189, 6189, ... 

Resampling results across tuning parameters:

  C     Accuracy  Kappa   Sens   Spec   Accuracy SD  Kappa SD  Sens SD  Spec SD
  4     0.818     0.105   0.343  0.849  0.016        0.0399    0.0853   0.0186 
  8     0.842     0.116   0.309  0.876  0.0142       0.0339    0.0605   0.0159 
  16    0.855     0.105   0.256  0.893  0.0192       0.0442    0.0602   0.0207 
  32    0.869     0.11    0.234  0.909  0.0159       0.0507    0.0633   0.0167 
  64    0.876     0.0948  0.195  0.919  0.0173       0.0426    0.0435   0.0179 
  128   0.879     0.0865  0.175  0.924  0.0155       0.049     0.0503   0.0155 
  256   0.88      0.0843  0.17   0.925  0.0154       0.0419    0.0386   0.0154 
  512   0.879     0.0739  0.161  0.925  0.015        0.0501    0.0557   0.0157 
  1020  0.88      0.073   0.158  0.925  0.0148       0.0511    0.0569   0.0154 

Tuning parameter 'sigma' was held constant at a value of 0.002454182
Kappa was used to select the optimal model using  the largest value.
The final values used for the model were sigma = 0.00245 and C = 8. 
> 
> svmWtEvalCM <- confusionMatrix(predict(svmWtFit, evaluationInd[, noNZVSet]), evalResults$CARAVAN)
> svmWtEvalCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          17         123
  noinsurance        42         801
                                         
               Accuracy : 0.8321         
                 95% CI : (0.8073, 0.855)
    No Information Rate : 0.94           
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.0944         
 Mcnemar's Test P-Value : 4.725e-10      
                                         
            Sensitivity : 0.28814        
            Specificity : 0.86688        
         Pos Pred Value : 0.12143        
         Neg Pred Value : 0.95018        
             Prevalence : 0.06002        
         Detection Rate : 0.01729        
   Detection Prevalence : 0.14242        
      Balanced Accuracy : 0.57751        
                                         
       'Positive' Class : insurance      
                                         
> 
> svmWtTestCM <- confusionMatrix(predict(svmWtFit, testingInd[, noNZVSet]), testingInd$CARAVAN)
> svmWtTestCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          40         223
  noinsurance        76        1623
                                          
               Accuracy : 0.8476          
                 95% CI : (0.8309, 0.8632)
    No Information Rate : 0.9409          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1406          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.34483         
            Specificity : 0.87920         
         Pos Pred Value : 0.15209         
         Neg Pred Value : 0.95527         
             Prevalence : 0.05912         
         Detection Rate : 0.02039         
   Detection Prevalence : 0.13405         
      Balanced Accuracy : 0.61201         
                                          
       'Positive' Class : insurance       
                                          
> 
> 
> initialRpart <- rpart(CARAVAN ~ ., data = training,
+                       control = rpart.control(cp = 0.0001))
> rpartGrid <- data.frame(cp = initialRpart$cptable[, "CP"])
> 
> cmat <- list(loss = matrix(c(0, 1, 20, 0), ncol = 2))
> set.seed(1401)
> cartWMod <- train(x = training[,predictors],
+                   y = training$CARAVAN,
+                   method = "rpart",
+                   trControl = ctrlNoProb,
+                   tuneGrid = rpartGrid,
+                   metric = "Kappa",
+                   parms = cmat)
> cartWMod
CART 

6877 samples
  85 predictors
   2 classes: 'insurance', 'noinsurance' 

No pre-processing
Resampling: Cross-Validated (10 fold) 

Summary of sample sizes: 6189, 6190, 6190, 6189, 6189, 6189, ... 

Resampling results across tuning parameters:

  cp        Accuracy  Kappa   Sens   Spec   Accuracy SD  Kappa SD  Sens SD
  1e-04     0.797     0.0734  0.316  0.828  0.018        0.0435    0.0918 
  0.000487  0.797     0.0744  0.319  0.827  0.0189       0.0423    0.0892 
  0.00122   0.778     0.0768  0.36   0.805  0.02         0.037     0.0883 
  0.00162   0.762     0.0844  0.411  0.785  0.0181       0.0298    0.0794 
  0.00243   0.722     0.0805  0.48   0.737  0.024        0.0253    0.0786 
  0.00278   0.707     0.0773  0.499  0.72   0.0229       0.0299    0.0916 
  Spec SD
  0.0203 
  0.0212 
  0.0229 
  0.0208 
  0.0274 
  0.0256 

Kappa was used to select the optimal model using  the largest value.
The final value used for the model was cp = 0.00162. 
> 
> 
> library(C50)
> c5Grid <- expand.grid(model = c("tree", "rules"),
+                       trials = c(1, (1:10)*10),
+                       winnow = FALSE)
> 
> finalCost <- matrix(c(0, 20, 1, 0), ncol = 2)
> rownames(finalCost) <- colnames(finalCost) <- levels(training$CARAVAN)
> set.seed(1401)
> C5CostFit <- train(training[, predictors],
+                    training$CARAVAN,
+                    method = "C5.0",
+                    metric = "Kappa",
+                    tuneGrid = c5Grid,
+                    cost = finalCost,
+                    control = C5.0Control(earlyStopping = FALSE),
+                    trControl = ctrlNoProb)
> 
> C5CostCM <- confusionMatrix(predict(C5CostFit, testing), testing$CARAVAN)
> C5CostCM
Confusion Matrix and Statistics

             Reference
Prediction    insurance noinsurance
  insurance          64         623
  noinsurance        52        1223
                                         
               Accuracy : 0.656          
                 95% CI : (0.6345, 0.677)
    No Information Rate : 0.9409         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.0648         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.55172        
            Specificity : 0.66251        
         Pos Pred Value : 0.09316        
         Neg Pred Value : 0.95922        
             Prevalence : 0.05912        
         Detection Rate : 0.03262        
   Detection Prevalence : 0.35015        
      Balanced Accuracy : 0.60712        
                                         
       'Positive' Class : insurance      
                                         
> 
> 
> ################################################################################
> ### Session Information
> 
> sessionInfo()
R version 3.0.1 (2013-05-16)
Platform: x86_64-apple-darwin10.8.0 (64-bit)

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

attached base packages:
[1] grid      stats     graphics  grDevices utils     datasets  methods  
[8] base     

other attached packages:
 [1] C50_0.1.0-14       kernlab_0.9-16     DMwR_0.3.0         cluster_1.14.4    
 [5] abind_1.4-0        rpart_4.1-1        ROCR_1.0-4         gplots_2.11.0     
 [9] MASS_7.3-26        KernSmooth_2.23-10 caTools_1.14       gdata_2.12.0      
[13] gtools_2.7.0       quantmod_0.4-0     TTR_0.21-1         Defaults_1.1-1    
[17] xts_0.9-3          zoo_1.7-9          mda_0.4-2          earth_3.2-3       
[21] plotrix_3.4-6      plotmo_1.3-2       leaps_2.9          e1071_1.6-1       
[25] class_7.3-7        pROC_1.5.4         plyr_1.8           randomForest_4.6-7
[29] caret_6.0-22       ggplot2_0.9.3.1    DWD_0.10           Matrix_1.0-12     
[33] lattice_0.20-15   

loaded via a namespace (and not attached):
 [1] bitops_1.0-5       car_2.0-16         codetools_0.2-8    colorspace_1.2-1  
 [5] compiler_3.0.1     dichromat_2.0-0    digest_0.6.3       foreach_1.4.0     
 [9] gtable_0.1.2       iterators_1.0.6    labeling_0.1       munsell_0.4       
[13] proto_0.3-10       RColorBrewer_1.0-5 reshape2_1.2.2     scales_0.2.3      
[17] stringr_0.6.2      tools_3.0.1       
> 
> q("no")
> proc.time()
      user     system    elapsed 
243437.520    682.066 244138.032 
