How to plot ROC curves for every cross-validations using Caret

问题

I have the following code:

library(mlbench)
library(caret)
library(ggplot2)
set.seed(998)

# Prepare data ------------------------------------------------------------

data(Sonar)
my_data <- Sonar

# Cross Validation Definition ---------------------------------------------------

fitControl <-
  trainControl(
    method = "cv",
    number = 10,
    classProbs = T,
    savePredictions = T,
    summaryFunction = twoClassSummary
  )


# Training with Random Forest ----------------------------------------------------------------


model <- train(
  Class ~ .,
  data = my_data,
  method = "rf",
  trControl = fitControl,
  metric = "ROC"
)

for_lift <- data.frame(Class = model$pred$obs, rf = model$pred$R)
lift_obj <- lift(Class ~ rf, data = for_lift, class = "R")


# Plot ROC ----------------------------------------------------------------

ggplot(lift_obj$data) +
  geom_line(aes(1 - Sp, Sn, color = liftModelVar)) +
  scale_color_discrete(guide = guide_legend(title = "method"))

It produces this plot.

Notice that I am performing 10 fold cross-validation. The ROC curve produces there is only for the final average value.

What I want to do is to have 10 ROC curves, for each cross-validation. How can I achieve that?

回答1:

library(mlbench)
library(caret)
library(ggplot2)
set.seed(998)

# Prepare data ------------------------------------------------------------

data(Sonar)
my_data <- Sonar

# Cross Validation Definition ---------------------------------------------------

fitControl <-
  trainControl(
    method = "cv",
    number = 10,
    classProbs = T,
    savePredictions = T,
    summaryFunction = twoClassSummary
  )


# Training with Random Forest ----------------------------------------------------------------


model <- train(
  Class ~ .,
  data = my_data,
  method = "rf",
  trControl = fitControl,
  metric = "ROC"
)


for_lift <- data.frame(Class = model$pred$obs, rf = model$pred$R, resample = model$pred$Resample)
lift_df <-  data.frame()
for (fold in unique(for_lift$resample)) {
  fold_df <- dplyr::filter(for_lift, resample == fold)
  lift_obj_data <- lift(Class ~ rf, data = fold_df, class = "R")$data
  lift_obj_data$fold = fold
  lift_df = rbind(lift_df, lift_obj_data)
}
lift_obj <- lift(Class ~ rf, data = for_lift, class = "R")


# Plot ROC ----------------------------------------------------------------

ggplot(lift_df) +
  geom_line(aes(1 - Sp, Sn, color = fold)) +
  scale_color_discrete(guide = guide_legend(title = "Fold"))

To calculate AUC:

model <- train(
  Class ~ .,
  data = my_data,
  method = "rf",
  trControl = fitControl,
  metric = "ROC"
)

library(plyr)
library(MLmetrics)
ddply(model$pred, "Resample", summarise,
      accuracy = Accuracy(pred, obs))

Output:

   Resample  accuracy
1    Fold01 0.8253968
2    Fold02 0.8095238
3    Fold03 0.8000000
4    Fold04 0.8253968
5    Fold05 0.8095238
6    Fold06 0.8253968
7    Fold07 0.8333333
8    Fold08 0.8253968
9    Fold09 0.9841270
10   Fold10 0.7936508

来源：https://stackoverflow.com/questions/51904700/how-to-plot-roc-curves-for-every-cross-validations-using-caret

标签

machine-learning

r-caret