Create Models
Dayne Filer & Chris Kassotis
2022-03-02
CreateModels.RmdLoad packages & needed data:
library(kassotis2020)
library(data.table)
data(ph1data)
data(ph3data)
data(ph3zscr)
data(chemData)
data(kassotis)
data(janesick)
## PhI assays
ph1 <- c("ATG_PPRE_CIS", "ATG_PPARg_TRANS",
"NVS_NR_hPPARg", "NCGC_PPARg_Agonist",
"NCGC_GR_Agonist", "NVS_NR_hGR",
"ATG_GR_TRANS", "ATG_GRE_CIS",
"ATG_LXRa_TRANS", "ATG_LXRb_TRANS",
"NCGC_LXR_Agonist", "ATG_DR4_LXR_CIS",
"ATG_C_EBP_CIS", "ATG_SREBP_CIS",
"ATG_RXRa_TRANS", "NCGC_RXRa_Agonist")
## PhIII assays
ph3 <- c("ATG_PPRE_CIS_up", "ATG_PPARg_TRANS_up",
"TOX21_PPARg_BLA_Agonist_ratio", "NVS_NR_hPPARg",
"OT_PPARg_PPARgSRC1_0480", "OT_PPARg_PPARgSRC1_1440",
"ATG_PPARa_TRANS_up", "NVS_NR_hPPARa",
"ATG_PPARd_TRANS_up", "TOX21_PPARd_BLA_agonist_ratio",
"TOX21_GR_BLA_Agonist_ratio", "NVS_NR_hGR",
"ATG_GR_TRANS_up", "ATG_GRE_CIS_up",
"ATG_LXRa_TRANS_up", "ATG_LXRb_TRANS_up",
"ATG_DR4_LXR_CIS_up", "ATG_C_EBP_CIS_up",
"ATG_SREBP_CIS_up", "ATG_RXRa_TRANS_up",
"TOX21_RXR_BLA_Agonist_ratio", "OT_NURR1_NURR1RXRa_0480",
"OT_NURR1_NURR1RXRa_1440", "ATG_RXRb_TRANS_up",
"ATG_RXRg_TRANS2_up", "ATG_PXRE_CIS_up",
"ATG_PXR_TRANS_up", "NVS_NR_hPXR",
"ACEA_AR_antagonist_80hr", "ATG_AR_TRANS_up",
"NVS_NR_cAR", "NVS_NR_hAR",
"NVS_NR_rAR", "OT_AR_ARELUC_AG_1440",
"OT_AR_ARSRC1_0480", "OT_AR_ARSRC1_0960",
"TOX21_AR_BLA_Antagonist_ratio",
"TOX21_AR_LUC_MDAKB2_Antagonist_10nM_R1881",
"ATG_CAR_TRANS_up", "NVS_NR_hCAR_Agonist",
"TOX21_CAR_Agonist", "ATG_FXR_TRANS_up",
"ATG_IR1_CIS_up", "NVS_NR_hFXR_Agonist",
"OT_FXR_FXRSRC1_0480", "OT_FXR_FXRSRC1_1440",
"TOX21_FXR_BLA_agonist_ratio", "ATG_THRa1_TRANS_up",
"ATG_THRb_TRANS2_up", "NVS_NR_hTRa_Antagonist",
"TOX21_TR_LUC_GH3_Antagonist", "ATG_DR5_CIS_up",
"ATG_RARa_TRANS_up", "NVS_NR_hRARa_Agonist",
"TOX21_RAR_LUC_Agonist", "ATG_RARb_TRANS_up",
"ATG_RARg_TRANS_up", "BSK_CASM3C_LDLR_up",
"NVS_ENZ_hIGF1R", "NVS_ENZ_hIGF1R_Activator",
"NVS_ENZ_hInsR", "NVS_ENZ_hInsR_Activator")
## 8-slice model used in Janesick et al. w/ phI assays
ja1 <- list(slices = list(PPRE = ph1[1], PPARg = ph1[2:4],
GR = ph1[5:8], LXR = ph1[9:11],
LXRE = ph1[12], CEBP = ph1[13],
SREBP = ph1[14], RXRa = ph1[15:16]),
weights = c(1, 1, 1, 1, 1, 1, 1, 1))
## 8-slice model used in Janesick et al. w/ phIII assays
ja3 <- list(slices = list(PPRE = ph3[1], PPARg = ph3[2:4],
GR = ph3[11:14], LXR = ph3[15:16],
LXRE = ph3[17], CEBP = ph3[18],
SREBP = ph3[19], RXRa = ph3[20:21]),
weights = c(1, 1, 1, 1, 1, 1, 1, 1))
## 5-slice model used in Auerbach et al. w/ phIII assays
au3 <- list(slices = list(PPARg = ph3[c(1:4, 6)], GR = ph3[11:14],
LXR = ph3[15:17], Other = ph3[18:19],
RXRa = ph3[20L]),
weights = c(1, 1, 1, 1, 1))We can then calculate the ToxPi scores and look at prediction metrics.
## setup for z-score filtering/adding
filtAdd <- function(coff) {
tmp <- cbind(ph3zscr, ph3zscr)
replace(ph3data, tmp < coff, 0) + replace(tmp, tmp < coff, 0)
}
## Caculate ToxPi models for the Janesick model
ja1Non <- calculateToxpi(ph1data, ja1)
ja3Non <- calculateToxpi(ph3data, ja3)
ja3AdZ <- calculateToxpi(filtAdd(0), ja3)
ja3Rm1 <- calculateToxpi(filtAdd(1), ja3)
ja3Rm2 <- calculateToxpi(filtAdd(2), ja3)
ja3Rm3 <- calculateToxpi(filtAdd(3), ja3)
## Calculate ToxPi models for the Auerbach model
au3Non <- calculateToxpi(ph3data, au3)
au3AdZ <- calculateToxpi(filtAdd(0), au3)
au3Rm1 <- calculateToxpi(filtAdd(1), au3)
au3Rm2 <- calculateToxpi(filtAdd(2), au3)
au3Rm3 <- calculateToxpi(filtAdd(3), au3)
smry <- list(## Baseline comparison of Kassotis 3T3 to lit review
with(kassotis, evalPred(cellActive, litActive)),
## 8-slice model using Janesick 3T3 results
evalPred(ja1Non[janesick$code, "Score"], janesick$cellActive),
evalPred(ja3Non[janesick$code, "Score"], janesick$cellActive),
evalPred(ja3AdZ[janesick$code, "Score"], janesick$cellActive),
evalPred(ja3Rm1[janesick$code, "Score"], janesick$cellActive),
evalPred(ja3Rm2[janesick$code, "Score"], janesick$cellActive),
evalPred(ja3Rm3[janesick$code, "Score"], janesick$cellActive),
## 8-slice model using Kassotis 3T3 results
evalPred(ja3Non[kassotis$code, "Score"], kassotis$cellActive),
evalPred(ja3AdZ[kassotis$code, "Score"], kassotis$cellActive),
evalPred(ja3Rm1[kassotis$code, "Score"], kassotis$cellActive),
evalPred(ja3Rm2[kassotis$code, "Score"], kassotis$cellActive),
evalPred(ja3Rm3[kassotis$code, "Score"], kassotis$cellActive),
## 8-slice model using Kassotis literature results
evalPred(ja3Non[kassotis$code, "Score"], kassotis$litActive),
evalPred(ja3AdZ[kassotis$code, "Score"], kassotis$litActive),
evalPred(ja3Rm1[kassotis$code, "Score"], kassotis$litActive),
evalPred(ja3Rm2[kassotis$code, "Score"], kassotis$litActive),
evalPred(ja3Rm3[kassotis$code, "Score"], kassotis$litActive),
## 5-slice model using Janesick 3T3 results
evalPred(au3Non[janesick$code, "Score"], janesick$cellActive),
evalPred(au3AdZ[janesick$code, "Score"], janesick$cellActive),
evalPred(au3Rm1[janesick$code, "Score"], janesick$cellActive),
evalPred(au3Rm2[janesick$code, "Score"], janesick$cellActive),
evalPred(au3Rm3[janesick$code, "Score"], janesick$cellActive),
## 5-slice model using Kassotis 3T3 results
evalPred(au3Non[kassotis$code, "Score"], kassotis$cellActive),
evalPred(au3AdZ[kassotis$code, "Score"], kassotis$cellActive),
evalPred(au3Rm1[kassotis$code, "Score"], kassotis$cellActive),
evalPred(au3Rm2[kassotis$code, "Score"], kassotis$cellActive),
evalPred(au3Rm3[kassotis$code, "Score"], kassotis$cellActive),
## 5-slice model using Kassotis lit results
evalPred(au3Non[kassotis$code, "Score"], kassotis$litActive),
evalPred(au3AdZ[kassotis$code, "Score"], kassotis$litActive),
evalPred(au3Rm1[kassotis$code, "Score"], kassotis$litActive),
evalPred(au3Rm2[kassotis$code, "Score"], kassotis$litActive),
evalPred(au3Rm3[kassotis$code, "Score"], kassotis$litActive))
smryTbl <- as.data.table(t(sapply(smry, '[[', "metrics")))
mdlFac <- factor(c("8-Slice", "5-Slice"), levels = c("8-Slice", "5-Slice"))
smryTbl[ , Model := mdlFac[c(NA, rep(1, 16), rep(2, 15))]]
phFac <- factor(c("PhI", "PhIII"), levels = c("PhI", "PhIII"))
smryTbl[ , Phase := phFac[c(NA, 1, rep(2, 30))]]
csFac <- factor(c("Janesick", "Kassotis"), levels = c("Janesick", "Kassotis"))
smryTbl[ , CtrlSet := csFac[c(NA, 1, rep(c(rep(1, 5), rep(2, 10)), 2))]]
ctFac <- factor(c("Cell", "Literature"), levels = c("Cell", "Literature"))
smryTbl[ , CtrlType := ctFac[c(NA, 1, rep(c(rep(1, 10), rep(2, 5)), 2))]]
zsFac <- factor(c("None", ">0", ">1", ">2", ">3"),
levels = c("None", ">0", ">1", ">2", ">3"))
smryTbl[ , ZScore := zsFac[c(NA, 1, rep(1:5, 6))]]
setcolorder(smryTbl, c("Model", "Phase", "CtrlSet", "CtrlType", "ZScore"))
smryTbl[ , SensPlusSpec := Sensitivity + Specificity]
smryTbl## Model Phase CtrlSet CtrlType ZScore ROC PRC cutpoint
## 1: <NA> <NA> <NA> <NA> <NA> NA NA NA
## 2: 8-Slice PhI Janesick Cell None 0.6030769 0.4577176 0.173827499
## 3: 8-Slice PhIII Janesick Cell None 0.5323077 0.3667604 0.253038690
## 4: 8-Slice PhIII Janesick Cell >0 0.5076923 0.3473691 0.060875974
## 5: 8-Slice PhIII Janesick Cell >1 0.5230769 0.3485085 0.054554379
## 6: 8-Slice PhIII Janesick Cell >2 0.5600000 0.3665002 0.043041202
## 7: 8-Slice PhIII Janesick Cell >3 0.5569231 0.3716083 0.010663124
## 8: 8-Slice PhIII Kassotis Cell None 0.6676136 0.6592216 0.019038892
## 9: 8-Slice PhIII Kassotis Cell >0 0.6732955 0.7227558 0.005959943
## 10: 8-Slice PhIII Kassotis Cell >1 0.6221591 0.7266713 0.043229845
## 11: 8-Slice PhIII Kassotis Cell >2 0.5568182 0.6644535 0.009686304
## 12: 8-Slice PhIII Kassotis Cell >3 0.6278409 0.7192921 0.009686304
## 13: 8-Slice PhIII Kassotis Literature None 0.8640000 0.9696405 0.019038892
## 14: 8-Slice PhIII Kassotis Literature >0 0.8800000 0.9746470 0.005959943
## 15: 8-Slice PhIII Kassotis Literature >1 0.8080000 0.9614434 0.009686304
## 16: 8-Slice PhIII Kassotis Literature >2 0.7600000 0.9480094 0.009686304
## 17: 8-Slice PhIII Kassotis Literature >3 0.7200000 0.9382319 0.009686304
## 18: 5-Slice PhIII Janesick Cell None 0.4800000 0.3563010 0.290682709
## 19: 5-Slice PhIII Janesick Cell >0 0.4523077 0.3321553 0.112858259
## 20: 5-Slice PhIII Janesick Cell >1 0.4861538 0.3366059 0.014660227
## 21: 5-Slice PhIII Janesick Cell >2 0.5292308 0.3545212 0.014660227
## 22: 5-Slice PhIII Janesick Cell >3 0.5446154 0.3711091 0.014660227
## 23: 5-Slice PhIII Kassotis Cell None 0.7017045 0.6646312 0.044543635
## 24: 5-Slice PhIII Kassotis Cell >0 0.7045455 0.7176431 0.006298864
## 25: 5-Slice PhIII Kassotis Cell >1 0.6250000 0.7070970 0.040507655
## 26: 5-Slice PhIII Kassotis Cell >2 0.5681818 0.6417947 0.011690393
## 27: 5-Slice PhIII Kassotis Cell >3 0.6335227 0.7029110 0.011690393
## 28: 5-Slice PhIII Kassotis Literature None 0.9040000 0.9811478 0.042419772
## 29: 5-Slice PhIII Kassotis Literature >0 0.9040000 0.9811478 0.017964753
## 30: 5-Slice PhIII Kassotis Literature >1 0.8080000 0.9614434 0.015498087
## 31: 5-Slice PhIII Kassotis Literature >2 0.7680000 0.9507691 0.011690393
## 32: 5-Slice PhIII Kassotis Literature >3 0.7200000 0.9382319 0.011006433
## Model Phase CtrlSet CtrlType ZScore ROC PRC cutpoint
## Sensitivity Specificity PosPredValue NegPredValue Precision Recall
## 1: 0.7200000 1.0000 1.0000000 0.4166667 1.0000000 0.7200000
## 2: 0.6153846 0.6400 0.4705882 0.7619048 0.4705882 0.6153846
## 3: 0.3076923 0.8400 0.5000000 0.7000000 0.5000000 0.3076923
## 4: 0.7692308 0.4000 0.4000000 0.7692308 0.4000000 0.7692308
## 5: 0.6923077 0.4800 0.4090909 0.7500000 0.4090909 0.6923077
## 6: 0.6923077 0.5200 0.4285714 0.7647059 0.4285714 0.6923077
## 7: 0.7692308 0.4000 0.4000000 0.7692308 0.4000000 0.7692308
## 8: 0.9545455 0.5000 0.7241379 0.8888889 0.7241379 0.9545455
## 9: 0.9545455 0.5000 0.7241379 0.8888889 0.7241379 0.9545455
## 10: 0.5909091 0.6875 0.7222222 0.5500000 0.7222222 0.5909091
## 11: 0.5454545 0.6875 0.7058824 0.5238095 0.7058824 0.5454545
## 12: 0.5000000 0.7500 0.7333333 0.5217391 0.7333333 0.5000000
## 13: 0.8800000 0.8000 0.9565217 0.5714286 0.9565217 0.8800000
## 14: 0.8800000 0.8000 0.9565217 0.5714286 0.9565217 0.8800000
## 15: 0.6800000 1.0000 1.0000000 0.3846154 1.0000000 0.6800000
## 16: 0.5200000 1.0000 1.0000000 0.2941176 1.0000000 0.5200000
## 17: 0.4400000 1.0000 1.0000000 0.2631579 1.0000000 0.4400000
## 18: 0.1538462 0.9600 0.6666667 0.6857143 0.6666667 0.1538462
## 19: 0.3076923 0.8000 0.4444444 0.6896552 0.4444444 0.3076923
## 20: 0.8461538 0.2800 0.3793103 0.7777778 0.3793103 0.8461538
## 21: 0.8461538 0.3600 0.4074074 0.8181818 0.4074074 0.8461538
## 22: 0.7692308 0.4800 0.4347826 0.8000000 0.4347826 0.7692308
## 23: 0.7727273 0.6875 0.7727273 0.6875000 0.7727273 0.7727273
## 24: 0.9545455 0.5000 0.7241379 0.8888889 0.7241379 0.9545455
## 25: 0.5454545 0.7500 0.7500000 0.5454545 0.7500000 0.5454545
## 26: 0.5909091 0.6875 0.7222222 0.5500000 0.7222222 0.5909091
## 27: 0.5000000 0.8125 0.7857143 0.5416667 0.7857143 0.5000000
## 28: 0.7600000 1.0000 1.0000000 0.4545455 1.0000000 0.7600000
## 29: 0.7600000 1.0000 1.0000000 0.4545455 1.0000000 0.7600000
## 30: 0.6800000 1.0000 1.0000000 0.3846154 1.0000000 0.6800000
## 31: 0.5600000 1.0000 1.0000000 0.3125000 1.0000000 0.5600000
## 32: 0.4400000 1.0000 1.0000000 0.2631579 1.0000000 0.4400000
## Sensitivity Specificity PosPredValue NegPredValue Precision Recall
## F1 Prevalence DetectionRate DetectionPrevalence BalancedAccuracy
## 1: 0.8372093 0.8333333 0.60000000 0.60000000 0.8600000
## 2: 0.5333333 0.3421053 0.21052632 0.44736842 0.6276923
## 3: 0.3809524 0.3421053 0.10526316 0.21052632 0.5738462
## 4: 0.5263158 0.3421053 0.26315789 0.65789474 0.5846154
## 5: 0.5142857 0.3421053 0.23684211 0.57894737 0.5861538
## 6: 0.5294118 0.3421053 0.23684211 0.55263158 0.6061538
## 7: 0.5263158 0.3421053 0.26315789 0.65789474 0.5846154
## 8: 0.8235294 0.5789474 0.55263158 0.76315789 0.7272727
## 9: 0.8235294 0.5789474 0.55263158 0.76315789 0.7272727
## 10: 0.6500000 0.5789474 0.34210526 0.47368421 0.6392045
## 11: 0.6153846 0.5789474 0.31578947 0.44736842 0.6164773
## 12: 0.5945946 0.5789474 0.28947368 0.39473684 0.6250000
## 13: 0.9166667 0.8333333 0.73333333 0.76666667 0.8400000
## 14: 0.9166667 0.8333333 0.73333333 0.76666667 0.8400000
## 15: 0.8095238 0.8333333 0.56666667 0.56666667 0.8400000
## 16: 0.6842105 0.8333333 0.43333333 0.43333333 0.7600000
## 17: 0.6111111 0.8333333 0.36666667 0.36666667 0.7200000
## 18: 0.2500000 0.3421053 0.05263158 0.07894737 0.5569231
## 19: 0.3636364 0.3421053 0.10526316 0.23684211 0.5538462
## 20: 0.5238095 0.3421053 0.28947368 0.76315789 0.5630769
## 21: 0.5500000 0.3421053 0.28947368 0.71052632 0.6030769
## 22: 0.5555556 0.3421053 0.26315789 0.60526316 0.6246154
## 23: 0.7727273 0.5789474 0.44736842 0.57894737 0.7301136
## 24: 0.8235294 0.5789474 0.55263158 0.76315789 0.7272727
## 25: 0.6315789 0.5789474 0.31578947 0.42105263 0.6477273
## 26: 0.6500000 0.5789474 0.34210526 0.47368421 0.6392045
## 27: 0.6111111 0.5789474 0.28947368 0.36842105 0.6562500
## 28: 0.8636364 0.8333333 0.63333333 0.63333333 0.8800000
## 29: 0.8636364 0.8333333 0.63333333 0.63333333 0.8800000
## 30: 0.8095238 0.8333333 0.56666667 0.56666667 0.8400000
## 31: 0.7179487 0.8333333 0.46666667 0.46666667 0.7800000
## 32: 0.6111111 0.8333333 0.36666667 0.36666667 0.7200000
## F1 Prevalence DetectionRate DetectionPrevalence BalancedAccuracy
## Accuracy Kappa AccuracyLower AccuracyUpper AccuracyNull
## 1: 0.7666667 0.46153846 0.5771635 0.9006621 0.8333333
## 2: 0.6315789 0.23782235 0.4599428 0.7818750 0.6578947
## 3: 0.6578947 0.16271186 0.4864732 0.8036709 0.6578947
## 4: 0.5263158 0.13853904 0.3581834 0.6901927 0.6578947
## 5: 0.5526316 0.14775726 0.3829908 0.7137585 0.6578947
## 6: 0.5789474 0.18498660 0.4082145 0.7369018 0.6578947
## 7: 0.5263158 0.13853904 0.3581834 0.6901927 0.6578947
## 8: 0.7631579 0.48338369 0.5975876 0.8855583 0.5789474
## 9: 0.7631579 0.48338369 0.5975876 0.8855583 0.5789474
## 10: 0.6315789 0.26923077 0.4599428 0.7818750 0.5789474
## 11: 0.6052632 0.22343324 0.4338615 0.7596121 0.5789474
## 12: 0.6052632 0.23592493 0.4338615 0.7596121 0.5789474
## 13: 0.8666667 0.58620690 0.6927816 0.9624465 0.8333333
## 14: 0.8666667 0.58620690 0.6927816 0.9624465 0.8333333
## 15: 0.7333333 0.41463415 0.5411063 0.8772052 0.8333333
## 16: 0.6000000 0.26530612 0.4060349 0.7734424 0.8333333
## 17: 0.5333333 0.20754717 0.3432552 0.7165819 0.8333333
## 18: 0.6842105 0.13962264 0.5134729 0.8249747 0.6578947
## 19: 0.6315789 0.11627907 0.4599428 0.7818750 0.6578947
## 20: 0.4736842 0.09738717 0.3098073 0.6418166 0.6578947
## 21: 0.5263158 0.16381418 0.3581834 0.6901927 0.6578947
## 22: 0.5789474 0.21038961 0.4082145 0.7369018 0.6578947
## 23: 0.7368421 0.46022727 0.5689918 0.8659663 0.5789474
## 24: 0.7631579 0.48338369 0.5975876 0.8855583 0.5789474
## 25: 0.6315789 0.28108108 0.4599428 0.7818750 0.5789474
## 26: 0.6315789 0.26923077 0.4599428 0.7818750 0.5789474
## 27: 0.6315789 0.29255319 0.4599428 0.7818750 0.5789474
## 28: 0.8000000 0.51351351 0.6143335 0.9228645 0.8333333
## 29: 0.8000000 0.51351351 0.6143335 0.9228645 0.8333333
## 30: 0.7333333 0.41463415 0.5411063 0.8772052 0.8333333
## 31: 0.6333333 0.29787234 0.4385598 0.8007014 0.8333333
## 32: 0.5333333 0.20754717 0.3432552 0.7165819 0.8333333
## Accuracy Kappa AccuracyLower AccuracyUpper AccuracyNull
## AccuracyPValue McnemarPValue TP FN FP TN SensPlusSpec
## 1: 0.88631320 0.0233422020 18 7 0 5 1.720000
## 2: 0.70037449 0.4226780742 8 5 9 16 1.255385
## 3: 0.57465665 0.2672574932 4 9 4 21 1.147692
## 4: 0.96769274 0.0095218912 10 3 15 10 1.169231
## 5: 0.93577305 0.0523450633 9 4 13 12 1.172308
## 6: 0.88315818 0.0801183137 9 4 12 13 1.212308
## 7: 0.96769274 0.0095218912 10 3 15 10 1.169231
## 8: 0.01433379 0.0455002639 21 1 8 8 1.454545
## 9: 0.01433379 0.0455002639 21 1 8 8 1.454545
## 10: 0.31372889 0.4226780742 13 9 5 11 1.278409
## 11: 0.43825808 0.3016995825 12 10 5 11 1.232955
## 12: 0.43825808 0.1213352504 11 11 4 12 1.250000
## 13: 0.42433888 0.6170750775 22 3 1 4 1.680000
## 14: 0.42433888 0.6170750775 22 3 1 4 1.680000
## 15: 0.94943444 0.0133283288 17 8 0 5 1.680000
## 16: 0.99946013 0.0014961643 13 12 0 5 1.520000
## 17: 0.99997380 0.0005120045 11 14 0 5 1.440000
## 18: 0.43926820 0.0093747685 2 11 1 24 1.113846
## 19: 0.70037449 0.4226780742 4 9 5 20 1.107692
## 20: 0.99379566 0.0007962302 11 2 18 7 1.126154
## 21: 0.96769274 0.0021830447 11 2 16 9 1.206154
## 22: 0.88315818 0.0244489453 10 3 13 12 1.249231
## 23: 0.03303522 1.0000000000 17 5 5 11 1.460227
## 24: 0.01433379 0.0455002639 21 1 8 8 1.454545
## 25: 0.31372889 0.1814492077 12 10 4 12 1.295455
## 26: 0.31372889 0.4226780742 13 9 5 11 1.278409
## 27: 0.31372889 0.0613688291 11 11 3 13 1.312500
## 28: 0.77653712 0.0412268333 19 6 0 5 1.760000
## 29: 0.77653712 0.0412268333 19 6 0 5 1.760000
## 30: 0.94943444 0.0133283288 17 8 0 5 1.680000
## 31: 0.99796766 0.0025688315 14 11 0 5 1.560000
## 32: 0.99997380 0.0005120045 11 14 0 5 1.440000
## AccuracyPValue McnemarPValue TP FN FP TN SensPlusSpec