conformalize
conformalize.Rmd
library(misc)Example: Conformal Prediction with Out-of-Sample Coverage
In this example, we demonstrate how to use the
conformalize function to perform conformal prediction and
calculate the out-of-sample coverage rate.
Simulated Data
We will generate a simple dataset for demonstration purposes.
set.seed(123)
n <- 200
x <- matrix(runif(n * 2), ncol = 2)
y <- 3 * x[, 1] + 2 * x[, 2] + rnorm(n, sd = 0.5)
data <- data.frame(x1 = x[, 1], x2 = x[, 2], y = y)Fit Conformal Model
We will use a linear model (lm) as the
fit_func and its corresponding predict
function as the predict_func.
library(stats)
# Define fit and predict functions
fit_func <- function(formula, data, ...) lm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata = newdata, ...)
# Apply conformalize
conformal_model <- misc::conformalize(
formula = y ~ x1 + x2,
data = data,
fit_func = fit_func,
predict_func = predict_func,
split_ratio = 0.8,
seed = 123
)##
## [1] "cal_pred's value:"
## 10 15 18 19 28 33 45 47
## 1.7587270 0.4136018 0.9299653 2.6255858 2.9455003 3.2108057 1.0841493 1.5053460
## 49 58 59 61 65 66 73 75
## 2.7187733 3.1367864 3.7096010 3.8807275 3.5467970 2.5236275 2.3530161 2.9858474
## 82 95 102 104 106 107 113 114
## 2.3488926 2.4717286 1.0550776 4.3985255 3.7132151 3.1545958 1.5827951 3.0602809
## 115 119 120 133 136 142 146 148
## 2.9379282 2.5154630 2.9343982 3.5380945 3.0178942 1.5087646 1.4433821 0.9975480
## 150 151 152 160 161 174 175 199
## 3.4985747 2.9066515 3.4402581 2.3110797 1.8456383 3.2148796 3.1247111 0.7048373
##
##
## [1] "cal_y's value:"
## [1] 1.68717364 -0.09215227 0.58253361 2.00752825 2.54432351 3.26354598
## [7] 0.99154794 1.44649451 2.40032994 3.38409901 3.57146868 3.58620284
## [13] 4.62334939 3.27237760 2.61681617 3.39143939 2.64662691 3.47085590
## [19] 0.64098595 3.79594615 3.79846191 2.04948301 0.74535452 2.65303132
## [25] 2.69533179 3.30664012 3.38299733 3.13378986 1.93853703 1.94673711
## [31] 0.55729626 0.72078670 3.35304759 3.32291148 2.43114655 3.61021172
## [37] 1.69188315 3.56261005 3.32744997 0.06252609Generate Predictions and Prediction Intervals
We will use the predict.conformalize method to generate
predictions and calculate prediction intervals.
# New data for prediction
new_data <- data.frame(x1 = runif(50), x2 = runif(50))
# Predict with split conformal method
predictions <- predict(
conformal_model,
newdata = new_data,
level = 0.95,
method = "split"
)##
## [1] "object's value:"
## $fit
##
## Call:
## lm(formula = formula, data = data)
##
## Coefficients:
## (Intercept) x1 x2
## -0.000171 3.115230 1.954085
##
##
## $residuals
## 10 15 18 19 28 33
## -0.07155337 -0.50575404 -0.34743168 -0.61805759 -0.40117678 0.05274032
## 45 47 49 58 59 61
## -0.09260136 -0.05885147 -0.31844333 0.24731261 -0.13813234 -0.29452465
## 65 66 73 75 82 95
## 1.07655235 0.74875010 0.26380004 0.40559198 0.29773432 0.99912730
## 102 104 106 107 113 114
## -0.41409168 -0.60257935 0.08524682 -1.10511279 -0.83744059 -0.40724958
## 115 119 120 133 136 142
## -0.24259646 0.79117708 0.44859913 -0.40430463 -1.07935716 0.43797248
## 146 148 150 151 152 160
## -0.88608587 -0.27676132 -0.14552706 0.41625998 -1.00911160 1.29913204
## 161 174 175 199
## -0.15375511 0.34773043 0.20273890 -0.64231123
##
## $sd_residuals
## [1] 0.5868316
##
## $scaled_residuals
## [1] -0.1249228
##
## attr(,"class")
## [1] "conformalize"
head(predictions)## fit lwr upr
## 1 1.6023773 0.5217324 2.683022
## 2 2.4634938 1.3828489 3.544139
## 3 0.6216433 -0.4590017 1.702288
## 4 0.9257140 -0.1549310 2.006359
## 5 2.0106565 0.9300115 3.091301
## 6 0.7427247 -0.3379203 1.823370Calculate Out-of-Sample Coverage Rate
The coverage rate is the proportion of true values that fall within the prediction intervals.
# Simulate true values for the new data
true_y <- 3 * new_data$x1 + 2 * new_data$x2 + rnorm(50, sd = 0.5)
# Check if true values fall within the prediction intervals
coverage <- mean(true_y >= predictions[, "lwr"] & true_y <= predictions[, "upr"])
cat("Out-of-sample coverage rate:", coverage)## Out-of-sample coverage rate: 0.98Example: Conformal Prediction with the MASS::Boston
Dataset
In this example, we use the MASS::Boston dataset to
demonstrate conformal prediction.
Load the Data
We will use the MASS package to access the
Boston dataset.
## crim zn indus chas nox rm age dis rad tax ptratio black lstat
## 1 0.00632 18 2.31 0 0.538 6.575 65.2 4.0900 1 296 15.3 396.90 4.98
## 2 0.02731 0 7.07 0 0.469 6.421 78.9 4.9671 2 242 17.8 396.90 9.14
## 3 0.02729 0 7.07 0 0.469 7.185 61.1 4.9671 2 242 17.8 392.83 4.03
## 4 0.03237 0 2.18 0 0.458 6.998 45.8 6.0622 3 222 18.7 394.63 2.94
## 5 0.06905 0 2.18 0 0.458 7.147 54.2 6.0622 3 222 18.7 396.90 5.33
## 6 0.02985 0 2.18 0 0.458 6.430 58.7 6.0622 3 222 18.7 394.12 5.21
## medv
## 1 24.0
## 2 21.6
## 3 34.7
## 4 33.4
## 5 36.2
## 6 28.7Fit Conformal Model 1
# Define fit and predict functions
fit_func <- function(formula, data, ...) MASS::rlm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata, ...)
# Apply conformalize using the training data
conformal_model_boston <- misc::conformalize(
formula = medv ~ .,
data = train_data,
fit_func = fit_func,
predict_func = predict_func,
seed = 123
)##
## [1] "cal_pred's value:"
## 415 463 179 426 118 503
## -10.7510122 19.9230164 31.5185995 6.6786062 22.5467797 22.2089777
## 90 256 197 491 348 355
## 31.8908549 19.3577108 35.0827312 2.2851814 23.8991951 12.6975920
## 7 501 485 254 211 43
## 22.3623719 20.2498033 18.0662477 34.7639612 20.2588665 25.2168997
## 373 332 425 330 23 411
## 21.9027095 18.4788194 9.6184467 24.7052019 16.4429496 7.1363943
## 309 135 290 72 76 63
## 28.6541061 12.2797649 25.7463640 20.8901521 23.2575239 23.5587732
## 210 493 294 41 492 316
## 13.8915941 15.8309919 25.1340335 32.8324851 13.7123527 19.4328196
## 16 94 342 39 240 4
## 19.0849447 26.9822909 31.4961969 22.4645410 27.5833419 30.2272580
## 13 409 308 89 25 291
## 20.6180699 10.7243130 31.3363712 31.2185111 15.7459318 31.0598677
## 286 396 110 158 398 67
## 25.9575625 20.0212237 19.4838085 32.9054613 14.7002950 21.5645193
## 335 85 136 178 236 98
## 22.8363610 24.8795083 18.1195364 28.1527758 24.0979392 38.2768168
## 214 127 212 273 310 232
## 25.3294182 13.3480313 14.2333019 27.5823872 22.5939958 34.3251093
## 366 416 350 407 280 154
## 7.0687360 7.8282801 23.9417669 1.6627501 34.3544056 17.8033020
## 102 255 326 272 470 288
## 26.3044792 21.7141826 25.6599103 26.3604529 15.8676929 24.8664209
## 440 55 331 478 184 459
## 12.0857775 13.3906303 21.8053054 7.9757105 29.9670035 16.8385981
## 196 432 352 20 502 177
## 40.5135589 16.7206773 20.8573574 17.6801529 24.6479376 24.7861400
## 42 405 380 395 194 249
## 29.3599058 3.9327280 15.4752366 16.1358989 30.9911385 21.3805830
## 200 377 250 292 434 33
## 28.8764096 17.5373924 25.1091895 32.6915343 16.0014411 9.1230169
## 152 54 430 289 185 205
## 18.3026283 23.1593766 11.4940532 25.1037143 20.3957809 42.2577310
## 334 215 318 57 105 279
## 23.3475381 10.8445036 17.9629655 23.2551312 20.7019461 28.6382860
## 129 106 480 369 406 287
## 19.7588518 17.2377805 19.6030886 18.1008648 3.2428357 18.4863446
## 356 225 117 452 400 341
## 15.3592737 40.9207212 22.8719139 20.1249319 10.2406435 21.2020459
## 176 220 191 53 104 447
## 30.6648246 27.9061697 30.5701378 27.6415501 19.7787486 17.7108334
## 320 354 275 261 2 336
## 21.5330307 24.8829332 33.5210609 35.8664158 24.9337907 21.2926956
## 497 394 448 327 419 112
## 13.1945357 19.2910225 18.0289641 24.2997967 -0.5370632 26.9174610
## 36 87 399 111 461 351
## 22.5015340 21.5992251 3.4521847 20.7249385 19.5192170 21.1407909
## 31 73 424 333 122 92
## 11.3470747 23.7686787 10.1458420 22.0764050 21.3973045 26.8734678
## 234 338 472 300 323 314
## 40.1021613 19.3254374 20.7501692 31.1592506 22.4066161 24.9947580
## 443 387 376 208 499 206
## 18.7201079 1.8760970 25.5735072 16.9928743 21.1111805 22.0502650
## 297 219 82 169 413 228
## 27.2139228 22.6363937 26.1307032 24.9567002 -4.7890163 32.7195829
## 423 258 385 482 498 192
## 14.7312882 47.0965692 -1.5888701 25.8169897 18.6489127 29.5093562
## 386 414 50 260 97 183
## 5.8590075 5.5807664 16.2640999 34.6740553 23.6527792 34.0873388
## 100 274 484 296 357 227
## 33.6611952 36.1678621 18.8250647 28.8095974 18.7538084 39.5641575
## 190 329 156 131 8 468
## 33.8903426 20.7253064 19.7782737 20.7893637 19.5814632 14.9795254
## 70 213 382 182
## 19.7314621 20.6109097 17.9718921 26.5244034
##
##
## [1] "cal_y's value:"
## [1] 7.0 19.5 29.9 8.3 19.2 20.6 28.7 20.9 33.3 8.1 23.1 18.2 22.9 16.8 20.6
## [16] 42.8 21.7 25.3 50.0 17.1 11.7 22.6 15.2 15.0 22.8 15.6 24.8 21.7 21.4 22.2
## [31] 20.0 20.1 23.9 34.9 13.6 16.2 19.9 25.0 32.7 24.7 23.3 33.4 21.7 17.2 28.2
## [46] 23.6 15.6 28.5 22.0 13.1 19.4 41.3 8.5 19.4 20.7 23.9 18.1 24.6 24.0 38.7
## [61] 28.1 15.7 19.3 24.4 20.3 31.7 27.5 7.2 26.6 11.9 35.1 19.4 26.5 21.9 24.6
## [76] 25.2 20.1 23.2 12.8 18.9 19.8 12.0 32.5 14.9 50.0 14.1 24.1 18.2 22.4 23.2
## [91] 26.6 8.5 10.2 12.7 31.1 24.5 34.9 13.9 26.2 37.3 14.3 13.2 19.6 23.4 9.5
## [106] 22.3 26.4 50.0 22.2 23.7 19.8 24.7 20.1 29.1 18.0 19.5 21.4 50.0 5.0 20.1
## [121] 20.6 44.8 21.2 15.2 6.3 18.7 29.4 23.0 37.0 25.0 19.3 14.9 21.0 30.1 32.4
## [136] 33.8 21.6 21.1 19.7 13.8 12.6 23.0 8.8 22.8 18.9 22.5 5.0 21.7 16.4 22.9
## [151] 12.7 22.8 13.4 19.4 20.3 22.0 48.3 18.5 19.6 29.0 20.4 21.6 18.4 10.5 15.0
## [166] 22.5 21.2 22.6 27.1 21.5 23.9 23.8 17.9 31.6 20.8 50.0 8.8 23.7 18.3 30.5
## [181] 7.2 16.3 19.4 30.1 21.4 37.9 33.2 35.2 21.8 28.6 17.8 37.6 34.9 19.3 15.6
## [196] 19.2 27.1 19.1 20.9 22.4 10.9 36.2Generate Predictions and Prediction Intervals 1
We will use the predict.conformalize method to generate
predictions and calculate prediction intervals for the test set.
# Predict with split conformal method on the test data
predictions_boston <- predict(
conformal_model_boston,
newdata = test_data,
level = 0.95,
method = "split"
)##
## [1] "object's value:"
## $fit
## Call:
## rlm(formula = formula, data = data)
## Converged in 12 iterations
##
## Coefficients:
## (Intercept) crim zn indus chas nox
## 10.053413833 -0.163015431 0.009957524 -0.013355275 1.557351564 -5.289064676
## rm age dis rad tax ptratio
## 6.050082402 -0.014130776 -1.046959248 0.204156664 -0.010752932 -0.815953461
## black lstat
## 0.014196664 -0.436463429
##
## Degrees of freedom: 202 total; 188 residual
## Scale estimate: 2.78
##
## $residuals
## 415 463 179 426 118 503
## 17.75101224 -0.42301639 -1.61859953 1.62139385 -3.34677969 -1.60897767
## 90 256 197 491 348 355
## -3.19085491 1.54228923 -1.78273122 5.81481859 -0.79919511 5.50240802
## 7 501 485 254 211 43
## 0.53762811 -3.44980333 2.53375234 8.03603880 1.44113352 0.08310029
## 373 332 425 330 23 411
## 28.09729049 -1.37881939 2.08155331 -2.10520186 -1.24294958 7.86360572
## 309 135 290 72 76 63
## -5.85410610 3.32023507 -0.94636403 0.80984792 -1.85752392 -1.35877320
## 210 493 294 41 492 316
## 6.10840592 4.26900808 -1.23403353 2.06751492 -0.11235268 -3.23281956
## 16 94 342 39 240 4
## 0.81505531 -1.98229092 1.20380307 2.23545905 -4.28334192 3.17274197
## 13 409 308 89 25 291
## 1.08193007 6.47568699 -3.13637115 -7.61851115 -0.14593182 -2.55986770
## 286 396 110 158 398 67
## -3.95756249 -6.92122367 -0.08380848 8.39453871 -6.20029505 -2.16451929
## 335 85 136 178 236 98
## -2.13636104 -0.97950834 -0.01953636 -3.55277580 -0.09793921 0.42318320
## 214 127 212 273 310 232
## 2.77058176 2.35196874 5.06669808 -3.18238722 -2.29399575 -2.62510928
## 366 416 350 407 280 154
## 20.43126404 -0.62828007 2.65823308 10.23724986 0.74559441 1.59669803
## 102 255 326 272 470 288
## 0.19552077 0.18581742 -1.05991028 -1.16045291 4.23230707 -1.66642094
## 440 55 331 478 184 459
## 0.71422248 5.50936970 -2.00530538 4.02428952 2.53299654 -1.93859807
## 196 432 352 20 502 177
## 9.48644109 -2.62067731 3.24264261 0.51984714 -2.24793764 -1.58613996
## 42 405 380 395 194 249
## -2.75990583 4.56727197 -5.27523660 -3.43589886 0.10886146 3.11941696
## 200 377 250 292 434 33
## 6.02359042 -3.63739240 1.09081050 4.60846568 -1.70144111 4.07698311
## 152 54 430 289 185 205
## 1.29737169 0.24062338 -1.99405324 -2.80371431 6.00421913 7.74226900
## 334 215 318 57 105 279
## -1.14753812 12.85549644 1.83703448 1.44486879 -0.60194612 0.46171405
## 129 106 480 369 406 287
## -1.75885183 2.26221951 1.79691136 31.89913520 1.75716435 1.61365538
## 356 225 117 452 400 341
## 5.24072632 3.87927876 -1.67191387 -4.92493185 -3.94064347 -2.50204595
## 176 220 191 53 104 447
## -1.26482459 -4.90616974 6.42986225 -2.64155010 -0.47874865 -2.81083340
## 320 354 275 261 2 336
## -0.53303073 5.21706683 -1.12106092 -2.06641580 -3.33379070 -0.19269560
## 497 394 448 327 419 112
## 6.50546425 -5.49102250 -5.42896409 -1.29979669 9.33706323 -4.11746097
## 36 87 399 111 461 351
## -3.60153400 0.90077494 1.54781531 0.97506146 -3.11921704 1.75920907
## 31 73 424 333 122 92
## 1.35292531 -0.96867871 3.25415805 -2.67640505 -1.09730455 -4.87346776
## 234 338 472 300 323 314
## 8.19783868 -0.82543739 -1.15016922 -2.15925065 -2.00661608 -3.39475797
## 443 387 376 208 499 206
## -0.32010786 8.62390298 -10.57350717 5.50712569 0.08881949 0.54973502
## 297 219 82 169 413 228
## -0.11392277 -1.13639371 -2.23070320 -1.15670018 22.68901628 -1.11958292
## 423 258 385 482 498 192
## 6.06871179 2.90343078 10.38887008 -2.11698974 -0.34891269 0.99064382
## 386 414 50 260 97 183
## 1.34099245 10.71923362 3.13590006 -4.57405529 -2.25277923 3.81266117
## 100 274 484 296 357 227
## -0.46119518 -0.96786208 2.97493526 -0.20959741 -0.95380842 -1.96415748
## 190 329 156 131 8 468
## 1.00965741 -1.42530635 -4.17827374 -1.58936371 7.51853676 4.12047455
## 70 213 382 182
## 1.16853787 1.78909033 -7.07189205 9.67559663
##
## $sd_residuals
## [1] 5.384367
##
## $scaled_residuals
## [1] 0.1888384
##
## attr(,"class")
## [1] "conformalize"
head(predictions_boston)## fit lwr upr
## 1 29.92942 20.263283 39.59556
## 15 19.30837 9.642229 28.97451
## 17 20.71124 11.045100 30.37738
## 19 14.86650 5.200365 24.53264
## 28 14.79883 5.132688 24.46497
## 37 20.98752 11.321382 30.65366Calculate Out-of-Sample Coverage Rate 1
The coverage rate is the proportion of true values in the test set that fall within the prediction intervals.
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston[, "lwr"] & true_y_boston <= predictions_boston[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9509804Fit Conformal Model 2
# Define fit and predict functions
fit_func <- function(formula, data, ...) stats::glm(formula, data = data, ...)
predict_func <- function(fit, newdata, ...) predict(fit, newdata, ...)
# Apply conformalize using the training data
conformal_model_boston <- misc::conformalize(
formula = medv ~ .,
data = train_data,
fit_func = fit_func,
predict_func = predict_func,
seed = 123
)##
## [1] "cal_pred's value:"
## 415 463 179 426 118 503 90
## -8.7644778 20.4887700 32.4431800 8.0537364 23.8121734 22.2938278 31.6500690
## 256 197 491 348 355 7 501
## 19.0300341 35.7306104 1.4585692 23.4878970 11.1516613 22.7052511 20.2938143
## 485 254 211 43 373 332 425
## 18.9060798 30.5070434 21.5618408 24.1459959 25.9721831 17.9036330 12.5399210
## 330 23 411 309 135 290 72
## 23.3226121 15.3677673 12.8320590 29.6251696 12.3248995 25.5594140 20.2696103
## 76 63 210 493 294 41 492
## 23.4200877 23.8745524 15.4131455 15.5476909 25.0260630 33.2030309 13.3378217
## 316 16 94 342 39 240 4
## 20.1703415 18.5664504 28.2417841 30.5026168 21.9246764 27.9948440 29.0931062
## 13 409 308 89 25 291 286
## 19.8035888 12.8261140 33.0949021 31.8487090 15.2011380 32.3218400 25.9934342
## 396 110 158 398 67 335 85
## 21.1528112 19.5747079 35.1293630 16.4121377 23.1791003 21.1203071 24.4056852
## 136 178 236 98 214 127 212
## 17.3779995 29.6825766 25.1125546 37.6996950 24.6829527 13.6314186 15.2709160
## 273 310 232 366 416 350 407
## 28.3734896 23.5577783 34.9175378 12.9957994 7.9899564 21.1785344 5.9424726
## 280 154 102 255 326 272 470
## 35.4886870 17.4120084 26.0339039 21.8928182 24.0415187 26.1183844 17.8232456
## 288 440 55 331 478 184 459
## 25.7538563 12.7911744 12.5504228 20.6041636 10.3361111 32.0606716 17.2599642
## 196 432 352 20 502 177 42
## 41.0881102 18.2461988 19.0540202 17.6366627 23.6808022 25.1921824 27.5022924
## 405 380 395 194 249 200 377
## 5.1723821 16.8672259 18.1006331 30.8682121 20.5225776 28.4212598 17.9233812
## 250 292 434 33 152 54 430
## 23.2677742 33.5271731 16.7486737 6.8414592 18.6530154 22.7714683 12.0467175
## 289 185 205 334 215 318 57
## 26.2412653 22.2977151 43.3929013 21.7699815 7.9719117 17.7004893 23.0554043
## 105 279 129 106 480 369 406
## 21.3706619 29.5260739 19.2713597 18.0736162 22.1926480 24.1558127 5.9687277
## 287 356 225 117 452 400 341
## 17.4436708 13.7473070 40.9180042 23.3522495 20.3660747 9.8032508 20.9623055
## 176 220 191 53 104 447 320
## 30.8175490 30.0356168 30.1363936 27.0229588 20.1027091 18.2050509 20.9066090
## 354 275 261 2 336 497 394
## 24.1108414 35.2517632 36.3976123 25.3136956 19.8363708 12.7245499 20.8504024
## 448 327 419 112 36 87 399
## 18.7870677 23.1387268 2.6959032 27.2667753 23.5168621 21.2154997 4.6992212
## 111 461 351 31 73 424 333
## 19.9418111 19.6213424 19.2806184 10.2712594 23.3276945 11.5543249 21.6217116
## 122 92 234 338 472 300 323
## 22.4897982 27.6669215 39.5246817 18.6624370 23.6006840 30.7842961 22.3775258
## 314 443 387 376 208 499 206
## 25.9959974 19.6012817 4.1922756 26.7380187 16.9621199 20.9530051 21.4977779
## 297 219 82 169 413 228 423
## 27.5602693 24.4871580 26.9210023 27.0052004 -1.9533084 33.7920242 18.0024708
## 258 385 482 498 192 386 414
## 46.5858214 0.5286619 28.1513628 18.5667871 29.7077926 6.7573764 9.8123753
## 50 260 97 183 100 274 484
## 16.0115456 36.7297450 24.3784267 35.4727852 33.2315844 35.7702568 20.5618277
## 296 357 227 190 329 156 131
## 28.5041263 19.7332012 40.1317263 34.5174330 19.8456908 19.1819223 20.5464449
## 8 468 70 213 382 182
## 19.4785721 16.7704167 19.3717667 21.1761913 18.8995599 27.6566413
##
##
## [1] "cal_y's value:"
## [1] 7.0 19.5 29.9 8.3 19.2 20.6 28.7 20.9 33.3 8.1 23.1 18.2 22.9 16.8 20.6
## [16] 42.8 21.7 25.3 50.0 17.1 11.7 22.6 15.2 15.0 22.8 15.6 24.8 21.7 21.4 22.2
## [31] 20.0 20.1 23.9 34.9 13.6 16.2 19.9 25.0 32.7 24.7 23.3 33.4 21.7 17.2 28.2
## [46] 23.6 15.6 28.5 22.0 13.1 19.4 41.3 8.5 19.4 20.7 23.9 18.1 24.6 24.0 38.7
## [61] 28.1 15.7 19.3 24.4 20.3 31.7 27.5 7.2 26.6 11.9 35.1 19.4 26.5 21.9 24.6
## [76] 25.2 20.1 23.2 12.8 18.9 19.8 12.0 32.5 14.9 50.0 14.1 24.1 18.2 22.4 23.2
## [91] 26.6 8.5 10.2 12.7 31.1 24.5 34.9 13.9 26.2 37.3 14.3 13.2 19.6 23.4 9.5
## [106] 22.3 26.4 50.0 22.2 23.7 19.8 24.7 20.1 29.1 18.0 19.5 21.4 50.0 5.0 20.1
## [121] 20.6 44.8 21.2 15.2 6.3 18.7 29.4 23.0 37.0 25.0 19.3 14.9 21.0 30.1 32.4
## [136] 33.8 21.6 21.1 19.7 13.8 12.6 23.0 8.8 22.8 18.9 22.5 5.0 21.7 16.4 22.9
## [151] 12.7 22.8 13.4 19.4 20.3 22.0 48.3 18.5 19.6 29.0 20.4 21.6 18.4 10.5 15.0
## [166] 22.5 21.2 22.6 27.1 21.5 23.9 23.8 17.9 31.6 20.8 50.0 8.8 23.7 18.3 30.5
## [181] 7.2 16.3 19.4 30.1 21.4 37.9 33.2 35.2 21.8 28.6 17.8 37.6 34.9 19.3 15.6
## [196] 19.2 27.1 19.1 20.9 22.4 10.9 36.2Generate Predictions and Prediction Intervals 2
We will use the predict.conformalize method to generate
predictions and calculate prediction intervals for the test set.
# Predict with split conformal method on the test data
predictions_boston <- predict(
conformal_model_boston,
newdata = test_data,
level = 0.95,
method = "split"
)##
## [1] "object's value:"
## $fit
##
## Call: stats::glm(formula = formula, data = data)
##
## Coefficients:
## (Intercept) crim zn indus chas nox
## 29.78825 -0.14238 0.03251 0.03061 2.17196 -16.17824
## rm age dis rad tax ptratio
## 4.70344 0.01813 -1.50717 0.35519 -0.01344 -1.04058
## black lstat
## 0.01303 -0.58559
##
## Degrees of Freedom: 201 Total (i.e. Null); 188 Residual
## Null Deviance: 18580
## Residual Deviance: 3991 AIC: 1206
##
## $residuals
## 415 463 179 426 118
## 15.764477831 -0.988770013 -2.543179984 0.246263589 -4.612173369
## 503 90 256 197 491
## -1.693827822 -2.950069022 1.869965859 -2.430610387 6.641430762
## 348 355 7 501 485
## -0.387896963 7.048338684 0.194748907 -3.493814316 1.693920234
## 254 211 43 373 332
## 12.292956642 0.138159179 1.154004128 24.027816881 -0.803632969
## 425 330 23 411 309
## -0.839920969 -0.722612132 -0.167767308 2.167940990 -6.825169566
## 135 290 72 76 63
## 3.275100495 -0.759414026 1.430389678 -2.020087653 -1.674552399
## 210 493 294 41 492
## 4.586854531 4.552309079 -1.126062979 1.696969057 0.262178313
## 316 16 94 342 39
## -3.970341530 1.333549618 -3.241784129 2.197383166 2.775323621
## 240 4 13 409 308
## -4.694843972 4.306893821 1.896411213 4.373885971 -4.894902111
## 89 25 291 286 396
## -8.248709022 0.398861978 -3.821840049 -3.993434219 -8.052811215
## 110 158 398 67 335
## -0.174707897 6.170637038 -7.912137723 -3.779100281 -0.420307060
## 85 136 178 236 98
## -0.505685158 0.722000550 -5.082576649 -1.112554638 1.000305006
## 214 127 212 273 310
## 3.417047314 2.068581362 4.029083985 -3.973489628 -3.257778331
## 232 366 416 350 407
## -3.217537805 14.504200597 -0.789956450 5.421465622 5.957527419
## 280 154 102 255 326
## -0.388687016 1.987991575 0.466096118 0.007181811 0.558481278
## 272 470 288 440 55
## -0.918384377 2.276754357 -2.553856315 0.008825628 6.349577246
## 331 478 184 459 196
## -0.804163641 1.663888937 0.439328371 -2.359964178 8.911889804
## 432 352 20 502 177
## -4.146198759 5.045979801 0.563337346 -1.280802168 -1.992182379
## 42 405 380 395 194
## -0.902292374 3.327617862 -6.667225863 -5.400633140 0.231787928
## 249 200 377 250 292
## 3.977422436 6.478740182 -4.023381192 2.932225794 3.772826861
## 434 33 152 54 430
## -2.448673653 6.358540758 0.946984605 0.628531680 -2.546717499
## 289 185 205 334 215
## -3.941265332 4.102284871 6.607098719 0.430018524 15.728088344
## 318 57 105 279 129
## 2.099510664 1.644595746 -1.270661910 -0.426073914 -1.271359683
## 106 480 369 406 287
## 1.426383844 -0.792647956 25.844187265 -0.968727677 2.656329159
## 356 225 117 452 400
## 6.852693030 3.881995799 -2.152249451 -5.166074745 -3.503250830
## 341 176 220 191 53
## -2.262305509 -1.417549048 -7.035616819 6.863606354 -2.022958762
## 104 447 320 354 275
## -0.802709122 -3.305050943 0.093391012 5.989158592 -2.851763168
## 261 2 336 497 394
## -2.597612328 -3.713695585 1.263629192 6.975450133 -7.050402445
## 448 327 419 112 36
## -6.187067749 -0.138726762 6.104096769 -4.466775330 -4.616862136
## 87 399 111 461 351
## 1.284500325 0.300778763 1.758188903 -3.221342364 3.619381597
## 31 73 424 333 122
## 2.428740626 -0.527694477 1.845675138 -2.221711574 -2.189798218
## 92 234 338 472 300
## -5.666921491 8.775318305 -0.162437018 -4.000684029 -1.784296138
## 323 314 443 387 376
## -1.977525789 -4.395997397 -1.201281672 6.307724361 -11.738018736
## 208 499 206 297 219
## 5.537880093 0.246994887 1.102222072 -0.460269305 -2.987158004
## 82 169 413 228 423
## -3.021002286 -3.205200388 19.853308439 -2.192024195 2.797529204
## 258 385 482 498 192
## 3.414178564 8.271338091 -4.451362820 -0.266787057 0.792207439
## 386 414 50 260 97
## 0.442623562 6.487624676 3.388454435 -6.629745005 -2.978426708
## 183 100 274 484 296
## 2.427214778 -0.031584384 -0.570256827 1.238172323 0.095873701
## 357 227 190 329 156
## -1.933201226 -2.531726289 0.382566988 -0.545690834 -3.581922258
## 131 8 468 70 213
## -1.346444916 7.621427858 2.329583294 1.528233254 1.223808670
## 382 182
## -7.999559935 8.543358664
##
## $sd_residuals
## [1] 5.091575
##
## $scaled_residuals
## [1] 0.1125522
##
## attr(,"class")
## [1] "conformalize"
head(predictions_boston)## fit lwr upr
## 1 30.39434 21.864586 38.92410
## 15 19.10867 10.578908 27.63842
## 17 19.48015 10.950390 28.00990
## 19 14.18587 5.656113 22.71563
## 28 13.85324 5.323482 22.38300
## 37 21.66451 13.134753 30.19427
# Predict with split conformal method on the test data
predictions_boston2 <- predict(
conformal_model_boston,
newdata = test_data,
level = 0.95,
method = "kde"
)##
## [1] "object's value:"
## $fit
##
## Call: stats::glm(formula = formula, data = data)
##
## Coefficients:
## (Intercept) crim zn indus chas nox
## 29.78825 -0.14238 0.03251 0.03061 2.17196 -16.17824
## rm age dis rad tax ptratio
## 4.70344 0.01813 -1.50717 0.35519 -0.01344 -1.04058
## black lstat
## 0.01303 -0.58559
##
## Degrees of Freedom: 201 Total (i.e. Null); 188 Residual
## Null Deviance: 18580
## Residual Deviance: 3991 AIC: 1206
##
## $residuals
## 415 463 179 426 118
## 15.764477831 -0.988770013 -2.543179984 0.246263589 -4.612173369
## 503 90 256 197 491
## -1.693827822 -2.950069022 1.869965859 -2.430610387 6.641430762
## 348 355 7 501 485
## -0.387896963 7.048338684 0.194748907 -3.493814316 1.693920234
## 254 211 43 373 332
## 12.292956642 0.138159179 1.154004128 24.027816881 -0.803632969
## 425 330 23 411 309
## -0.839920969 -0.722612132 -0.167767308 2.167940990 -6.825169566
## 135 290 72 76 63
## 3.275100495 -0.759414026 1.430389678 -2.020087653 -1.674552399
## 210 493 294 41 492
## 4.586854531 4.552309079 -1.126062979 1.696969057 0.262178313
## 316 16 94 342 39
## -3.970341530 1.333549618 -3.241784129 2.197383166 2.775323621
## 240 4 13 409 308
## -4.694843972 4.306893821 1.896411213 4.373885971 -4.894902111
## 89 25 291 286 396
## -8.248709022 0.398861978 -3.821840049 -3.993434219 -8.052811215
## 110 158 398 67 335
## -0.174707897 6.170637038 -7.912137723 -3.779100281 -0.420307060
## 85 136 178 236 98
## -0.505685158 0.722000550 -5.082576649 -1.112554638 1.000305006
## 214 127 212 273 310
## 3.417047314 2.068581362 4.029083985 -3.973489628 -3.257778331
## 232 366 416 350 407
## -3.217537805 14.504200597 -0.789956450 5.421465622 5.957527419
## 280 154 102 255 326
## -0.388687016 1.987991575 0.466096118 0.007181811 0.558481278
## 272 470 288 440 55
## -0.918384377 2.276754357 -2.553856315 0.008825628 6.349577246
## 331 478 184 459 196
## -0.804163641 1.663888937 0.439328371 -2.359964178 8.911889804
## 432 352 20 502 177
## -4.146198759 5.045979801 0.563337346 -1.280802168 -1.992182379
## 42 405 380 395 194
## -0.902292374 3.327617862 -6.667225863 -5.400633140 0.231787928
## 249 200 377 250 292
## 3.977422436 6.478740182 -4.023381192 2.932225794 3.772826861
## 434 33 152 54 430
## -2.448673653 6.358540758 0.946984605 0.628531680 -2.546717499
## 289 185 205 334 215
## -3.941265332 4.102284871 6.607098719 0.430018524 15.728088344
## 318 57 105 279 129
## 2.099510664 1.644595746 -1.270661910 -0.426073914 -1.271359683
## 106 480 369 406 287
## 1.426383844 -0.792647956 25.844187265 -0.968727677 2.656329159
## 356 225 117 452 400
## 6.852693030 3.881995799 -2.152249451 -5.166074745 -3.503250830
## 341 176 220 191 53
## -2.262305509 -1.417549048 -7.035616819 6.863606354 -2.022958762
## 104 447 320 354 275
## -0.802709122 -3.305050943 0.093391012 5.989158592 -2.851763168
## 261 2 336 497 394
## -2.597612328 -3.713695585 1.263629192 6.975450133 -7.050402445
## 448 327 419 112 36
## -6.187067749 -0.138726762 6.104096769 -4.466775330 -4.616862136
## 87 399 111 461 351
## 1.284500325 0.300778763 1.758188903 -3.221342364 3.619381597
## 31 73 424 333 122
## 2.428740626 -0.527694477 1.845675138 -2.221711574 -2.189798218
## 92 234 338 472 300
## -5.666921491 8.775318305 -0.162437018 -4.000684029 -1.784296138
## 323 314 443 387 376
## -1.977525789 -4.395997397 -1.201281672 6.307724361 -11.738018736
## 208 499 206 297 219
## 5.537880093 0.246994887 1.102222072 -0.460269305 -2.987158004
## 82 169 413 228 423
## -3.021002286 -3.205200388 19.853308439 -2.192024195 2.797529204
## 258 385 482 498 192
## 3.414178564 8.271338091 -4.451362820 -0.266787057 0.792207439
## 386 414 50 260 97
## 0.442623562 6.487624676 3.388454435 -6.629745005 -2.978426708
## 183 100 274 484 296
## 2.427214778 -0.031584384 -0.570256827 1.238172323 0.095873701
## 357 227 190 329 156
## -1.933201226 -2.531726289 0.382566988 -0.545690834 -3.581922258
## 131 8 468 70 213
## -1.346444916 7.621427858 2.329583294 1.528233254 1.223808670
## 382 182
## -7.999559935 8.543358664
##
## $sd_residuals
## [1] 5.091575
##
## $scaled_residuals
## [1] 0.1125522
##
## attr(,"class")
## [1] "conformalize"
head(predictions_boston2)## fit lwr upr
## 1 30.39434 22.37589 49.43768
## 15 19.10867 11.96002 28.22319
## 17 19.48015 11.91617 34.06139
## 19 14.18587 6.77628 23.56133
## 28 13.85324 6.61761 27.71907
## 37 21.66451 13.49167 37.83532
# Predict with split conformal method on the test data
predictions_boston3 <- predict(
conformal_model_boston,
newdata = test_data,
level = 0.95,
method = "surrogate"
)##
## [1] "object's value:"
## $fit
##
## Call: stats::glm(formula = formula, data = data)
##
## Coefficients:
## (Intercept) crim zn indus chas nox
## 29.78825 -0.14238 0.03251 0.03061 2.17196 -16.17824
## rm age dis rad tax ptratio
## 4.70344 0.01813 -1.50717 0.35519 -0.01344 -1.04058
## black lstat
## 0.01303 -0.58559
##
## Degrees of Freedom: 201 Total (i.e. Null); 188 Residual
## Null Deviance: 18580
## Residual Deviance: 3991 AIC: 1206
##
## $residuals
## 415 463 179 426 118
## 15.764477831 -0.988770013 -2.543179984 0.246263589 -4.612173369
## 503 90 256 197 491
## -1.693827822 -2.950069022 1.869965859 -2.430610387 6.641430762
## 348 355 7 501 485
## -0.387896963 7.048338684 0.194748907 -3.493814316 1.693920234
## 254 211 43 373 332
## 12.292956642 0.138159179 1.154004128 24.027816881 -0.803632969
## 425 330 23 411 309
## -0.839920969 -0.722612132 -0.167767308 2.167940990 -6.825169566
## 135 290 72 76 63
## 3.275100495 -0.759414026 1.430389678 -2.020087653 -1.674552399
## 210 493 294 41 492
## 4.586854531 4.552309079 -1.126062979 1.696969057 0.262178313
## 316 16 94 342 39
## -3.970341530 1.333549618 -3.241784129 2.197383166 2.775323621
## 240 4 13 409 308
## -4.694843972 4.306893821 1.896411213 4.373885971 -4.894902111
## 89 25 291 286 396
## -8.248709022 0.398861978 -3.821840049 -3.993434219 -8.052811215
## 110 158 398 67 335
## -0.174707897 6.170637038 -7.912137723 -3.779100281 -0.420307060
## 85 136 178 236 98
## -0.505685158 0.722000550 -5.082576649 -1.112554638 1.000305006
## 214 127 212 273 310
## 3.417047314 2.068581362 4.029083985 -3.973489628 -3.257778331
## 232 366 416 350 407
## -3.217537805 14.504200597 -0.789956450 5.421465622 5.957527419
## 280 154 102 255 326
## -0.388687016 1.987991575 0.466096118 0.007181811 0.558481278
## 272 470 288 440 55
## -0.918384377 2.276754357 -2.553856315 0.008825628 6.349577246
## 331 478 184 459 196
## -0.804163641 1.663888937 0.439328371 -2.359964178 8.911889804
## 432 352 20 502 177
## -4.146198759 5.045979801 0.563337346 -1.280802168 -1.992182379
## 42 405 380 395 194
## -0.902292374 3.327617862 -6.667225863 -5.400633140 0.231787928
## 249 200 377 250 292
## 3.977422436 6.478740182 -4.023381192 2.932225794 3.772826861
## 434 33 152 54 430
## -2.448673653 6.358540758 0.946984605 0.628531680 -2.546717499
## 289 185 205 334 215
## -3.941265332 4.102284871 6.607098719 0.430018524 15.728088344
## 318 57 105 279 129
## 2.099510664 1.644595746 -1.270661910 -0.426073914 -1.271359683
## 106 480 369 406 287
## 1.426383844 -0.792647956 25.844187265 -0.968727677 2.656329159
## 356 225 117 452 400
## 6.852693030 3.881995799 -2.152249451 -5.166074745 -3.503250830
## 341 176 220 191 53
## -2.262305509 -1.417549048 -7.035616819 6.863606354 -2.022958762
## 104 447 320 354 275
## -0.802709122 -3.305050943 0.093391012 5.989158592 -2.851763168
## 261 2 336 497 394
## -2.597612328 -3.713695585 1.263629192 6.975450133 -7.050402445
## 448 327 419 112 36
## -6.187067749 -0.138726762 6.104096769 -4.466775330 -4.616862136
## 87 399 111 461 351
## 1.284500325 0.300778763 1.758188903 -3.221342364 3.619381597
## 31 73 424 333 122
## 2.428740626 -0.527694477 1.845675138 -2.221711574 -2.189798218
## 92 234 338 472 300
## -5.666921491 8.775318305 -0.162437018 -4.000684029 -1.784296138
## 323 314 443 387 376
## -1.977525789 -4.395997397 -1.201281672 6.307724361 -11.738018736
## 208 499 206 297 219
## 5.537880093 0.246994887 1.102222072 -0.460269305 -2.987158004
## 82 169 413 228 423
## -3.021002286 -3.205200388 19.853308439 -2.192024195 2.797529204
## 258 385 482 498 192
## 3.414178564 8.271338091 -4.451362820 -0.266787057 0.792207439
## 386 414 50 260 97
## 0.442623562 6.487624676 3.388454435 -6.629745005 -2.978426708
## 183 100 274 484 296
## 2.427214778 -0.031584384 -0.570256827 1.238172323 0.095873701
## 357 227 190 329 156
## -1.933201226 -2.531726289 0.382566988 -0.545690834 -3.581922258
## 131 8 468 70 213
## -1.346444916 7.621427858 2.329583294 1.528233254 1.223808670
## 382 182
## -7.999559935 8.543358664
##
## $sd_residuals
## [1] 5.091575
##
## $scaled_residuals
## [1] 0.1125522
##
## attr(,"class")
## [1] "conformalize"## Registered S3 method overwritten by 'quantmod':
## method from
## as.zoo.data.frame zoo
head(predictions_boston3)## fit lwr upr
## 1 30.39434 22.482206 39.30623
## 15 19.10867 12.073048 31.40162
## 17 19.48015 12.698948 34.93286
## 19 14.18587 7.396239 29.63858
## 28 13.85324 6.134993 22.68221
## 37 21.66451 14.614108 33.95747
# Predict with split conformal method on the test data
predictions_boston4 <- predict(
conformal_model_boston,
newdata = test_data,
level = 0.95,
method = "bootstrap"
)##
## [1] "object's value:"
## $fit
##
## Call: stats::glm(formula = formula, data = data)
##
## Coefficients:
## (Intercept) crim zn indus chas nox
## 29.78825 -0.14238 0.03251 0.03061 2.17196 -16.17824
## rm age dis rad tax ptratio
## 4.70344 0.01813 -1.50717 0.35519 -0.01344 -1.04058
## black lstat
## 0.01303 -0.58559
##
## Degrees of Freedom: 201 Total (i.e. Null); 188 Residual
## Null Deviance: 18580
## Residual Deviance: 3991 AIC: 1206
##
## $residuals
## 415 463 179 426 118
## 15.764477831 -0.988770013 -2.543179984 0.246263589 -4.612173369
## 503 90 256 197 491
## -1.693827822 -2.950069022 1.869965859 -2.430610387 6.641430762
## 348 355 7 501 485
## -0.387896963 7.048338684 0.194748907 -3.493814316 1.693920234
## 254 211 43 373 332
## 12.292956642 0.138159179 1.154004128 24.027816881 -0.803632969
## 425 330 23 411 309
## -0.839920969 -0.722612132 -0.167767308 2.167940990 -6.825169566
## 135 290 72 76 63
## 3.275100495 -0.759414026 1.430389678 -2.020087653 -1.674552399
## 210 493 294 41 492
## 4.586854531 4.552309079 -1.126062979 1.696969057 0.262178313
## 316 16 94 342 39
## -3.970341530 1.333549618 -3.241784129 2.197383166 2.775323621
## 240 4 13 409 308
## -4.694843972 4.306893821 1.896411213 4.373885971 -4.894902111
## 89 25 291 286 396
## -8.248709022 0.398861978 -3.821840049 -3.993434219 -8.052811215
## 110 158 398 67 335
## -0.174707897 6.170637038 -7.912137723 -3.779100281 -0.420307060
## 85 136 178 236 98
## -0.505685158 0.722000550 -5.082576649 -1.112554638 1.000305006
## 214 127 212 273 310
## 3.417047314 2.068581362 4.029083985 -3.973489628 -3.257778331
## 232 366 416 350 407
## -3.217537805 14.504200597 -0.789956450 5.421465622 5.957527419
## 280 154 102 255 326
## -0.388687016 1.987991575 0.466096118 0.007181811 0.558481278
## 272 470 288 440 55
## -0.918384377 2.276754357 -2.553856315 0.008825628 6.349577246
## 331 478 184 459 196
## -0.804163641 1.663888937 0.439328371 -2.359964178 8.911889804
## 432 352 20 502 177
## -4.146198759 5.045979801 0.563337346 -1.280802168 -1.992182379
## 42 405 380 395 194
## -0.902292374 3.327617862 -6.667225863 -5.400633140 0.231787928
## 249 200 377 250 292
## 3.977422436 6.478740182 -4.023381192 2.932225794 3.772826861
## 434 33 152 54 430
## -2.448673653 6.358540758 0.946984605 0.628531680 -2.546717499
## 289 185 205 334 215
## -3.941265332 4.102284871 6.607098719 0.430018524 15.728088344
## 318 57 105 279 129
## 2.099510664 1.644595746 -1.270661910 -0.426073914 -1.271359683
## 106 480 369 406 287
## 1.426383844 -0.792647956 25.844187265 -0.968727677 2.656329159
## 356 225 117 452 400
## 6.852693030 3.881995799 -2.152249451 -5.166074745 -3.503250830
## 341 176 220 191 53
## -2.262305509 -1.417549048 -7.035616819 6.863606354 -2.022958762
## 104 447 320 354 275
## -0.802709122 -3.305050943 0.093391012 5.989158592 -2.851763168
## 261 2 336 497 394
## -2.597612328 -3.713695585 1.263629192 6.975450133 -7.050402445
## 448 327 419 112 36
## -6.187067749 -0.138726762 6.104096769 -4.466775330 -4.616862136
## 87 399 111 461 351
## 1.284500325 0.300778763 1.758188903 -3.221342364 3.619381597
## 31 73 424 333 122
## 2.428740626 -0.527694477 1.845675138 -2.221711574 -2.189798218
## 92 234 338 472 300
## -5.666921491 8.775318305 -0.162437018 -4.000684029 -1.784296138
## 323 314 443 387 376
## -1.977525789 -4.395997397 -1.201281672 6.307724361 -11.738018736
## 208 499 206 297 219
## 5.537880093 0.246994887 1.102222072 -0.460269305 -2.987158004
## 82 169 413 228 423
## -3.021002286 -3.205200388 19.853308439 -2.192024195 2.797529204
## 258 385 482 498 192
## 3.414178564 8.271338091 -4.451362820 -0.266787057 0.792207439
## 386 414 50 260 97
## 0.442623562 6.487624676 3.388454435 -6.629745005 -2.978426708
## 183 100 274 484 296
## 2.427214778 -0.031584384 -0.570256827 1.238172323 0.095873701
## 357 227 190 329 156
## -1.933201226 -2.531726289 0.382566988 -0.545690834 -3.581922258
## 131 8 468 70 213
## -1.346444916 7.621427858 2.329583294 1.528233254 1.223808670
## 382 182
## -7.999559935 8.543358664
##
## $sd_residuals
## [1] 5.091575
##
## $scaled_residuals
## [1] 0.1125522
##
## attr(,"class")
## [1] "conformalize"
head(predictions_boston4)## fit lwr upr
## 1 30.39434 23.343942 45.84706
## 15 19.10867 11.055854 34.56138
## 17 19.48015 11.427336 33.98435
## 19 14.18587 6.133060 25.25477
## 28 13.85324 5.800429 21.80538
## 37 21.66451 13.611700 36.16871Fit Conformal Model 2
# Define fit and predict functions
fit_func <- function(formula, data, ...) ranger::ranger(formula, data = data)
predict_func <- function(fit, newdata, ...) predict(fit, newdata)$predictions
# Apply conformalize using the training data
conformal_model_boston_rf <- misc::conformalize(
formula = medv ~ .,
data = train_data,
fit_func = fit_func,
predict_func = predict_func,
seed = 123
)##
## [1] "cal_pred's value:"
## [1] 10.419252 20.448372 27.224677 10.898150 20.123587 17.793151 31.526340
## [8] 22.391453 35.789190 11.665246 25.236577 21.457890 20.277560 19.945675
## [15] 20.844042 38.007683 19.701472 24.895207 27.314342 21.082500 14.371342
## [22] 24.222233 16.660823 21.905261 28.799110 15.444499 24.407960 21.550467
## [29] 22.330151 24.106895 17.826588 18.560532 22.959393 34.590640 14.865118
## [36] 20.339177 19.819318 24.210000 33.370880 21.636240 25.011786 34.424087
## [43] 19.416483 13.969285 29.060077 30.809290 16.560430 30.772120 25.188883
## [50] 13.930568 18.371245 32.221203 12.333186 22.143470 23.763023 23.666013
## [57] 17.002369 25.055677 22.993030 39.054477 23.336043 15.914310 18.324967
## [64] 25.350043 21.407187 36.351613 21.562164 10.454903 27.878533 18.113082
## [71] 30.653660 16.295641 23.167847 24.010590 25.177501 25.061933 18.591062
## [78] 23.960330 12.537551 20.268659 23.398423 13.123332 28.750127 15.631447
## [85] 43.982997 15.656024 25.782773 19.002122 20.605008 22.407643 27.984863
## [92] 10.373169 11.670001 11.828249 28.181570 22.810380 31.332937 12.283840
## [99] 25.592183 34.371947 15.904249 15.087963 18.328225 21.934827 13.236408
## [106] 23.679770 24.285343 45.620510 24.629900 19.165807 18.981140 25.674893
## [113] 19.713469 26.392217 17.045184 18.102572 19.649490 30.833157 10.368585
## [120] 24.932007 22.689930 44.005293 20.775961 16.112724 12.266196 20.558970
## [127] 27.911280 22.765835 28.884250 26.359113 19.584089 14.891015 21.644770
## [134] 31.241963 31.702887 35.185103 23.137040 21.628967 17.497420 14.282768
## [141] 14.577078 24.385645 10.148723 22.377710 21.797343 21.362853 9.545621
## [148] 20.645150 16.658624 25.468693 14.173340 23.654933 13.719573 22.469270
## [155] 20.238633 24.431423 43.391260 20.392460 21.033908 30.354687 22.265293
## [162] 22.718777 15.856220 10.036760 26.145921 19.103820 20.652243 21.881643
## [169] 24.579362 19.982683 25.319243 21.965685 13.186102 34.058633 19.401634
## [176] 42.328435 11.475717 24.782687 20.371097 27.915230 10.845967 13.301828
## [183] 19.002512 32.118284 24.504970 35.727872 33.880980 35.471110 21.034690
## [190] 25.421736 16.085362 44.463493 33.370627 22.705393 18.015633 18.985264
## [197] 17.514547 15.174432 20.548163 20.660860 11.102269 26.035857
##
##
## [1] "cal_y's value:"
## [1] 7.0 19.5 29.9 8.3 19.2 20.6 28.7 20.9 33.3 8.1 23.1 18.2 22.9 16.8 20.6
## [16] 42.8 21.7 25.3 50.0 17.1 11.7 22.6 15.2 15.0 22.8 15.6 24.8 21.7 21.4 22.2
## [31] 20.0 20.1 23.9 34.9 13.6 16.2 19.9 25.0 32.7 24.7 23.3 33.4 21.7 17.2 28.2
## [46] 23.6 15.6 28.5 22.0 13.1 19.4 41.3 8.5 19.4 20.7 23.9 18.1 24.6 24.0 38.7
## [61] 28.1 15.7 19.3 24.4 20.3 31.7 27.5 7.2 26.6 11.9 35.1 19.4 26.5 21.9 24.6
## [76] 25.2 20.1 23.2 12.8 18.9 19.8 12.0 32.5 14.9 50.0 14.1 24.1 18.2 22.4 23.2
## [91] 26.6 8.5 10.2 12.7 31.1 24.5 34.9 13.9 26.2 37.3 14.3 13.2 19.6 23.4 9.5
## [106] 22.3 26.4 50.0 22.2 23.7 19.8 24.7 20.1 29.1 18.0 19.5 21.4 50.0 5.0 20.1
## [121] 20.6 44.8 21.2 15.2 6.3 18.7 29.4 23.0 37.0 25.0 19.3 14.9 21.0 30.1 32.4
## [136] 33.8 21.6 21.1 19.7 13.8 12.6 23.0 8.8 22.8 18.9 22.5 5.0 21.7 16.4 22.9
## [151] 12.7 22.8 13.4 19.4 20.3 22.0 48.3 18.5 19.6 29.0 20.4 21.6 18.4 10.5 15.0
## [166] 22.5 21.2 22.6 27.1 21.5 23.9 23.8 17.9 31.6 20.8 50.0 8.8 23.7 18.3 30.5
## [181] 7.2 16.3 19.4 30.1 21.4 37.9 33.2 35.2 21.8 28.6 17.8 37.6 34.9 19.3 15.6
## [196] 19.2 27.1 19.1 20.9 22.4 10.9 36.2
# Predict with split conformal method on the test data
predictions_boston_rf <- predict(
conformal_model_boston_rf,
newdata = test_data,
predict_func = predict_func,
level = 0.95,
method = "kde"
)##
## [1] "object's value:"
## $fit
## Ranger result
##
## Call:
## ranger::ranger(formula, data = data)
##
## Type: Regression
## Number of trees: 500
## Sample size: 202
## Number of independent variables: 13
## Mtry: 3
## Target node size: 5
## Variable importance mode: none
## Splitrule: variance
## OOB prediction error (MSE): 13.55259
## R squared (OOB): 0.8533691
##
## $residuals
## [1] -3.419252485 -0.948371904 2.675323333 -2.598149629 -0.923586667
## [6] 2.806848571 -2.826340000 -1.491453333 -2.489190000 -3.565245897
## [11] -2.136576667 -3.257890000 2.622440000 -3.145675238 -0.244042279
## [16] 4.792316667 1.998528333 0.404793333 22.685658431 -3.982500000
## [21] -2.671341579 -1.622233333 -1.460822619 -6.905260537 -5.999110000
## [26] 0.155501429 0.392040000 0.149533333 -0.930151111 -1.906894762
## [31] 2.173411667 1.539468254 0.940606667 0.309360000 -1.265117857
## [36] -4.139176667 0.080682063 0.790000000 -0.670880000 3.063760000
## [41] -1.711786190 -1.024086667 2.283516667 3.230715208 -0.860076667
## [46] -7.209290000 -0.960430079 -2.272120000 -3.188883333 -0.830568072
## [51] 1.028754762 9.078797302 -3.833185873 -2.743470000 -3.063023333
## [56] 0.233986667 1.097631190 -0.455676667 1.006970000 -0.354476667
## [61] 4.763956667 -0.214310000 0.975033333 -0.950043333 -1.107186667
## [66] -4.651613333 5.937836459 -3.254903344 -1.278533333 -6.213082086
## [71] 4.446340000 3.104358571 3.332153333 -2.110590000 -0.577501429
## [76] 0.138066667 1.508937921 -0.760330000 0.262449105 -1.368658889
## [81] -3.598423333 -1.123331889 3.749873333 -0.731447167 6.017003333
## [86] -1.556023540 -1.682773333 -0.802122381 1.794991905 0.792356667
## [91] -1.384863333 -1.873168579 -1.470001346 0.871751192 2.918430000
## [96] 1.689620000 3.567063333 1.616160292 0.607816667 2.928053333
## [101] -1.604249246 -1.887962857 1.271774740 1.465173333 -3.736407921
## [106] -1.379770000 2.114656667 4.379490000 -2.429900000 4.534193333
## [111] 0.818860000 -0.974893333 0.386531429 2.707783333 0.954815952
## [116] 1.397428095 1.750509706 19.166842727 -5.368585249 -4.832006667
## [121] -2.089930000 0.794706667 0.424038571 -0.912724448 -5.966195684
## [126] -1.858970000 1.488720000 0.234165000 8.115750000 -1.359113333
## [131] -0.284088571 0.008984731 -0.644770000 -1.141963333 0.697113333
## [136] -1.385102684 -1.537040000 -0.528966667 2.202580000 -0.482767638
## [141] -1.977078277 -1.385644762 -1.348723163 0.422290000 -2.897343333
## [146] 1.137146667 -4.545621403 1.054850000 -0.258624034 -2.568693333
## [151] -1.473339524 -0.854933333 -0.319572550 -3.069270000 0.061366667
## [156] -2.431423333 4.908740000 -1.892460000 -1.433907712 -1.354686667
## [161] -1.865293333 -1.118776667 2.543779556 0.463239708 -11.145920805
## [166] 3.396180000 0.547756667 0.718356667 2.520637619 1.517316667
## [171] -1.419243333 1.834314921 4.713897526 -2.458633333 1.398365831
## [176] 7.671565411 -2.675716524 -1.082686528 -2.071096667 2.584770000
## [181] -3.645966658 2.998171877 0.397487619 -2.018283636 -3.104970000
## [186] 2.172127619 -0.680980000 -0.271110000 0.765309626 3.178264286
## [191] 1.714637993 -6.863493333 1.529373333 -3.405393333 -2.415632886
## [196] 0.214736190 9.585452619 3.925567955 0.351836667 1.739140000
## [201] -0.202268616 10.164143333
##
## $sd_residuals
## [1] 3.624106
##
## $scaled_residuals
## [1] 0.01260864
##
## attr(,"class")
## [1] "conformalize"
head(predictions_boston_rf)## fit lwr upr
## [1,] 27.03134 21.991838 32.43038
## [2,] 19.20299 13.542260 25.05314
## [3,] 21.34472 17.000993 30.77696
## [4,] 18.77455 12.341589 25.88818
## [5,] 15.60764 9.157478 21.48264
## [6,] 21.31355 14.591954 29.75374
Calculate Out-of-Sample Coverage Rate 2
The coverage rate is the proportion of true values in the test set that fall within the prediction intervals.
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston[, "lwr"] & true_y_boston <= predictions_boston[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9411765
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston2[, "lwr"] & true_y_boston <= predictions_boston2[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9607843
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston3[, "lwr"] & true_y_boston <= predictions_boston3[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9705882
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston4[, "lwr"] & true_y_boston <= predictions_boston4[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9607843
# True values for the test set
true_y_boston <- test_data$medv
# Check if true values fall within the prediction intervals
coverage_boston <- mean(true_y_boston >= predictions_boston_rf[, "lwr"] & true_y_boston <= predictions_boston_rf[, "upr"])
cat("Out-of-sample coverage rate for Boston dataset:", coverage_boston)## Out-of-sample coverage rate for Boston dataset: 0.9215686