library(crossvalidation)## Loading required package: doSNOW## Loading required package: foreach## Loading required package: iterators## Loading required package: snow## Loading required package: forecast## Registered S3 method overwritten by 'quantmod':
## method from
## as.zoo.data.frame zoo
data("AirPassengers")
# Example 1 -----
eval_metric <- function(predicted, observed)
{
error <- observed - predicted$mean
me <- mean(error)
rmse <- sqrt(mean(error^2))
mae <- mean(abs(error))
# ----- 80% interval -----
lower80 <- predicted$lower[, 1]
upper80 <- predicted$upper[, 1]
coverage80 <- mean(
observed >= lower80 & observed <= upper80
)
alpha80 <- 0.20
winkler80 <- ifelse(
observed < lower80,
(upper80 - lower80) + (2 / alpha80) * (lower80 - observed),
ifelse(
observed > upper80,
(upper80 - lower80) + (2 / alpha80) * (observed - upper80),
(upper80 - lower80)
)
)
# ----- 95% interval -----
lower95 <- predicted$lower[, 2]
upper95 <- predicted$upper[, 2]
coverage95 <- mean(
observed >= lower95 & observed <= upper95
)
alpha95 <- 0.05
winkler95 <- ifelse(
observed < lower95,
(upper95 - lower95) + (2 / alpha95) * (lower95 - observed),
ifelse(
observed > upper95,
(upper95 - lower95) + (2 / alpha95) * (observed - upper95),
(upper95 - lower95)
)
)
c(
ME = me,
RMSE = rmse,
MAE = mae,
Coverage80 = coverage80,
Winkler80 = mean(winkler80),
Coverage95 = coverage95,
Winkler95 = mean(winkler95)
)
}
(res <- crossval_ts(y=AirPassengers, initial_window = 10,
horizon = 3, fcast_func = forecast::thetaf, eval_metric = eval_metric))## | | | 0% | |= | 1% | |= | 2% | |== | 2% | |== | 3% | |=== | 4% | |=== | 5% | |==== | 5% | |==== | 6% | |===== | 7% | |===== | 8% | |====== | 8% | |====== | 9% | |======= | 10% | |======= | 11% | |======== | 11% | |======== | 12% | |========= | 13% | |========== | 14% | |=========== | 15% | |=========== | 16% | |============ | 17% | |============= | 18% | |============= | 19% | |============== | 20% | |=============== | 21% | |=============== | 22% | |================ | 23% | |================= | 24% | |================== | 25% | |================== | 26% | |=================== | 27% | |==================== | 28% | |==================== | 29% | |===================== | 30% | |====================== | 31% | |====================== | 32% | |======================= | 33% | |======================== | 34% | |======================== | 35% | |========================= | 36% | |========================== | 37% | |=========================== | 38% | |=========================== | 39% | |============================ | 39% | |============================ | 40% | |============================= | 41% | |============================= | 42% | |============================== | 42% | |============================== | 43% | |=============================== | 44% | |=============================== | 45% | |================================ | 45% | |================================ | 46% | |================================= | 47% | |================================= | 48% | |================================== | 48% | |================================== | 49% | |=================================== | 50% | |==================================== | 51% | |==================================== | 52% | |===================================== | 52% | |===================================== | 53% | |====================================== | 54% | |====================================== | 55% | |======================================= | 55% | |======================================= | 56% | |======================================== | 57% | |======================================== | 58% | |========================================= | 58% | |========================================= | 59% | |========================================== | 60% | |========================================== | 61% | |=========================================== | 61% | |=========================================== | 62% | |============================================ | 63% | |============================================= | 64% | |============================================== | 65% | |============================================== | 66% | |=============================================== | 67% | |================================================ | 68% | |================================================ | 69% | |================================================= | 70% | |================================================== | 71% | |================================================== | 72% | |=================================================== | 73% | |==================================================== | 74% | |==================================================== | 75% | |===================================================== | 76% | |====================================================== | 77% | |======================================================= | 78% | |======================================================= | 79% | |======================================================== | 80% | |========================================================= | 81% | |========================================================= | 82% | |========================================================== | 83% | |=========================================================== | 84% | |=========================================================== | 85% | |============================================================ | 86% | |============================================================= | 87% | |============================================================== | 88% | |============================================================== | 89% | |=============================================================== | 89% | |=============================================================== | 90% | |================================================================ | 91% | |================================================================ | 92% | |================================================================= | 92% | |================================================================= | 93% | |================================================================== | 94% | |================================================================== | 95% | |=================================================================== | 95% | |=================================================================== | 96% | |==================================================================== | 97% | |==================================================================== | 98% | |===================================================================== | 98% | |===================================================================== | 99% | |======================================================================| 100%## ME RMSE MAE Coverage80 Winkler80 Coverage95
## result.1 -28.794660 29.300287 28.794660 0.0000000 153.10992 0.3333333
## result.2 16.198526 16.894302 16.198526 1.0000000 45.01795 1.0000000
## result.3 11.201494 15.993359 12.578276 1.0000000 45.05996 1.0000000
## result.4 21.430125 22.483895 21.430125 0.6666667 63.01207 1.0000000
## result.5 10.055765 11.527746 10.055765 1.0000000 45.99967 1.0000000
## result.6 -2.640822 10.676714 9.999466 1.0000000 46.56907 1.0000000
## result.7 14.296434 23.709132 20.531135 0.6666667 75.04186 1.0000000
## result.8 38.247497 39.529998 38.247497 0.0000000 198.74990 0.3333333
## result.9 23.043159 23.947630 23.043159 0.3333333 93.83463 1.0000000
## result.10 -21.689067 27.907560 21.689067 0.6666667 90.23377 1.0000000
## result.11 -41.782157 46.664199 41.782157 0.3333333 222.06310 0.3333333
## result.12 -34.934831 36.512081 34.934831 0.3333333 162.38092 0.6666667
## result.13 -4.002700 12.728771 9.999100 1.0000000 59.64475 1.0000000
## result.14 30.349582 30.588761 30.349582 0.6666667 72.14355 1.0000000
## result.15 21.192349 25.806712 21.192349 0.6666667 71.39094 1.0000000
## result.16 23.193143 25.914875 23.193143 0.6666667 91.70660 1.0000000
## result.17 30.081542 30.679960 30.081542 0.3333333 111.58689 1.0000000
## result.18 -6.530509 9.111376 6.999059 1.0000000 69.51704 1.0000000
## result.19 19.907586 23.010762 19.907586 1.0000000 67.03506 1.0000000
## result.20 17.631089 19.829355 17.631089 1.0000000 67.97573 1.0000000
## result.21 11.738022 14.718185 12.229846 1.0000000 61.61617 1.0000000
## result.22 -21.787490 28.489509 21.787490 0.6666667 93.30920 1.0000000
## result.23 -43.557571 47.527244 43.557571 0.3333333 206.77368 0.6666667
## result.24 -34.473558 35.514155 34.473558 0.3333333 146.63288 0.6666667
## result.25 -4.699360 10.498595 7.201224 1.0000000 60.07550 1.0000000
## result.26 25.974138 26.581272 25.974138 1.0000000 63.01942 1.0000000
## result.27 16.905109 19.474600 16.905109 1.0000000 58.04472 1.0000000
## result.28 15.218760 16.352917 15.218760 1.0000000 55.27721 1.0000000
## result.29 7.625241 8.933828 7.625241 1.0000000 55.27718 1.0000000
## result.30 2.261970 17.595326 15.666212 1.0000000 57.13292 1.0000000
## result.31 30.404914 36.592516 30.404914 0.3333333 147.79689 1.0000000
## result.32 46.412292 47.386236 46.412292 0.0000000 252.29076 0.0000000
## result.33 4.651769 15.532402 15.001095 1.0000000 66.12818 1.0000000
## result.34 -23.538570 33.729923 29.025988 0.6666667 100.23631 1.0000000
## result.35 -60.690177 63.572981 60.690177 0.0000000 364.77263 0.0000000
## result.36 -30.676401 32.211315 30.676401 0.6666667 99.22128 1.0000000
## result.37 -8.577804 12.511784 8.577804 1.0000000 72.20676 1.0000000
## result.38 21.858632 21.864774 21.858632 1.0000000 75.18682 1.0000000
## result.39 15.511300 24.464353 15.511300 1.0000000 78.40492 1.0000000
## result.40 27.351766 33.292899 27.351766 0.6666667 82.41942 1.0000000
## result.41 40.030438 40.086293 40.030438 0.3333333 129.74165 1.0000000
## result.42 1.913204 6.585371 5.082179 1.0000000 80.67976 1.0000000
## result.43 10.484756 17.849104 14.434313 1.0000000 79.27719 1.0000000
## result.44 28.459896 30.643222 28.459896 1.0000000 78.20235 1.0000000
## result.45 9.494149 19.388080 18.988959 1.0000000 65.59401 1.0000000
## result.46 -31.346158 42.472223 34.784073 0.3333333 147.50136 0.6666667
## result.47 -73.108813 78.130452 73.108813 0.0000000 464.77698 0.0000000
## result.48 -47.391791 49.482768 47.391791 0.0000000 174.72623 0.6666667
## result.49 -20.263420 22.215913 20.263420 0.6666667 93.64951 1.0000000
## result.50 18.627251 19.778162 18.627251 1.0000000 90.31375 1.0000000
## result.51 12.850828 24.491415 18.283609 1.0000000 79.61489 1.0000000
## result.52 18.630902 29.191109 27.309290 1.0000000 79.69377 1.0000000
## result.53 51.629084 51.820741 51.629084 0.0000000 188.97033 0.6666667
## result.54 13.235345 22.822393 16.377438 1.0000000 99.26308 1.0000000
## result.55 44.587147 53.679665 44.587147 0.6666667 174.96335 1.0000000
## result.56 54.072225 56.591957 54.072225 0.3333333 172.04505 1.0000000
## result.57 23.241489 30.809955 26.322921 0.6666667 114.32397 1.0000000
## result.58 -48.442256 57.148030 48.442256 0.3333333 188.53314 1.0000000
## result.59 -76.172134 81.227076 76.172134 0.0000000 397.07229 0.3333333
## result.60 -49.727782 51.411167 49.727782 0.6666667 154.24091 1.0000000
## result.61 -12.015506 17.811915 13.001128 1.0000000 105.22437 1.0000000
## result.62 28.827561 29.294075 28.827561 1.0000000 108.31205 1.0000000
## result.63 19.978440 24.903081 19.978440 1.0000000 96.57555 1.0000000
## result.64 17.110530 24.500048 22.184318 1.0000000 97.42509 1.0000000
## result.65 40.684200 40.816355 40.684200 0.6666667 101.57637 1.0000000
## result.66 22.682439 32.492379 22.682439 1.0000000 99.71364 1.0000000
## result.67 50.630668 64.359435 50.630668 0.3333333 252.53872 0.6666667
## result.68 71.054598 73.834184 71.054598 0.0000000 362.78107 0.6666667
## result.69 18.573839 30.848787 28.001381 0.6666667 131.50974 1.0000000
## result.70 -67.050346 75.890258 67.050346 0.3333333 293.72913 0.6666667
## result.71 -89.147978 96.654776 89.147978 0.0000000 478.31378 0.3333333
## result.72 -62.834495 65.591765 62.834495 0.6666667 206.58602 1.0000000
## result.73 -17.192156 24.448634 17.192156 1.0000000 116.32088 1.0000000
## result.74 39.147639 39.310613 39.147639 1.0000000 124.70472 1.0000000
## result.75 15.464746 23.499340 15.599360 1.0000000 127.96355 1.0000000
## result.76 21.388632 28.622347 25.036615 1.0000000 128.15803 1.0000000
## result.77 44.769285 44.904644 44.769285 1.0000000 128.45131 1.0000000
## result.78 24.337654 38.632254 24.890329 1.0000000 125.82671 1.0000000
## result.79 60.018016 72.625037 60.018016 0.3333333 268.46578 0.6666667
## result.80 78.806534 80.547853 78.806534 0.0000000 355.81507 0.6666667
## result.81 11.811108 31.117683 30.676430 1.0000000 117.97375 1.0000000
## result.82 -73.039284 86.689647 73.039284 0.3333333 338.25187 0.6666667
## result.83 -112.674445 120.216380 112.674445 0.0000000 675.09729 0.3333333
## result.84 -75.193608 77.210395 75.193608 0.3333333 289.55078 0.6666667
## result.85 -18.005367 23.707491 18.005367 1.0000000 147.40848 1.0000000
## result.86 34.032748 34.590333 34.032748 1.0000000 156.59367 1.0000000
## result.87 21.143051 32.190728 22.381017 1.0000000 139.88798 1.0000000
## result.88 25.770886 36.725269 33.180194 1.0000000 138.21209 1.0000000
## result.89 61.405131 61.602493 61.405131 0.6666667 170.34194 1.0000000
## result.90 28.569615 46.690860 30.711084 1.0000000 138.54837 1.0000000
## result.91 72.884393 87.299176 72.884393 0.3333333 330.70092 0.6666667
## result.92 97.243170 99.503251 97.243170 0.0000000 452.63679 0.6666667
## result.93 17.748567 36.930435 36.435381 1.0000000 138.09695 1.0000000
## result.94 -76.493543 94.890278 76.493543 0.3333333 355.90060 0.6666667
## result.95 -136.613003 145.258940 136.613003 0.0000000 841.14448 0.0000000
## result.96 -92.679937 95.004482 92.679937 0.0000000 339.88416 0.6666667
## result.97 -32.018423 33.966942 32.018423 1.0000000 162.78660 1.0000000
## result.98 22.498638 24.963399 22.498638 1.0000000 170.17564 1.0000000
## result.99 7.476001 20.150283 17.158667 1.0000000 160.38132 1.0000000
## result.100 9.982584 22.773032 22.210356 1.0000000 160.15109 1.0000000
## result.101 52.362071 53.101568 52.362071 1.0000000 162.26240 1.0000000
## result.102 34.853629 55.893897 39.234038 0.6666667 154.93804 1.0000000
## result.103 95.047754 111.264404 95.047754 0.3333333 485.70958 0.6666667
## result.104 121.335201 125.765542 121.335201 0.0000000 646.57500 0.3333333
## result.105 27.661546 53.669523 52.335667 0.6666667 149.46688 1.0000000
## result.106 -82.928463 106.536749 87.398376 0.3333333 439.04755 0.6666667
## result.107 -168.429957 174.864018 168.429957 0.0000000 1125.85339 0.0000000
## result.108 -86.047368 89.349691 86.047368 0.6666667 241.50861 1.0000000
## result.109 -35.392983 38.646195 35.392983 1.0000000 192.33139 1.0000000
## result.110 32.273683 33.691668 32.273683 1.0000000 199.99776 1.0000000
## result.111 35.911969 45.528574 35.911969 1.0000000 195.20693 1.0000000
## result.112 28.584481 41.791438 38.166544 1.0000000 196.54087 1.0000000
## result.113 78.144295 79.313101 78.144295 1.0000000 196.93429 1.0000000
## result.114 37.152546 52.614035 39.210445 1.0000000 192.54870 1.0000000
## result.115 95.078342 110.886020 95.078342 0.6666667 366.36760 1.0000000
## result.116 109.166178 116.176118 109.166178 0.3333333 406.73972 1.0000000
## result.117 41.289554 62.020845 57.334902 0.3333333 215.15767 1.0000000
## result.118 -92.399494 116.617773 92.824074 0.3333333 466.72849 0.6666667
## result.119 -175.618445 183.279552 175.618445 0.0000000 1143.54792 0.0000000
## result.120 -94.580461 97.360392 94.580461 0.6666667 277.78465 1.0000000
## result.121 -27.751828 32.935586 27.751828 1.0000000 202.13742 1.0000000
## result.122 36.177008 38.166461 36.177008 1.0000000 208.63520 1.0000000
## result.123 5.992278 14.161847 13.997426 1.0000000 200.00976 1.0000000
## result.124 12.637863 33.652691 27.980298 1.0000000 200.18280 1.0000000
## result.125 71.834372 76.950725 71.834372 1.0000000 200.57526 1.0000000
## result.126 85.518711 93.750937 85.518711 0.6666667 252.56383 1.0000000
## result.127 94.429064 115.523968 94.429064 0.6666667 407.36364 0.6666667
## result.128 173.325805 177.666522 173.325805 0.0000000 1129.61408 0.0000000
## result.129 33.890665 63.841907 61.668605 0.6666667 242.69010 1.0000000
## result.130 -119.059067 137.736851 119.059067 0.3333333 619.41659 0.3333333
## result.131 -180.821172 190.452409 180.821172 0.0000000 1152.49495 0.0000000
## result.132 -103.156396 108.618814 103.156396 0.6666667 330.03996 1.0000000
## Winkler95
## result.1 207.58384
## result.2 68.84902
## result.3 68.91326
## result.4 68.84778
## result.5 70.35043
## result.6 71.22125
## result.7 67.58381
## result.8 212.44029
## result.9 64.19366
## result.10 84.12361
## result.11 345.16553
## result.12 212.58117
## result.13 91.21878
## result.14 99.76932
## result.15 101.02401
## result.16 76.57925
## result.17 75.78459
## result.18 106.31714
## result.19 102.52128
## result.20 103.95991
## result.21 94.23380
## result.22 88.70090
## result.23 216.50078
## result.24 173.17046
## result.25 91.87755
## result.26 96.37989
## result.27 88.77173
## result.28 84.53920
## result.29 84.53916
## result.30 87.37725
## result.31 80.90179
## result.32 317.63733
## result.33 101.13432
## result.34 94.76752
## result.35 668.66045
## result.36 105.34530
## result.37 110.43071
## result.38 114.98831
## result.39 119.90998
## result.40 117.74622
## result.41 117.68367
## result.42 123.38905
## result.43 121.24401
## result.44 119.60018
## result.45 100.31739
## result.46 131.46872
## result.47 989.73570
## result.48 197.04903
## result.49 124.70705
## result.50 138.12296
## result.51 121.76047
## result.52 121.88110
## result.53 198.97225
## result.54 151.80978
## result.55 143.53838
## result.56 145.84502
## result.57 125.91572
## result.58 145.32572
## result.59 533.47427
## result.60 150.28420
## result.61 160.92678
## result.62 165.64899
## result.63 147.69956
## result.64 148.99882
## result.65 149.39144
## result.66 152.49885
## result.67 346.57310
## result.68 466.65911
## result.69 144.45417
## result.70 286.63335
## result.71 749.94137
## result.72 194.10701
## result.73 177.89744
## result.74 190.71941
## result.75 195.70336
## result.76 196.00079
## result.77 196.44933
## result.78 192.43535
## result.79 190.18498
## result.80 297.52296
## result.81 180.42528
## result.82 365.98675
## result.83 1270.03337
## result.84 258.96612
## result.85 225.44181
## result.86 239.48935
## result.87 213.94020
## result.88 211.37715
## result.89 212.85518
## result.90 211.89144
## result.91 235.02356
## result.92 321.81781
## result.93 211.20106
## result.94 367.10470
## result.95 1650.76356
## result.96 253.79792
## result.97 248.96061
## result.98 260.26118
## result.99 245.28206
## result.100 244.92996
## result.101 248.15893
## result.102 228.50390
## result.103 594.35490
## result.104 772.48177
## result.105 226.74993
## result.106 391.00341
## result.107 2680.36706
## result.108 281.33247
## result.109 294.14548
## result.110 305.87018
## result.111 298.54324
## result.112 300.58332
## result.113 301.18501
## result.114 294.47782
## result.115 274.91510
## result.116 277.44051
## result.117 222.41274
## result.118 445.35713
## result.119 2574.24087
## result.120 293.25899
## result.121 309.14251
## result.122 319.08001
## result.123 305.88853
## result.124 306.15317
## result.125 306.75339
## result.126 295.04962
## result.127 417.25657
## result.128 2547.86175
## result.129 230.38847
## result.130 668.97862
## result.131 2469.39362
## result.132 302.16748## ME RMSE MAE Coverage80 Winkler80 Coverage95
## 2.6570822 51.4271704 46.5118747 0.6590909 218.4527816 0.8459596
## Winkler95
## 312.1383104
eval_metric <- function(predicted, observed)
{
error <- observed - predicted$mean
me <- mean(error)
rmse <- sqrt(mean(error^2))
mae <- mean(abs(error))
# Only one interval returned
lower <- predicted$lower
upper <- predicted$upper
coverage <- mean(
observed >= lower & observed <= upper
)
alpha <- 0.05
winkler <- ifelse(
observed < lower,
(upper - lower) + (2 / alpha) * (lower - observed),
ifelse(
observed > upper,
(upper - lower) + (2 / alpha) * (observed - upper),
(upper - lower)
)
)
c(
ME = me,
RMSE = rmse,
MAE = mae,
Coverage95 = coverage,
Winkler95 = mean(winkler)
)
}
fcast_func <- function(y, h, ...)
{
forecast::thetaf(
y,
h = h,
level = 95
)
}
res <- crossval_ts(
y = AirPassengers,
initial_window = 10,
horizon = 3,
fcast_func = fcast_func,
eval_metric = eval_metric
)## | | | 0% | |= | 1% | |= | 2% | |== | 2% | |== | 3% | |=== | 4% | |=== | 5% | |==== | 5% | |==== | 6% | |===== | 7% | |===== | 8% | |====== | 8% | |====== | 9% | |======= | 10% | |======= | 11% | |======== | 11% | |======== | 12% | |========= | 13% | |========== | 14% | |=========== | 15% | |=========== | 16% | |============ | 17% | |============= | 18% | |============= | 19% | |============== | 20% | |=============== | 21% | |=============== | 22% | |================ | 23% | |================= | 24% | |================== | 25% | |================== | 26% | |=================== | 27% | |==================== | 28% | |==================== | 29% | |===================== | 30% | |====================== | 31% | |====================== | 32% | |======================= | 33% | |======================== | 34% | |======================== | 35% | |========================= | 36% | |========================== | 37% | |=========================== | 38% | |=========================== | 39% | |============================ | 39% | |============================ | 40% | |============================= | 41% | |============================= | 42% | |============================== | 42% | |============================== | 43% | |=============================== | 44% | |=============================== | 45% | |================================ | 45% | |================================ | 46% | |================================= | 47% | |================================= | 48% | |================================== | 48% | |================================== | 49% | |=================================== | 50% | |==================================== | 51% | |==================================== | 52% | |===================================== | 52% | |===================================== | 53% | |====================================== | 54% | |====================================== | 55% | |======================================= | 55% | |======================================= | 56% | |======================================== | 57% | |======================================== | 58% | |========================================= | 58% | |========================================= | 59% | |========================================== | 60% | |========================================== | 61% | |=========================================== | 61% | |=========================================== | 62% | |============================================ | 63% | |============================================= | 64% | |============================================== | 65% | |============================================== | 66% | |=============================================== | 67% | |================================================ | 68% | |================================================ | 69% | |================================================= | 70% | |================================================== | 71% | |================================================== | 72% | |=================================================== | 73% | |==================================================== | 74% | |==================================================== | 75% | |===================================================== | 76% | |====================================================== | 77% | |======================================================= | 78% | |======================================================= | 79% | |======================================================== | 80% | |========================================================= | 81% | |========================================================= | 82% | |========================================================== | 83% | |=========================================================== | 84% | |=========================================================== | 85% | |============================================================ | 86% | |============================================================= | 87% | |============================================================== | 88% | |============================================================== | 89% | |=============================================================== | 89% | |=============================================================== | 90% | |================================================================ | 91% | |================================================================ | 92% | |================================================================= | 92% | |================================================================= | 93% | |================================================================== | 94% | |================================================================== | 95% | |=================================================================== | 95% | |=================================================================== | 96% | |==================================================================== | 97% | |==================================================================== | 98% | |===================================================================== | 98% | |===================================================================== | 99% | |======================================================================| 100%## ME RMSE MAE Coverage95 Winkler95
## 2.6570822 51.4271704 46.5118747 0.8459596 312.1383104