Chapter 8. Backtesting and Stress Testing (in R/Julia)


Copyright 2011, 2016, 2018 Jon Danielsson. This code is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This code is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. The GNU General Public License is available at: https://www.gnu.org/licenses/.
The original 2011 R code will not fully work on a recent R because there have been some changes to libraries. The latest version of the Matlab code only uses functions from Matlab toolboxes.
The GARCH functionality in the econometric toolbox in Matlab is trying to be too clever, but can't deliver and could well be buggy. If you want to try that, here are the docs (estimate). Besides, it can only do univariate GARCH and so can't be used in Chapter 3. Kevin Sheppard's MFE toolbox is much better, while not as user friendly, it is much better written and is certainly more comprehensive. It can be downloaded here and the documentation here is quite detailed.


Listing 8.1/8.2: Load data in R
Last updated August 2016

library(tseries)
library(zoo)
p = zoo(read.csv('index.csv',header=TRUE,sep=','))
y=diff(log(p))                                     # get returns
y=coredata(y)                                      # strip date information
		
Listing 8.1/8.2: Load data in Julia
Last updated June 2018

using CSV;
price = CSV.read("index.csv", nullable = false);
y = diff(log.(price[:,1]));                      # get returns
		

Listing 8.3/8.4: Set backtest up in R
Last updated August 2016

TT = length(y)                                    # number of obs for y
WE = 1000                                         # estimation window length
p = 0.01                                          # probability
l1 = WE*p                                         # HS observation
value = 1;                                        # portfolio value
VaR = matrix(nrow=TT,ncol=4)                      # matrix for forecasts
## EWMA setup
lambda = 0.94;
s11 = var(y[1:30]);
for(t in 2:WE) s11=lambda*s11+(1-lambda)*y[t-1]^2
library(fGarch)
		
Listing 8.3/8.4: Set backtest up in Julia
Last updated June 2018

T = length(y)                          # number of obs for return y
WE = 1000                              # estimation window length
p = 0.01                               # probability
l1 = convert(Int, WE*p)                # HS observation
value = 1                              # portfolio value
VaR = fill!(Array{Float64}(T,4), NaN)  # matrix for forecasts
## EWMA setup
lambda = 0.94
s11 = var(y[1:30])
for t in range(2,WE-1)
    s11=lambda*s11+(1-lambda)*y[t-1]^2
end
		

Listing 8.5/8.6: Running backtest in R
Last updated August 2016

for (t in (WE+1):TT){
  t1 = t-WE;                                     # start of the data window
  t2 = t-1;                                      # end of the data window
  window = y[t1:t2]                              # data for estimation
  s11=lambda*s11+(1-lambda)*y[t-1]^2
  VaR[t,1] = -qnorm(p) * sqrt(s11) * value       # EWMA
  VaR[t,2] = - sd(window) * qnorm(p)*value       # MA
  ys = sort(window)
  VaR[t,3] = -ys[l1]*value                       # HS
  g=garchFit(formula=~garch(1,1),window,
             trace=FALSE,include.mean=FALSE)
  par=g@fit$matcoef
  s4=par[1]+par[2]*window[WE]^2+par[3]*g@h.t[WE]
  VaR[t,4] = -qnorm(p) * sqrt(s4) * value        # GARCH(1,1)
}
		
Listing 8.5/8.6: Running backtest in Julia
Last updated June 2018

using Distributions;
using FRFGarch;
for t in range(WE+1, T-WE)
    t1 = t - WE                                         # start of data window
    t2 = t - 1                                          # end of data window
    window = y[t1:t2]                                   # data for estimation
                                                        ## EWMA
    s11 = lambda * s11 + (1-lambda) * y[t-1]^2
    VaR[t,1]=-quantile(Normal(0,1),p)*sqrt(s11)*value   # EWMA
    VaR[t,2]=-std(window)*quantile(Normal(0,1),p)*value # MA
    ys = sort(window)
    VaR[t,3] = -ys[l1] * value                          # HS
    res = GARCHfit(window)
    s4 = res.seForecast
    VaR[t,4]=-s4*quantile(Normal(0,1),p)*value          # GARCH(1,1)
end
## GARCH VaR estimation will be slightly different from other languages
## this is due to GARCHfit choosing initial conditional vol = sample vol
		

Listing 8.7/8.8: Backtesting analysis in R
Last updated June 2018

W1=WE+1
for (i in 1:4){
  VR = sum(y[W1:TT]< -VaR[W1:TT,i])/(p*(TT-WE))
  s = sd(VaR[W1:TT,i])
  cat(i,"VR",VR,"VaR vol",s,"\n")
}
matplot(cbind(y[W1:TT],VaR[W1:TT,]),type='l',col=1:5,las=1,ylab="",lty=1:5)
legend("topleft",legend=c("Returns","EWMA","MA","HS","GARCH"),lty=1:5,col=1:5,bty="n")
		
Listing 8.7/8.8: Backtesting analysis in Julia
Last updated June 2018

W1 = WE + 1
for i in range(1,4)
    VR = sum(y[W1:T] .< -VaR[W1:T, i]) / (p * (T - WE))
    s = std(VaR[W1:T,i])
    println([i, "VR", VR, "VaR vol", s])
end
using Plots;
return plot([y, VaR[:,1], VaR[:,2], VaR[:,3], VaR[:,4]])
		

Listing 8.9/8.10: Bernoulli coverage test in R
Last updated August 2016

bern_test=function(p,v){
  lv=length(v)
  sv=sum(v)
  al=log(p)*sv+log(1-p)*(lv-sv)
  bl=log(sv/lv)*sv +log(1-sv/lv)*(lv-sv)
  return(-2*(al-bl))
}
		
Listing 8.9/8.10: Bernoulli coverage test in Julia
Last updated June 2018

function bern_test(p,v)
    lv = length(v)
    sv = sum(v)
    al = log(p)*sv + log(1-p)*(lv-sv)
    bl = log(sv/lv)*sv + log(1-sv/lv)*(lv-sv)
    return (-2*(al-bl))
end
		

Listing 8.11/8.12: Independence test in R
Last updated June 2018

ind_test=function(V){
  J=matrix(ncol=4,nrow=length(V))
  for (i in 2:length(V)){
    J[i,1]=V[i-1]==0 & V[i]==0
    J[i,2]=V[i-1]==0 & V[i]==1
    J[i,3]=V[i-1]==1 & V[i]==0
    J[i,4]=V[i-1]==1 & V[i]==1
  }
  V_00=sum(J[,1],na.rm=TRUE)
  V_01=sum(J[,2],na.rm=TRUE)
  V_10=sum(J[,3],na.rm=TRUE)
  V_11=sum(J[,4],na.rm=TRUE)
  p_00=V_00/(V_00+V_01)
  p_01=V_01/(V_00+V_01)
  p_10=V_10/(V_10+V_11)
  p_11=V_11/(V_10+V_11)
  hat_p=(V_01+V_11)/(V_00+V_01+V_10+V_11)
  al = log(1-hat_p)*(V_00+V_10) + log(hat_p)*(V_01+V_11)
  bl = log(p_00)*V_00 + log(p_01)*V_01 + log(p_10)*V_10 + log(p_11)*V_11
  return(-2*(al-bl))
}
		
Listing 8.11/8.12: Independence test in Julia
Last updated June 2018

function ind_test(V)
    J = fill!(Array{Float64}(T,4), 0)
    for i in range(2,length(V)-1)
        J[i,1] = (V[i-1] == 0) & (V[i] == 0)
        J[i,2] = (V[i-1] == 0) & (V[i] == 1)
        J[i,3] = (V[i-1] == 1) & (V[i] == 0)
        J[i,4] = (V[i-1] == 1) & (V[i] == 1)
    end
    V_00 = sum(J[:,1])
    V_01 = sum(J[:,2])
    V_10 = sum(J[:,3])
    V_11 = sum(J[:,4])
    p_00=V_00/(V_00+V_01)
    p_01=V_01/(V_00+V_01)
    p_10=V_10/(V_10+V_11)
    p_11=V_11/(V_10+V_11)
    hat_p = (V_01+V_11)/(V_00+V_01+V_10+V_11)
    al = log(1-hat_p)*(V_00+V_10) + log(hat_p)*(V_01+V_11)
    bl = log(p_00)*V_00 + log(p_01)*V_01 + log(p_10)*V_10 + log(p_11)*V_11
    return (-2*(al-bl))
end
		

Listing 8.13/8.14: Backtesting S&P 500 in R
Last updated August 2016

W1=WE+1
ya=y[W1:TT]
VaRa=VaR[W1:TT,]
m=c("EWMA","MA","HS","GARCH")
for (i in 1:4){
  q= y[W1:TT]< -VaR[W1:TT,i]
  v=VaRa*0
  v[q,i]=1
  ber=bern_test(p,v[,i])
  ind=ind_test(v[,i])
  cat(i,m[i],'Bernoulli',ber,1-pchisq(ber,1),"independence",ind,1-pchisq(ind,1),"\n")
}
		
Listing 8.13/8.14: Backtesting S&P 500 in Julia
Last updated June 2018

using Distributions;
W1 = WE+1
ya = y[W1:T]
VaRa = VaR[W1:T,:]
m = ["EWMA", "MA", "HS", "GARCH"]
for i in range(1,4)
    q = y[W1:T] .< -VaR[W1:T,i]
    v = VaRa .* 0
    v[q,i] = 1
    ber = bern_test(p, v[:,i])
    ind = ind_test(v[:,i])
    println([i, m[i], ber, 1-cdf(Chisq(1), ber), ind, 1-cdf(Chisq(1), ind)])
end
		

Listing 8.15/8.16: Backtest ES in R
Last updated August 2016

VaR = matrix(nrow=TT,ncol=2)                  # VaR forecasts for 2 models
ES = matrix(nrow=TT,ncol=2)                   # ES forecasts for 2 models
for (t in (WE+1):TT){
  t1 = t-WE;
  t2 = t-1;
  window = y[t1:t2]
  s11 = lambda * s11  + (1-lambda) * y[t-1]^2
  VaR[t,1] = -qnorm(p) * sqrt(s11) * value    # EWMA
  ES[t,1] = sqrt(s11) * dnorm(qnorm(p)) / p
  ys = sort(window)
  VaR[t,2] = -ys[l1]*value                    # HS
  ES[t,2] = -mean(ys[1:l1]) * value
}
		
Listing 8.15/8.16: Backtest ES in Julia
Last updated June 2018

using Distributions;
VaR = fill!(Array{Float64}(T,2), NaN)                            # VaR forecasts
ES = fill!(Array{Float64}(T,2), NaN)                             # ES forecasts
for t in range(WE+1, T-WE)
    t1 = t - WE
    t2 = t - 1
    window = y[t1:t2]
    s11 = lambda*s11+(1-lambda)*y[t-1]^2
    VaR[t,1]=-quantile(Normal(0,1),p)*sqrt(s11)*value            # EWMA
    ES[t,1]=sqrt(s11)*pdf(Normal(0,1),quantile(Normal(0,1),p))/p
    ys = sort(window)
    VaR[t,2] = -ys[l1] * value                                   # HS
    ES[t,2] = -mean(ys[1:l1]) * value
end
		

Listing 8.17/8.18: Backtest ES in R
Last updated August 2016

ESa = ES[W1:TT,]
VaRa = VaR[W1:TT,]
for (i in 1:2){
  q = ya <= -VaRa[,i]
  nES = mean(ya[q] / -ESa[q,i])
  cat(i,"nES",nES,"\n")
}
		
Listing 8.17/8.18: ES in Julia
Last updated June 2018

ESa = ES[W1:T,:]
VaRa = VaR[W1:T,:]
m = ["EWMA", "HS"]
for i in range(1,2)
    q = ya .<= -VaRa[:,i]
    nES = mean(ya[q] ./ -ESa[q,i])
    println([i, m[i], "nES", nES])
end