Chapter 8. Backtesting and Stress Testing (in R/Python)

  1. Home
  2. Book code
  3. Chapter 8. Backtesting and Stress Testing (in R/Python)
Copyright 2011, 2016, 2018 Jon Danielsson. This code is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This code is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. The GNU General Public License is available at: https://www.gnu.org/licenses/.
The original 2011 R code will not fully work on a recent R because there have been some changes to libraries. The latest version of the Matlab code only uses functions from Matlab toolboxes.
The GARCH functionality in the econometric toolbox in Matlab is trying to be too clever, but can't deliver and could well be buggy. If you want to try that, here are the docs (estimate). Besides, it can only do univariate GARCH and so can't be used in Chapter 3. Kevin Sheppard's MFE toolbox is much better, while not as user friendly, it is much better written and is certainly more comprehensive. It can be downloaded here and the documentation here is quite detailed.

Listing 8.1/8.2: Load data in R
Last updated August 2016 

library(tseries)
library(zoo)
p = zoo(read.csv('index.csv',header=TRUE,sep=','))
y=diff(log(p))                                     # get returns
y=coredata(y)                                      # strip date information
Listing 8.1/8.2: Load data in Python
Last updated June 2018 

import numpy as np
price = np.loadtxt('index.csv',delimiter=',',skiprows=1)
y = np.diff(np.log(price), n=1, axis=0)                  # get returns
Listing 8.3/8.4: Set backtest up in R
Last updated August 2016 

TT = length(y)                                    # number of obs for y
WE = 1000                                         # estimation window length
p = 0.01                                          # probability
l1 = WE*p                                         # HS observation
value = 1;                                        # portfolio value
VaR = matrix(nrow=TT,ncol=4)                      # matrix for forecasts
## EWMA setup
lambda = 0.94;
s11 = var(y[1:30]);
for(t in 2:WE) s11=lambda*s11+(1-lambda)*y[t-1]^2
library(fGarch)
Listing 8.3/8.4: Set backtest up in Python
Last updated June 2018 

import numpy as np
T = len(y)                            # number of obs for y
WE = 1000                             # estimation window length
p = 0.01                              # probability
l1 = int(WE * p)                      # HS observation
value = 1                             # portfolio value
VaR = np.full([T,4], np.nan)          # matrix for forecasts
## EWMA setup
lmbda = 0.94
s11 = np.var(y[1:30])
for t in range(1,WE):
    s11=lmbda*s11+(1-lmbda)*y[t-1]**2
Listing 8.5/8.6: Running backtest in R
Last updated August 2016 

for (t in (WE+1):TT){
  t1 = t-WE;                                     # start of the data window
  t2 = t-1;                                      # end of the data window
  window = y[t1:t2]                              # data for estimation
  s11=lambda*s11+(1-lambda)*y[t-1]^2
  VaR[t,1] = -qnorm(p) * sqrt(s11) * value       # EWMA
  VaR[t,2] = - sd(window) * qnorm(p)*value       # MA
  ys = sort(window)
  VaR[t,3] = -ys[l1]*value                       # HS
  g=garchFit(formula=~garch(1,1),window,
             trace=FALSE,include.mean=FALSE)
  par=g@fit$matcoef
  s4=par[1]+par[2]*window[WE]^2+par[3]*g@h.t[WE]
  VaR[t,4] = -qnorm(p) * sqrt(s4) * value        # GARCH(1,1)
}
Listing 8.5/8.6: Running backtest in Python
Last updated June 2018 

import numpy as np
from scipy import stats
from arch import arch_model
for t in range(WE, T):
    t1 = t - WE                                             # start of data window
    t2 = t - 1                                              # end of data window
    window = y[t1:t2+1]                                     # data for estimation
    s11 = lmbda * s11 + (1-lmbda) * y[t-1]**2
    VaR[t,0]=-stats.norm.ppf(p)*np.sqrt(s11)*value          # EWMA
    VaR[t,1]=-np.std(window,ddof=1)*stats.norm.ppf(p)*value # MA
    ys = np.sort(window)
    VaR[t,2] = -ys[l1 - 1] * value                          # HS
    am = arch_model(window, mean = 'Zero',vol='Garch',
                    p=1, o=0, q=1, dist='Normal')
    res = am.fit(update_freq=5)
    par = [res.params[0], res.params[1], res.params[2]]
    print (len(window))
    print (WE)
    s4 = par[0] + par[1] * window[WE - 1]**2 + par[
        2] * res.conditional_volatility[-1]**2
    VaR[t,3] = -np.sqrt(s4) * stats.norm.ppf(p) * value     # GARCH(1,1)
## GARCH optimization in Python has some convergence issues
Listing 8.7/8.8: Backtesting analysis in R
Last updated June 2018 

W1=WE+1
for (i in 1:4){
  VR = sum(y[W1:TT]< -VaR[W1:TT,i])/(p*(TT-WE))
  s = sd(VaR[W1:TT,i])
  cat(i,"VR",VR,"VaR vol",s,"\n")
}
matplot(cbind(y[W1:TT],VaR[W1:TT,]),type='l',col=1:5,las=1,ylab="",lty=1:5)
legend("topleft",legend=c("Returns","EWMA","MA","HS","GARCH"),lty=1:5,col=1:5,bty="n")
Listing 8.7/8.8: Backtesting analysis in Python
Last updated June 2018 

W1 = WE                                         # Python index starts at 0
m = ["EWMA", "MA", "HS", "GARCH"]
for i in range(4):
    VR = sum(y[W1:T] < -VaR[W1:T,i])/(p*(T-WE))
    s = np.std(VaR[W1:T, i], ddof=1)
    print ([i, m[i], VR, s])
plt.plot(y[W1:T])
plt.plot(VaR[W1:T])
plt.show()
plt.close()
Listing 8.9/8.10: Bernoulli coverage test in R
Last updated August 2016 

bern_test=function(p,v){
  lv=length(v)
  sv=sum(v)
  al=log(p)*sv+log(1-p)*(lv-sv)
  bl=log(sv/lv)*sv +log(1-sv/lv)*(lv-sv)
  return(-2*(al-bl))
}
Listing 8.9/8.10: Bernoulli coverage test in Python
Last updated June 2018 

def bern_test(p,v):
    lv = len(v)
    sv = sum(v)
    al = np.log(p)*sv + np.log(1-p)*(lv-sv)
    bl = np.log(sv/lv)*sv + np.log(1-sv/lv)*(lv-sv)
    return (-2*(al-bl))
Listing 8.11/8.12: Independence test in R
Last updated June 2018 

ind_test=function(V){
  J=matrix(ncol=4,nrow=length(V))
  for (i in 2:length(V)){
    J[i,1]=V[i-1]==0 & V[i]==0
    J[i,2]=V[i-1]==0 & V[i]==1
    J[i,3]=V[i-1]==1 & V[i]==0
    J[i,4]=V[i-1]==1 & V[i]==1
  }
  V_00=sum(J[,1],na.rm=TRUE)
  V_01=sum(J[,2],na.rm=TRUE)
  V_10=sum(J[,3],na.rm=TRUE)
  V_11=sum(J[,4],na.rm=TRUE)
  p_00=V_00/(V_00+V_01)
  p_01=V_01/(V_00+V_01)
  p_10=V_10/(V_10+V_11)
  p_11=V_11/(V_10+V_11)
  hat_p=(V_01+V_11)/(V_00+V_01+V_10+V_11)
  al = log(1-hat_p)*(V_00+V_10) + log(hat_p)*(V_01+V_11)
  bl = log(p_00)*V_00 + log(p_01)*V_01 + log(p_10)*V_10 + log(p_11)*V_11
  return(-2*(al-bl))
}
Listing 8.11/8.12: Independence test in Python
Last updated June 2018 

def ind_test(V):
    J = np.full([T,4], 0)
    for i in range(1,len(V)-1):
        J[i,0] = (V[i-1] == 0) & (V[i] == 0)
        J[i,1] = (V[i-1] == 0) & (V[i] == 1)
        J[i,2] = (V[i-1] == 1) & (V[i] == 0)
        J[i,3] = (V[i-1] == 1) & (V[i] == 1)
    V_00 = sum(J[:,0])
    V_01 = sum(J[:,1])
    V_10 = sum(J[:,2])
    V_11 = sum(J[:,3])
    p_00=V_00/(V_00+V_01)
    p_01=V_01/(V_00+V_01)
    p_10=V_10/(V_10+V_11)
    p_11=V_11/(V_10+V_11)
    hat_p = (V_01+V_11)/(V_00+V_01+V_10+V_11)
    al = np.log(1-hat_p)*(V_00+V_10) + np.log(hat_p)*(V_01+V_11)
    bl = np.log(p_00)*V_00 + np.log(p_01)*V_01 + np.log(p_10)*V_10 + np.log(p_11)*V_11
    return (-2*(al-bl))
Listing 8.13/8.14: Backtesting S&P 500 in R
Last updated August 2016 

W1=WE+1
ya=y[W1:TT]
VaRa=VaR[W1:TT,]
m=c("EWMA","MA","HS","GARCH")
for (i in 1:4){
  q= y[W1:TT]< -VaR[W1:TT,i]
  v=VaRa*0
  v[q,i]=1
  ber=bern_test(p,v[,i])
  ind=ind_test(v[,i])
  cat(i,m[i],'Bernoulli',ber,1-pchisq(ber,1),"independence",ind,1-pchisq(ind,1),"\n")
}
Listing 8.13/8.14: Backtesting S&P 500 in Python
Last updated June 2018 

W1 = WE
ya = y[W1:T]
VaRa = VaR[W1:T,]
m = ['EWMA', 'MA', 'HS', 'GARCH']
for i in range(4):
    q = y[W1:T] < -VaR[W1:T,i]
    v = VaRa*0
    v[q,i] = 1
    ber = bern_test(p, v[:,i])
    ind = ind_test(v[:,i])
    print ([i, m[i], ber, 1 - stats.chi2.cdf(ber, 1), ind, 1 - stats.chi2.cdf(ind, 1)])
Listing 8.15/8.16: Backtest ES in R
Last updated August 2016 

VaR = matrix(nrow=TT,ncol=2)                  # VaR forecasts for 2 models
ES = matrix(nrow=TT,ncol=2)                   # ES forecasts for 2 models
for (t in (WE+1):TT){
  t1 = t-WE;
  t2 = t-1;
  window = y[t1:t2]
  s11 = lambda * s11  + (1-lambda) * y[t-1]^2
  VaR[t,1] = -qnorm(p) * sqrt(s11) * value    # EWMA
  ES[t,1] = sqrt(s11) * dnorm(qnorm(p)) / p
  ys = sort(window)
  VaR[t,2] = -ys[l1]*value                    # HS
  ES[t,2] = -mean(ys[1:l1]) * value
}
Listing 8.15/8.16: Backtest ES in Python
Last updated June 2018 

VaR = np.full([T,2], np.nan)                                 # VaR forecasts
ES = np.full([T,2], np.nan)                                  # ES forecasts
for t in range(WE, T):
    t1 = t - WE
    t2 = t - 1
    window = y[t1:t2+1]
    s11 = lmbda * s11 + (1-lmbda) * y[t-1]**2
    VaR[t,0] = -stats.norm.ppf(p) * np.sqrt(s11)*value       # EWMA
    ES[t,0]=np.sqrt(s11)*stats.norm.pdf(stats.norm.ppf(p))/p
    ys = np.sort(window)
    VaR[t,1] = -ys[l1 - 1] * value                           # HS
    ES[t,1] = -np.mean(ys[0:l1]) * value
Listing 8.17/8.18: Backtest ES in R
Last updated August 2016 

ESa = ES[W1:TT,]
VaRa = VaR[W1:TT,]
for (i in 1:2){
  q = ya <= -VaRa[,i]
  nES = mean(ya[q] / -ESa[q,i])
  cat(i,"nES",nES,"\n")
}
Listing 8.17/8.18: ES in Python
Last updated June 2018 

ESa = ES[W1:T,:]
VaRa = VaR[W1:T,:]
m = ["EWMA", "HS"]
for i in range(2):
    q = ya <= -VaRa[:,i]
    nES = np.mean(ya[q] / -ESa[q,i])
    print ([i, m[i], 'nES', nES])