## functions

library(lubridate)
library(zoo)
library(reshape2)
library(xts)

ProcessRawData = function() {
  sp500 = read.csv("data/sp500.csv")
  names(sp500)[2] = "price"
  sp500$y = c(NA, diff(log(sp500$price)))
  sp500 = sp500[!is.na(sp500$y), ]
  sp500$date.ts = ymd(sp500$date)
  sp500$y.ts = zoo(sp500$y, order.by = sp500$date.ts)


  sp500tr = read.csv("data/sp500tr.csv")
  names(sp500tr)[2] = "price"
  sp500tr$y = c(NA, diff(log(sp500tr$price)))
  sp500tr = sp500tr[!is.na(sp500tr$y), ]
  sp500tr$date.ts = ymd(sp500tr$date)
  sp500tr$y.ts = zoo(sp500tr$y, order.by = sp500tr$date.ts)


  stocks = read.csv("data/stocks.csv")
  names(stocks)[3:4] = c("UnAdjustedPrice", "price")
  stocks$date = as.Date(as.character(stocks$date), format = "%Y%m%d")
  Price = dcast(stocks, date ~ ticker, value.var = "price")
  head(Price, 2)
  UnAdjustedPrice = dcast(stocks, date ~ ticker, value.var = "UnAdjustedPrice")

  Return = Price
  for (i in 2:dim(Price)[2]) Return[, i] = c(NA, diff(log(Price[, i])))

  mask = complete.cases(Return[, -1])
  Price = Price[mask, ]
  UnAdjustedPrice = UnAdjustedPrice[mask, ]
  Return = Return[mask, ]

  data = list(
    Return = Return,
    Price = Price,
    UnAdjustedPrice = UnAdjustedPrice,
    sp500 = sp500,
    sp500tr = sp500tr,
    Ticker = unique(stocks$ticker)
  )

  return(data)
}





myACF = function(x, n = 50, lwd = 2, col1 = co2[2], col2 = co2[1], main = NULL) {
  a = acf(x, n, plot = FALSE)
  significance_level = qnorm((1 + 0.95) / 2) / sqrt(sum(!is.na(x)))
  barplot(a$acf[2:n, , 1],
    las = 1, col = col1, border = FALSE,
    xlab = "lag", ylab = "ACF", main = main
  )
  abline(significance_level, 0, col = col2, lwd = lwd)
  abline(-significance_level, 0, col = col2, lwd = lwd)
  axis(side = 1)
}



NormalVaR = function(p, sd = 1) {
  return(-qnorm(p, sd = sd))
}

NormalES = function(p, sd = 1) {
  return(sd * dnorm(qnorm(p)) / (p))
}

# The variance normalisation divides by sqrt(df/(df-2)), which needs df > 2.
# Without the check, df = 2 makes the scale infinite and returns a VaR of zero:
# an undefined variance reported as an absence of risk.
tVaR = function(p, df, sd = 1, Standardized = TRUE) {
  if (Standardized && df <= 2) stop("Standardized Student-t needs df > 2: the variance is undefined at or below 2")
  Scale = 1
  if (Standardized) Scale = sqrt(df / (df - 2))
  return(-sd * qt(p = p, df = df) / Scale)
}
# ES carries a second restriction. The tail mean of a raw t is finite only for
# df > 1, so the unstandardised branch needs that even though it skips the
# variance normalisation.
tES = function(p, df, sd = 1, Standardized = TRUE) {
  if (Standardized && df <= 2) stop("Standardized Student-t needs df > 2: the variance is undefined at or below 2")
  if (df <= 1) stop("Student-t ES needs df > 1: the tail mean is infinite at or below 1")
  Scale = 1
  if (Standardized) Scale = sqrt(df / (df - 2))
  return((sd * dt(qt(p = p, df = df), df = df) * ((df + (qt(p = p, df = df))^2) / (df - 1)) / p) / Scale)
}


source("risk.r")  # EmpiricalES; chapters source functions.r with chdir = TRUE

Risk_HS = function(y, par, Print = FALSE) {
  ys = sort(tail(y, par$WE))
  k = ceiling(par$probability * par$WE)
  if (k < 1 || k > par$WE) stop("probability * WE must give a rank between 1 and WE")
  VaR = -ys[k] * par$value
  ES = -EmpiricalES(ys, par$probability) * par$value
  if (Print) cat("The HS ", par$probability * 100, "% VaR is $", round(VaR, 1), ", while the ES is $", round(ES, 1), "\n", sep = "")

  return(list(VaR = VaR, ES = ES, method = "HS", par = par))
}
Risk_EWMA = function(y, par, Print = FALSE) {
  y = tail(y, par$WE)
  sigma2 = var(y)
  for (i in 2:length(y)) {
    sigma2 = par$lambda * sigma2 + (1 - par$lambda) * y[i - 1]^2
  }
  sigma2 = par$lambda * sigma2 + (1 - par$lambda) * y[length(y)]^2
  sigma = sqrt(sigma2)
  VaR = NormalVaR(p = par$probability, sd = sigma) * par$value
  ES = NormalES(p = par$probability, sd = sigma) * par$value
  if (Print) {
    cat("The EWMA forecast volatility is", round(sigma, 4), "\n")
    cat("The EWMA ", par$probability * 100, "% VaR is $", round(VaR, 1), ", while the ES is $", round(ES, 1), "\n", sep = "")
  }
  return(list(VaR = VaR, ES = ES, method = "EWMA", sigma = sigma))
}

Risk_nGARCH = function(y, par, Print = FALSE) {
  library(tsgarch)
  ys = tail(y, par$WE)
  ys = ys - mean(ys)
  ys_xts = xts(ys, order.by = seq(as.Date("2000-01-01"), by = "day", length.out = length(ys)))

  spec = garch_modelspec(ys_xts, model = "garch", order = c(1, 1), constant = FALSE)
  res = estimate(spec)

  pred = predict(res, h = 1)
  sigma = as.numeric(pred$sigma)
  params = coef(res)

  VaR = NormalVaR(p = par$probability, sd = sigma) * par$value
  ES = NormalES(p = par$probability, sd = sigma) * par$value

  if (Print) {
    cat("The GARCH forecast volatility is", round(sigma, 4), "\n")
    cat("The GARCH ", par$probability * 100, "% VaR is $", round(VaR, 1), ", while the ES is $", round(ES, 1), "\n", sep = "")
  }
  return(list(VaR = VaR, ES = ES, method = "nGARCH", parameters = params, sigma = sigma))
}

Risk_tGARCH = function(y, par, Print = FALSE) {
  library(tsgarch)
  ys = tail(y, par$WE)
  ys = ys - mean(ys)
  ys_xts = xts(ys, order.by = seq(as.Date("2000-01-01"), by = "day", length.out = length(ys)))

  spec = garch_modelspec(ys_xts, model = "garch", order = c(1, 1),
                          constant = FALSE, distribution = "std")
  res = estimate(spec)

  pred = predict(res, h = 1)
  sigma = as.numeric(pred$sigma)
  params = coef(res)
  nu = params["shape"]

  VaR = tVaR(p = par$probability, sd = sigma, df = nu, Standardized = TRUE) * par$value
  ES = tES(p = par$probability, sd = sigma, df = nu, Standardized = TRUE) * par$value

  if (Print) {
    cat("The GARCH-t forecast volatility is", round(sigma, 4), "\n")
    cat("The GARCH-t ", par$probability * 100, "% VaR is $", round(VaR, 1), ", while the ES is $", round(ES, 1), "\n", sep = "")
  }
  return(list(VaR = VaR, ES = ES, method = "tGARCH", parameters = params, sigma = sigma, nu = nu))
}

RunOneDay = function(t = NULL, y = NULL, par = NULL, Methods, Print = FALSE, HeaderOnly = FALSE) {
  if (HeaderOnly) {
    header = c()
    for (i in 1:length(Methods)) {
      header[i] = paste0("VaR.", Methods[i])
      header[i + length(Methods)] = paste0("ES.", Methods[i])
    }
    header = c(header, "index", "y")
    return(header)
  }
  t2 = t - 1
  t1 = t2 - par$WE + 1
  if (Print) cat(t, t2, t1, "\n")
  res = rep(0, 2 * length(Methods))
  for (i in 1:length(Methods)) {
    x = do.call(what = paste0("Risk_", Methods[i]), args = list(y = y[t1:t2], par = par, Print = Print))
    res[i] = x$VaR
    res[i + length(Methods)] = x$ES
  }
  res = c(res, t)
  res = c(res, y[t])

  return(res)
}

LoadBacktestData = function() {
  cores = detectCores()
  data = ProcessRawData()
  sp500 = data$sp500
  par = list()
  par$probability = 0.01 # probability level (1% VaR)
  par$value = 1000 # portfolio value in currency units
  par$WE = 2000 # estimation window size
  par$T = length(sp500$y) # total sample size
  par$WT = par$T - par$WE # testing window size
  par$lambda = 0.94 # decay parameter for EWMA
  par$Methods = c("HS", "EWMA", "nGARCH", "tGARCH") # models we run
  setup = list(
    y = sp500$y,
    range = (par$WE + 1):par$T,
    Methods = par$Methods,
    date = sp500$date,
    par = par
  )
  file_name = paste0(digest(setup), ".RData")
  if (file.exists(file_name)) {
    load(file_name)
  } else {
    message("Running")
    backtest = mclapply(
      setup$range,
      RunOneDay,
      y = setup$y,
      Methods = par$Methods,
      mc.cores = cores,
      mc.preschedule = TRUE
    )

    backtest = data.frame(do.call(rbind, backtest))

    names(backtest) = RunOneDay(Methods = par$Methods, HeaderOnly = TRUE)
    backtest$date = setup$date[backtest$index]
    backtest$date.t = ymd(backtest$date)
    save(backtest, file = file_name)
  }
  return(list(par = par, backtest = backtest))
}
