# Common functions for Financial Risk Forecasting Notebook
# Functions are added as chapters are converted

using HTTP, JSON3, DataFrames, Dates, CSV

const EODH_TOKEN = get(ENV, "EODH_TOKEN", "demo")
const EODH_BASE_URL = "https://eodhd.com/api"

function get_prices(symbol, exchange)
    url = "$EODH_BASE_URL/eod/$symbol.$exchange?api_token=$EODH_TOKEN&fmt=json"
    response = HTTP.get(url)
    data = JSON3.read(String(response.body))
    return DataFrame(data)
end


"""
    ProcessRawData()

Load and process S&P 500 and stock data.

Returns a Dict containing:
- sp500: DataFrame with S&P 500 prices and log returns
- sp500tr: DataFrame with S&P 500 total returns
- Price: DataFrame with stock prices (wide format)
- UnAdjustedPrice: DataFrame with unadjusted stock prices
- Return: DataFrame with stock log returns
- Ticker: Vector of ticker symbols
"""
function ProcessRawData()
    # Process S&P 500
    sp500 = CSV.read("data/sp500.csv", DataFrame)
    rename!(sp500, :Close => :price)
    sp500.y = [missing; diff(log.(sp500.price))]
    sp500 = sp500[.!ismissing.(sp500.y), :]

    # Process S&P 500 Total Return
    sp500tr = CSV.read("data/sp500tr.csv", DataFrame)
    rename!(sp500tr, :Close => :price)
    sp500tr.y = [missing; diff(log.(sp500tr.price))]
    sp500tr = sp500tr[.!ismissing.(sp500tr.y), :]

    # Process stocks
    stocks = CSV.read("data/stocks.csv", DataFrame)
    rename!(stocks, :Close => :UnAdjustedPrice, :Adjusted_close => :price)
    stocks.date = Date.(string.(stocks.date), DateFormat("yyyymmdd"))

    # Reshape to wide format
    Price = unstack(stocks, :date, :ticker, :price)
    Price = Price[:, ["date"; sort(names(Price)[2:end])]]
    UnAdjustedPrice = unstack(stocks, :date, :ticker, :UnAdjustedPrice)
    UnAdjustedPrice = UnAdjustedPrice[:, ["date"; sort(names(UnAdjustedPrice)[2:end])]]

    # Calculate returns
    Return = copy(Price)
    for col in names(Return)[2:end]
        Return[!, col] = [missing; diff(log.(Price[!, col]))]
    end

    # Remove missing rows
    mask = completecases(Return[:, 2:end])
    Price = Price[mask, :]
    UnAdjustedPrice = UnAdjustedPrice[mask, :]
    Return = Return[mask, :]

    # Get unique tickers
    Ticker = unique(stocks.ticker)

    return Dict(
        "sp500" => sp500,
        "sp500tr" => sp500tr,
        "Price" => Price,
        "UnAdjustedPrice" => UnAdjustedPrice,
        "Return" => Return,
        "Ticker" => Ticker
    )
end


# Risk functions for backtesting
using Distributions, Statistics
using ARCHModels

"""Value at Risk under normal distribution"""
function NormalVaR(p, sd=1.0)
    return -quantile(Normal(0, sd), p)
end

"""Expected Shortfall under normal distribution"""
function NormalES(p, sd=1.0)
    return sd * pdf(Normal(), quantile(Normal(), p)) / p
end

"""
Value at Risk under Student-t distribution.

The variance normalisation divides by sqrt(df/(df-2)), which needs df > 2.
Without the check, df = 2 makes the scale infinite and returns a VaR of zero:
an undefined variance reported as an absence of risk.
"""
function tVaR(p, df, sd=1.0; Standardized=true)
    (!Standardized || df > 2) || error("Standardized Student-t needs df > 2: the variance is undefined at or below 2")
    scale = Standardized ? sqrt(df / (df - 2)) : 1.0
    return -sd * quantile(TDist(df), p) / scale
end

"""
Expected Shortfall under Student-t distribution.

ES carries a second restriction. The tail mean of a raw t is finite only for
df > 1, so the unstandardised branch needs that even though it skips the
variance normalisation.
"""
function tES(p, df, sd=1.0; Standardized=true)
    (!Standardized || df > 2) || error("Standardized Student-t needs df > 2: the variance is undefined at or below 2")
    df > 1 || error("Student-t ES needs df > 1: the tail mean is infinite at or below 1")
    scale = Standardized ? sqrt(df / (df - 2)) : 1.0
    q = quantile(TDist(df), p)
    return (sd * pdf(TDist(df), q) * ((df + q^2) / (df - 1)) / p) / scale
end

include(joinpath(@__DIR__, "risk.jl"))  # EmpiricalES

"""Historical Simulation VaR and ES"""
function Risk_HS(y, par)
    y_window = y[end-par["WE"]+1:end]
    ys = sort(y_window)
    k = ceil(Int, par["probability"] * par["WE"])
    (1 <= k <= par["WE"]) || error("probability * WE must give a rank between 1 and WE")
    VaR = -ys[k] * par["value"]
    ES = -EmpiricalES(ys, par["probability"]) * par["value"]
    return Dict("VaR" => VaR, "ES" => ES)
end

"""EWMA VaR and ES"""
function Risk_EWMA(y, par)
    y_window = y[end-par["WE"]+1:end]
    sigma2 = var(y_window)
    for i in 2:length(y_window)
        sigma2 = par["lambda"] * sigma2 + (1 - par["lambda"]) * y_window[i-1]^2
    end
    sigma2 = par["lambda"] * sigma2 + (1 - par["lambda"]) * y_window[end]^2
    sigma = sqrt(sigma2)
    VaR = NormalVaR(par["probability"], sigma) * par["value"]
    ES = NormalES(par["probability"], sigma) * par["value"]
    return Dict("VaR" => VaR, "ES" => ES)
end

"""GARCH(1,1) VaR and ES with normal distribution"""
function Risk_nGARCH(y, par)
    y_window = y[end-par["WE"]+1:end]
    y_dm = y_window .- mean(y_window)
    scale = std(y_dm)
    y_scaled = y_dm ./ scale

    model = fit(GARCH{1,1}, y_scaled; meanspec=NoIntercept)
    sigma2_forecast = predict(model, :variance, 1)[1]
    sigma = sqrt(sigma2_forecast) * scale
    VaR = NormalVaR(par["probability"], sigma) * par["value"]
    ES = NormalES(par["probability"], sigma) * par["value"]
    return Dict("VaR" => VaR, "ES" => ES)
end

"""GARCH(1,1) VaR and ES with Student-t distribution"""
function Risk_tGARCH(y, par)
    y_window = y[end-par["WE"]+1:end]
    y_dm = y_window .- mean(y_window)
    scale = std(y_dm)
    y_scaled = y_dm ./ scale

    model = fit(GARCH{1,1}, y_scaled; dist=StdT, meanspec=NoIntercept)
    sigma2_forecast = predict(model, :variance, 1)[1]
    sigma = sqrt(sigma2_forecast) * scale
    nu = model.dist.coefs[1]
    VaR = tVaR(par["probability"], nu, sigma) * par["value"]
    ES = tES(par["probability"], nu, sigma) * par["value"]
    return Dict("VaR" => VaR, "ES" => ES)
end

"""Run all risk models for a single day"""
function RunOneDay(t, y, par; Methods=["HS", "EWMA", "nGARCH", "tGARCH"])
    t2 = t - 1
    t1 = t2 - par["WE"] + 1
    y_window = y[t1:t2]

    risk_funcs = Dict(
        "HS" => Risk_HS,
        "EWMA" => Risk_EWMA,
        "nGARCH" => Risk_nGARCH,
        "tGARCH" => Risk_tGARCH
    )

    results = Dict{String, Any}("index" => t, "y" => y[t])
    for method in Methods
        res = risk_funcs[method](y_window, par)
        results["VaR.$method"] = res["VaR"]
        results["ES.$method"] = res["ES"]
    end

    return results
end
