Prediction and model training work
This commit is contained in:
@@ -0,0 +1,308 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "038bf2ce",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# Source of truth for model inputs (must match training order exactly)\n",
|
||||
"FEATURE_COLUMNS = [\n",
|
||||
" \"SPY_ret_5\",\n",
|
||||
" \"SPY_ret_20\",\n",
|
||||
" \"SPY_dist_sma50\",\n",
|
||||
" \"VIX_change_5\",\n",
|
||||
" \"VIX_rank_20\",\n",
|
||||
" \"TLT_ret_10\",\n",
|
||||
" \"USO_ret_5\",\n",
|
||||
" \"SPY_TLT_ratio_ret\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"DEFAULT_SYMBOLS = {\n",
|
||||
" \"SPY\": \"SPY\",\n",
|
||||
" \"VIXY\": \"VIX\", # Change to \"VIX\": \"VIX\" if using raw VIX Parquet\n",
|
||||
" \"TLT\": \"TLT\",\n",
|
||||
" \"USO\": \"USO\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_raw_close_prices(\n",
|
||||
" raw_data_dir: Path, symbols: dict[str, str]\n",
|
||||
") -> pd.DataFrame:\n",
|
||||
" \"\"\"Reads raw Parquet files and merges close prices into a single inner-joined DataFrame.\"\"\"\n",
|
||||
" frames = []\n",
|
||||
" for symbol, alias in symbols.items():\n",
|
||||
" path = raw_data_dir / f\"{symbol}.parquet\"\n",
|
||||
" if not path.exists():\n",
|
||||
" raise FileNotFoundError(f\"Missing raw data file: {path}\")\n",
|
||||
"\n",
|
||||
" frame = pd.read_parquet(path)\n",
|
||||
" if \"date\" in frame.columns:\n",
|
||||
" frame = frame.set_index(\"date\")\n",
|
||||
"\n",
|
||||
" frame.index = pd.to_datetime(frame.index)\n",
|
||||
" frame = frame.sort_index()\n",
|
||||
" frames.append(frame[[\"close\"]].rename(columns={\"close\": f\"{alias}_close\"}))\n",
|
||||
"\n",
|
||||
" return pd.concat(frames, axis=1, join=\"inner\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def compute_features(prices: pd.DataFrame) -> pd.DataFrame:\n",
|
||||
" \"\"\"Computes engineered features from raw merged price history.\"\"\"\n",
|
||||
" df = prices.copy()\n",
|
||||
"\n",
|
||||
" # Target asset features\n",
|
||||
" df[\"SPY_ret_5\"] = df[\"SPY_close\"].pct_change(5)\n",
|
||||
" df[\"SPY_ret_20\"] = df[\"SPY_close\"].pct_change(20)\n",
|
||||
"\n",
|
||||
" sma_50 = df[\"SPY_close\"].rolling(50).mean()\n",
|
||||
" df[\"SPY_dist_sma50\"] = (df[\"SPY_close\"] - sma_50) / sma_50\n",
|
||||
"\n",
|
||||
" # Volatility / Market Stress\n",
|
||||
" df[\"VIX_change_5\"] = df[\"VIX_close\"].pct_change(5)\n",
|
||||
" df[\"VIX_rank_20\"] = df[\"VIX_close\"].rolling(20).rank(pct=True)\n",
|
||||
"\n",
|
||||
" # Macro & Relative ratios\n",
|
||||
" df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
|
||||
" df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
|
||||
" df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
|
||||
"\n",
|
||||
" return df[FEATURE_COLUMNS]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_latest_inference_features(\n",
|
||||
" raw_data_dir: Path,\n",
|
||||
" symbols: dict[str, str] | None = None,\n",
|
||||
" max_age_days: int = 1,\n",
|
||||
") -> pd.DataFrame:\n",
|
||||
" \"\"\"Loads raw prices, computes features, verifies date freshness,\n",
|
||||
"\n",
|
||||
" and returns the latest single row for model prediction.\n",
|
||||
" \"\"\"\n",
|
||||
" symbols = symbols or DEFAULT_SYMBOLS\n",
|
||||
"\n",
|
||||
" # 1. Load prices & compute rolling features\n",
|
||||
" prices = load_raw_close_prices(raw_data_dir, symbols)\n",
|
||||
" features = compute_features(prices).dropna()\n",
|
||||
"\n",
|
||||
" if features.empty:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Not enough historical rows to compute 50-day rolling window features.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # 2. Extract latest available row as a 1-row DataFrame\n",
|
||||
" latest_row = features.iloc[[-1]]\n",
|
||||
" latest_date = latest_row.index[0]\n",
|
||||
"\n",
|
||||
" # 3. Check data freshness and raise a warning if stale\n",
|
||||
" now = pd.Timestamp.now()\n",
|
||||
" latest_date_naive = (\n",
|
||||
" latest_date.tz_localize(None)\n",
|
||||
" if latest_date.tz is not None\n",
|
||||
" else latest_date\n",
|
||||
" )\n",
|
||||
" days_old = (now.floor(\"D\") - latest_date_naive.floor(\"D\")).days\n",
|
||||
"\n",
|
||||
" if days_old > max_age_days:\n",
|
||||
" warnings.warn(\n",
|
||||
" f\"STALE DATA WARNING: Latest feature row is from {latest_date.strftime('%Y-%m-%d')} \"\n",
|
||||
" f\"({days_old} day(s) old). Update raw Parquet files before executing trades.\",\n",
|
||||
" UserWarning,\n",
|
||||
" stacklevel=2,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" return latest_row\n",
|
||||
"\n",
|
||||
"def get_target_exposure(\n",
|
||||
" p_pred: float, p_base: float, sensitivity: float = 5.0\n",
|
||||
") -> float:\n",
|
||||
" \"\"\"Maps predicted probability to a target portfolio equity allocation (0.0 to 1.0).\n",
|
||||
"\n",
|
||||
" - p_pred == p_base --> 50% Target Exposure (Neutral)\n",
|
||||
" - p_pred > p_base --> Scale up toward 100% (Bullish)\n",
|
||||
" - p_pred < p_base --> Scale down toward 0% (Bearish / Cash)\n",
|
||||
" \"\"\"\n",
|
||||
" # Calculate deviation from the historical average\n",
|
||||
" delta = p_pred - p_base\n",
|
||||
"\n",
|
||||
" # Base target allocation is 50% equity / 50% cash\n",
|
||||
" base_allocation = 0.50\n",
|
||||
"\n",
|
||||
" # Sensitivity controls how aggressively probability changes alter allocation\n",
|
||||
" # e.g., a +0.08 delta * 5.0 = +0.40 -> 90% Equity Allocation\n",
|
||||
" target_allocation = base_allocation + (delta * sensitivity)\n",
|
||||
"\n",
|
||||
" # Clamp bounds strictly between 0% (full cash) and 100% (full SPY)\n",
|
||||
" return float(np.clip(target_allocation, 0.0, 1.0))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "0b3d2c25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Date: 2026-07-27 | Prob: 0.5843 | Base: 0.5830\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"0.5064256139268726"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import xgboost as xgb\n",
|
||||
"\n",
|
||||
"# 1. Load model and metadata\n",
|
||||
"model = xgb.XGBClassifier()\n",
|
||||
"model.load_model(\"models/spy_xgb_v1.json\")\n",
|
||||
"\n",
|
||||
"with open(\"models/spy_xgb_v1_meta.json\", \"r\") as f:\n",
|
||||
" meta = json.load(f)\n",
|
||||
"\n",
|
||||
"# 2. Fetch latest features from raw parquet files\n",
|
||||
"RAW_DATA_DIR = Path(\"../data/ibkr/daily\")\n",
|
||||
"X_latest = get_latest_inference_features(RAW_DATA_DIR, max_age_days=1)\n",
|
||||
"\n",
|
||||
"# 3. Predict probability\n",
|
||||
"p_pred = float(model.predict_proba(X_latest[meta[\"feature_cols\"]])[0, 1])\n",
|
||||
"p_base = meta[\"p_base\"]\n",
|
||||
"\n",
|
||||
"print(\n",
|
||||
" f\"Date: {X_latest.index[0].date()} | Prob: {p_pred:.4f} | Base: {p_base:.4f}\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"get_target_exposure(p_pred, p_base, sensitivity=5.0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3f823fa6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>SPY_ret_5</th>\n",
|
||||
" <th>SPY_ret_20</th>\n",
|
||||
" <th>SPY_dist_sma50</th>\n",
|
||||
" <th>VIX_change_5</th>\n",
|
||||
" <th>VIX_rank_20</th>\n",
|
||||
" <th>TLT_ret_10</th>\n",
|
||||
" <th>USO_ret_5</th>\n",
|
||||
" <th>SPY_TLT_ratio_ret</th>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>date</th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>2026-07-27</th>\n",
|
||||
" <td>-0.004043</td>\n",
|
||||
" <td>0.013855</td>\n",
|
||||
" <td>-0.007938</td>\n",
|
||||
" <td>0.007065</td>\n",
|
||||
" <td>0.75</td>\n",
|
||||
" <td>-0.00262</td>\n",
|
||||
" <td>-0.005976</td>\n",
|
||||
" <td>-0.002378</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
||||
"date \n",
|
||||
"2026-07-27 -0.004043 0.013855 -0.007938 0.007065 0.75 \n",
|
||||
"\n",
|
||||
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret \n",
|
||||
"date \n",
|
||||
"2026-07-27 -0.00262 -0.005976 -0.002378 "
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"X_latest"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "476fd7fd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user