849 lines
29 KiB
Plaintext
849 lines
29 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# SPY Direction Training Dataset\n",
|
|
"\n",
|
|
"Build the first model-ready dataset from IBKR daily candle Parquets. The target is whether `SPY` closes above, below, or unchanged from today's close five trading days later."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Setup\n",
|
|
"\n",
|
|
"The notebook is deliberately plain pandas so feature logic remains easy to inspect. `VIX_INPUT_SYMBOL` defaults to `VIXY` because that is the VIX-related Parquet currently present locally; switch it to `VIX` after fetching a `VIX.parquet` file."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"(PosixPath('/home/jarno/repos/trading-bot'),\n",
|
|
" PosixPath('/home/jarno/repos/trading-bot/data/alpaca/daily'),\n",
|
|
" PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'))"
|
|
]
|
|
},
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"\n",
|
|
"def find_project_root(start: Path | None = None) -> Path:\n",
|
|
" current = (start or Path.cwd()).resolve()\n",
|
|
" for candidate in [current, *current.parents]:\n",
|
|
" if (candidate / \"pyproject.toml\").exists():\n",
|
|
" return candidate\n",
|
|
" raise RuntimeError(\"Could not find project root containing pyproject.toml\")\n",
|
|
"\n",
|
|
"\n",
|
|
"PROJECT_ROOT = find_project_root()\n",
|
|
"#RAW_DATA_DIR = PROJECT_ROOT / \"data\" / \"ibkr\" / \"daily\"\n",
|
|
"RAW_DATA_DIR = PROJECT_ROOT / \"data\" / \"alpaca\" / \"daily\"\n",
|
|
"OUTPUT_PATH = PROJECT_ROOT / \"data\" / \"training\" / \"spy_direction_5d.parquet\"\n",
|
|
"\n",
|
|
"SPY_SYMBOL = \"SPY\"\n",
|
|
"VIX_INPUT_SYMBOL = \"VIXY\"\n",
|
|
"TLT_SYMBOL = \"TLT\"\n",
|
|
"USO_SYMBOL = \"USO\"\n",
|
|
"\n",
|
|
"TRAIN_FRACTION = 0.70\n",
|
|
"VALIDATION_FRACTION = 0.15\n",
|
|
"TEST_FRACTION = 0.15\n",
|
|
"\n",
|
|
"PROJECT_ROOT, RAW_DATA_DIR, OUTPUT_PATH"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Load Close Prices\n",
|
|
"\n",
|
|
"Each symbol file is read, sorted by trading date, and reduced to a single close-price column. `SPY` defines the observation calendar through the inner join."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"<class 'pandas.DataFrame'>\n",
|
|
"DatetimeIndex: 1258 entries, 2021-08-02 to 2026-08-05\n",
|
|
"Data columns (total 4 columns):\n",
|
|
" # Column Non-Null Count Dtype \n",
|
|
"--- ------ -------------- ----- \n",
|
|
" 0 SPY_close 1258 non-null float64\n",
|
|
" 1 VIX_close 1258 non-null float64\n",
|
|
" 2 TLT_close 1258 non-null float64\n",
|
|
" 3 USO_close 1258 non-null float64\n",
|
|
"dtypes: float64(4)\n",
|
|
"memory usage: 49.1 KB\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>SPY_close</th>\n",
|
|
" <th>VIX_close</th>\n",
|
|
" <th>TLT_close</th>\n",
|
|
" <th>USO_close</th>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>date</th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>2021-08-02</th>\n",
|
|
" <td>437.59</td>\n",
|
|
" <td>25.68</td>\n",
|
|
" <td>150.67</td>\n",
|
|
" <td>49.18</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-08-03</th>\n",
|
|
" <td>441.15</td>\n",
|
|
" <td>24.40</td>\n",
|
|
" <td>150.75</td>\n",
|
|
" <td>48.85</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-08-04</th>\n",
|
|
" <td>438.98</td>\n",
|
|
" <td>24.38</td>\n",
|
|
" <td>151.06</td>\n",
|
|
" <td>47.20</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-08-05</th>\n",
|
|
" <td>441.76</td>\n",
|
|
" <td>23.74</td>\n",
|
|
" <td>150.29</td>\n",
|
|
" <td>48.10</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-08-06</th>\n",
|
|
" <td>442.49</td>\n",
|
|
" <td>23.14</td>\n",
|
|
" <td>147.78</td>\n",
|
|
" <td>47.57</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" SPY_close VIX_close TLT_close USO_close\n",
|
|
"date \n",
|
|
"2021-08-02 437.59 25.68 150.67 49.18\n",
|
|
"2021-08-03 441.15 24.40 150.75 48.85\n",
|
|
"2021-08-04 438.98 24.38 151.06 47.20\n",
|
|
"2021-08-05 441.76 23.74 150.29 48.10\n",
|
|
"2021-08-06 442.49 23.14 147.78 47.57"
|
|
]
|
|
},
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"def load_close(symbol: str, alias: str | None = None) -> pd.DataFrame:\n",
|
|
" alias = alias or symbol\n",
|
|
" path = RAW_DATA_DIR / f\"{symbol}.parquet\"\n",
|
|
" if not path.exists():\n",
|
|
" raise FileNotFoundError(f\"Missing raw data file: {path}\")\n",
|
|
"\n",
|
|
" frame = pd.read_parquet(path)\n",
|
|
" if \"date\" in frame.columns:\n",
|
|
" frame = frame.set_index(\"date\")\n",
|
|
" if \"close\" not in frame.columns:\n",
|
|
" raise ValueError(f\"{path} does not contain a close column\")\n",
|
|
"\n",
|
|
" frame = frame.copy()\n",
|
|
" frame.index = pd.to_datetime(frame.index)\n",
|
|
" frame.index.name = \"date\"\n",
|
|
" frame = frame.sort_index()\n",
|
|
" return frame[[\"close\"]].rename(columns={\"close\": f\"{alias}_close\"})\n",
|
|
"\n",
|
|
"\n",
|
|
"prices = pd.concat(\n",
|
|
" [\n",
|
|
" load_close(SPY_SYMBOL, \"SPY\"),\n",
|
|
" load_close(VIX_INPUT_SYMBOL, \"VIX\"),\n",
|
|
" load_close(TLT_SYMBOL, \"TLT\"),\n",
|
|
" load_close(USO_SYMBOL, \"USO\"),\n",
|
|
" ],\n",
|
|
" axis=1,\n",
|
|
" join=\"inner\",\n",
|
|
")\n",
|
|
"\n",
|
|
"prices.info()\n",
|
|
"prices.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Generate Features And Target\n",
|
|
"\n",
|
|
"Feature values use only current and prior rows. The target looks five trading rows ahead in `SPY_close`; unchanged future prices are encoded as `0.5`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>SPY_ret_5</th>\n",
|
|
" <th>SPY_ret_20</th>\n",
|
|
" <th>SPY_dist_sma50</th>\n",
|
|
" <th>VIX_change_5</th>\n",
|
|
" <th>VIX_rank_20</th>\n",
|
|
" <th>TLT_ret_10</th>\n",
|
|
" <th>USO_ret_5</th>\n",
|
|
" <th>SPY_TLT_ratio_ret</th>\n",
|
|
" <th>spy_up_5d</th>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>date</th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>2021-10-11</th>\n",
|
|
" <td>0.014114</td>\n",
|
|
" <td>-0.026625</td>\n",
|
|
" <td>-0.018145</td>\n",
|
|
" <td>-0.090869</td>\n",
|
|
" <td>0.20</td>\n",
|
|
" <td>-0.033135</td>\n",
|
|
" <td>0.031015</td>\n",
|
|
" <td>0.038908</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-10-12</th>\n",
|
|
" <td>0.001201</td>\n",
|
|
" <td>-0.023752</td>\n",
|
|
" <td>-0.020386</td>\n",
|
|
" <td>-0.074091</td>\n",
|
|
" <td>0.10</td>\n",
|
|
" <td>-0.001041</td>\n",
|
|
" <td>0.008628</td>\n",
|
|
" <td>-0.001303</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-10-13</th>\n",
|
|
" <td>0.000644</td>\n",
|
|
" <td>-0.028356</td>\n",
|
|
" <td>-0.016597</td>\n",
|
|
" <td>-0.081006</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>0.006928</td>\n",
|
|
" <td>0.036928</td>\n",
|
|
" <td>-0.005897</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-10-14</th>\n",
|
|
" <td>0.008754</td>\n",
|
|
" <td>-0.010443</td>\n",
|
|
" <td>-0.000214</td>\n",
|
|
" <td>-0.096759</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>0.010809</td>\n",
|
|
" <td>0.026192</td>\n",
|
|
" <td>-0.011991</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2021-10-15</th>\n",
|
|
" <td>0.018294</td>\n",
|
|
" <td>0.010127</td>\n",
|
|
" <td>0.007213</td>\n",
|
|
" <td>-0.079882</td>\n",
|
|
" <td>0.05</td>\n",
|
|
" <td>-0.002202</td>\n",
|
|
" <td>0.030287</td>\n",
|
|
" <td>-0.003823</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
|
"date \n",
|
|
"2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 0.20 \n",
|
|
"2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 0.10 \n",
|
|
"2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 0.05 \n",
|
|
"2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 0.05 \n",
|
|
"2021-10-15 0.018294 0.010127 0.007213 -0.079882 0.05 \n",
|
|
"\n",
|
|
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d \n",
|
|
"date \n",
|
|
"2021-10-11 -0.033135 0.031015 0.038908 1.0 \n",
|
|
"2021-10-12 -0.001041 0.008628 -0.001303 1.0 \n",
|
|
"2021-10-13 0.006928 0.036928 -0.005897 1.0 \n",
|
|
"2021-10-14 0.010809 0.026192 -0.011991 1.0 \n",
|
|
"2021-10-15 -0.002202 0.030287 -0.003823 1.0 "
|
|
]
|
|
},
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = prices.copy()\n",
|
|
"\n",
|
|
"df[\"SPY_ret_5\"] = df[\"SPY_close\"].pct_change(5)\n",
|
|
"df[\"SPY_ret_20\"] = df[\"SPY_close\"].pct_change(20)\n",
|
|
"\n",
|
|
"sma_50 = df[\"SPY_close\"].rolling(50).mean()\n",
|
|
"df[\"SPY_dist_sma50\"] = (df[\"SPY_close\"] - sma_50) / sma_50\n",
|
|
"\n",
|
|
"df[\"VIX_change_5\"] = df[\"VIX_close\"].pct_change(5)\n",
|
|
"df[\"VIX_rank_20\"] = df[\"VIX_close\"].rolling(20).rank(pct=True)\n",
|
|
"\n",
|
|
"df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
|
|
"df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
|
|
"df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
|
|
"\n",
|
|
"spy_forward_close = df[\"SPY_close\"].shift(-5)\n",
|
|
"df[\"spy_up_5d\"] = np.nan\n",
|
|
"df.loc[spy_forward_close > df[\"SPY_close\"], \"spy_up_5d\"] = 1.0\n",
|
|
"df.loc[spy_forward_close < df[\"SPY_close\"], \"spy_up_5d\"] = 0.0\n",
|
|
"df.loc[spy_forward_close == df[\"SPY_close\"], \"spy_up_5d\"] = 0.5\n",
|
|
"\n",
|
|
"FEATURE_COLUMNS = [\n",
|
|
" \"SPY_ret_5\",\n",
|
|
" \"SPY_ret_20\",\n",
|
|
" \"SPY_dist_sma50\",\n",
|
|
" \"VIX_change_5\",\n",
|
|
" \"VIX_rank_20\",\n",
|
|
" \"TLT_ret_10\",\n",
|
|
" \"USO_ret_5\",\n",
|
|
" \"SPY_TLT_ratio_ret\",\n",
|
|
"]\n",
|
|
"TARGET_COLUMN = \"spy_up_5d\"\n",
|
|
"\n",
|
|
"df_model = df[FEATURE_COLUMNS + [TARGET_COLUMN]].dropna().copy()\n",
|
|
"df_model.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Chronological Splits\n",
|
|
"\n",
|
|
"Splits are assigned by row order after feature/target cleanup: oldest rows for training, newer rows for validation, newest rows for test. The `split` column is metadata and should not be used as a model input."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>rows</th>\n",
|
|
" <th>start_date</th>\n",
|
|
" <th>end_date</th>\n",
|
|
" <th>target_mean</th>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>split</th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" <th></th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>train</th>\n",
|
|
" <td>842</td>\n",
|
|
" <td>2021-10-11</td>\n",
|
|
" <td>2025-02-18</td>\n",
|
|
" <td>0.581948</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>validation</th>\n",
|
|
" <td>180</td>\n",
|
|
" <td>2025-02-19</td>\n",
|
|
" <td>2025-11-04</td>\n",
|
|
" <td>0.638889</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>test</th>\n",
|
|
" <td>182</td>\n",
|
|
" <td>2025-11-05</td>\n",
|
|
" <td>2026-07-29</td>\n",
|
|
" <td>0.576923</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" rows start_date end_date target_mean\n",
|
|
"split \n",
|
|
"train 842 2021-10-11 2025-02-18 0.581948\n",
|
|
"validation 180 2025-02-19 2025-11-04 0.638889\n",
|
|
"test 182 2025-11-05 2026-07-29 0.576923"
|
|
]
|
|
},
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"def assign_chronological_splits(\n",
|
|
" frame: pd.DataFrame,\n",
|
|
" train_fraction: float,\n",
|
|
" validation_fraction: float,\n",
|
|
" test_fraction: float,\n",
|
|
") -> pd.Series:\n",
|
|
" total_fraction = train_fraction + validation_fraction + test_fraction\n",
|
|
" if not np.isclose(total_fraction, 1.0):\n",
|
|
" raise ValueError(f\"Split fractions must sum to 1.0, got {total_fraction}\")\n",
|
|
"\n",
|
|
" n_rows = len(frame)\n",
|
|
" train_end = int(n_rows * train_fraction)\n",
|
|
" validation_end = train_end + int(n_rows * validation_fraction)\n",
|
|
"\n",
|
|
" split = pd.Series(index=frame.index, dtype=\"object\")\n",
|
|
" split.iloc[:train_end] = \"train\"\n",
|
|
" split.iloc[train_end:validation_end] = \"validation\"\n",
|
|
" split.iloc[validation_end:] = \"test\"\n",
|
|
" return split\n",
|
|
"\n",
|
|
"\n",
|
|
"df_model[\"split\"] = assign_chronological_splits(\n",
|
|
" df_model,\n",
|
|
" TRAIN_FRACTION,\n",
|
|
" VALIDATION_FRACTION,\n",
|
|
" TEST_FRACTION,\n",
|
|
")\n",
|
|
"\n",
|
|
"split_summary = df_model.groupby(\"split\", sort=False).agg(\n",
|
|
" rows=(TARGET_COLUMN, \"size\"),\n",
|
|
" start_date=(TARGET_COLUMN, lambda values: values.index.min().date()),\n",
|
|
" end_date=(TARGET_COLUMN, lambda values: values.index.max().date()),\n",
|
|
" target_mean=(TARGET_COLUMN, \"mean\"),\n",
|
|
")\n",
|
|
"split_summary"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Inspect And Save\n",
|
|
"\n",
|
|
"`model_input_columns` is the source of truth for columns that are allowed into the model. The saved Parquet includes `date`, features, target, and split metadata."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Rows: 1,204\n",
|
|
"Feature columns: ['SPY_ret_5', 'SPY_ret_20', 'SPY_dist_sma50', 'VIX_change_5', 'VIX_rank_20', 'TLT_ret_10', 'USO_ret_5', 'SPY_TLT_ratio_ret']\n",
|
|
"Target column: spy_up_5d\n",
|
|
"Training matrix shape: (842, 8)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>SPY_ret_5</th>\n",
|
|
" <th>SPY_ret_20</th>\n",
|
|
" <th>SPY_dist_sma50</th>\n",
|
|
" <th>VIX_change_5</th>\n",
|
|
" <th>VIX_rank_20</th>\n",
|
|
" <th>TLT_ret_10</th>\n",
|
|
" <th>USO_ret_5</th>\n",
|
|
" <th>SPY_TLT_ratio_ret</th>\n",
|
|
" <th>spy_up_5d</th>\n",
|
|
" <th>split</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>count</th>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204.000000</td>\n",
|
|
" <td>1204</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>unique</th>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>3</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>top</th>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>train</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>freq</th>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>842</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>mean</th>\n",
|
|
" <td>0.002464</td>\n",
|
|
" <td>0.009816</td>\n",
|
|
" <td>0.010973</td>\n",
|
|
" <td>0.017820</td>\n",
|
|
" <td>0.404506</td>\n",
|
|
" <td>-0.004103</td>\n",
|
|
" <td>0.004938</td>\n",
|
|
" <td>0.004838</td>\n",
|
|
" <td>0.589701</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>std</th>\n",
|
|
" <td>0.023044</td>\n",
|
|
" <td>0.043515</td>\n",
|
|
" <td>0.037287</td>\n",
|
|
" <td>0.299050</td>\n",
|
|
" <td>0.337773</td>\n",
|
|
" <td>0.028469</td>\n",
|
|
" <td>0.052606</td>\n",
|
|
" <td>0.027586</td>\n",
|
|
" <td>0.492092</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>min</th>\n",
|
|
" <td>-0.114962</td>\n",
|
|
" <td>-0.123975</td>\n",
|
|
" <td>-0.141995</td>\n",
|
|
" <td>-0.387709</td>\n",
|
|
" <td>0.050000</td>\n",
|
|
" <td>-0.092880</td>\n",
|
|
" <td>-0.196652</td>\n",
|
|
" <td>-0.117208</td>\n",
|
|
" <td>0.000000</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>25%</th>\n",
|
|
" <td>-0.009749</td>\n",
|
|
" <td>-0.016557</td>\n",
|
|
" <td>-0.008704</td>\n",
|
|
" <td>-0.056606</td>\n",
|
|
" <td>0.100000</td>\n",
|
|
" <td>-0.023139</td>\n",
|
|
" <td>-0.025950</td>\n",
|
|
" <td>-0.010029</td>\n",
|
|
" <td>0.000000</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>50%</th>\n",
|
|
" <td>0.003749</td>\n",
|
|
" <td>0.015639</td>\n",
|
|
" <td>0.017755</td>\n",
|
|
" <td>-0.018430</td>\n",
|
|
" <td>0.300000</td>\n",
|
|
" <td>-0.004429</td>\n",
|
|
" <td>0.004869</td>\n",
|
|
" <td>0.005508</td>\n",
|
|
" <td>1.000000</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>75%</th>\n",
|
|
" <td>0.015950</td>\n",
|
|
" <td>0.038167</td>\n",
|
|
" <td>0.038124</td>\n",
|
|
" <td>0.030815</td>\n",
|
|
" <td>0.750000</td>\n",
|
|
" <td>0.014188</td>\n",
|
|
" <td>0.032295</td>\n",
|
|
" <td>0.021358</td>\n",
|
|
" <td>1.000000</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>max</th>\n",
|
|
" <td>0.082843</td>\n",
|
|
" <td>0.157566</td>\n",
|
|
" <td>0.088105</td>\n",
|
|
" <td>3.846154</td>\n",
|
|
" <td>1.000000</td>\n",
|
|
" <td>0.091322</td>\n",
|
|
" <td>0.327273</td>\n",
|
|
" <td>0.129204</td>\n",
|
|
" <td>1.000000</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
|
"count 1204.000000 1204.000000 1204.000000 1204.000000 1204.000000 \n",
|
|
"unique NaN NaN NaN NaN NaN \n",
|
|
"top NaN NaN NaN NaN NaN \n",
|
|
"freq NaN NaN NaN NaN NaN \n",
|
|
"mean 0.002464 0.009816 0.010973 0.017820 0.404506 \n",
|
|
"std 0.023044 0.043515 0.037287 0.299050 0.337773 \n",
|
|
"min -0.114962 -0.123975 -0.141995 -0.387709 0.050000 \n",
|
|
"25% -0.009749 -0.016557 -0.008704 -0.056606 0.100000 \n",
|
|
"50% 0.003749 0.015639 0.017755 -0.018430 0.300000 \n",
|
|
"75% 0.015950 0.038167 0.038124 0.030815 0.750000 \n",
|
|
"max 0.082843 0.157566 0.088105 3.846154 1.000000 \n",
|
|
"\n",
|
|
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
|
|
"count 1204.000000 1204.000000 1204.000000 1204.000000 1204 \n",
|
|
"unique NaN NaN NaN NaN 3 \n",
|
|
"top NaN NaN NaN NaN train \n",
|
|
"freq NaN NaN NaN NaN 842 \n",
|
|
"mean -0.004103 0.004938 0.004838 0.589701 NaN \n",
|
|
"std 0.028469 0.052606 0.027586 0.492092 NaN \n",
|
|
"min -0.092880 -0.196652 -0.117208 0.000000 NaN \n",
|
|
"25% -0.023139 -0.025950 -0.010029 0.000000 NaN \n",
|
|
"50% -0.004429 0.004869 0.005508 1.000000 NaN \n",
|
|
"75% 0.014188 0.032295 0.021358 1.000000 NaN \n",
|
|
"max 0.091322 0.327273 0.129204 1.000000 NaN "
|
|
]
|
|
},
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"model_input_columns = FEATURE_COLUMNS\n",
|
|
"model_target_column = TARGET_COLUMN\n",
|
|
"\n",
|
|
"X_train = df_model.loc[df_model[\"split\"] == \"train\", model_input_columns]\n",
|
|
"y_train = df_model.loc[df_model[\"split\"] == \"train\", model_target_column]\n",
|
|
"\n",
|
|
"print(f\"Rows: {len(df_model):,}\")\n",
|
|
"print(f\"Feature columns: {model_input_columns}\")\n",
|
|
"print(f\"Target column: {model_target_column}\")\n",
|
|
"print(f\"Training matrix shape: {X_train.shape}\")\n",
|
|
"\n",
|
|
"df_model.describe(include=\"all\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"(PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'),\n",
|
|
" (1204, 11),\n",
|
|
" date SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 \\\n",
|
|
" 0 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n",
|
|
" 1 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n",
|
|
" 2 2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 \n",
|
|
" 3 2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 \n",
|
|
" 4 2021-10-15 0.018294 0.010127 0.007213 -0.079882 \n",
|
|
" \n",
|
|
" VIX_rank_20 TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
|
|
" 0 0.20 -0.033135 0.031015 0.038908 1.0 train \n",
|
|
" 1 0.10 -0.001041 0.008628 -0.001303 1.0 train \n",
|
|
" 2 0.05 0.006928 0.036928 -0.005897 1.0 train \n",
|
|
" 3 0.05 0.010809 0.026192 -0.011991 1.0 train \n",
|
|
" 4 0.05 -0.002202 0.030287 -0.003823 1.0 train )"
|
|
]
|
|
},
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)\n",
|
|
"dataset_to_save = df_model.reset_index()\n",
|
|
"dataset_to_save.to_parquet(OUTPUT_PATH, index=False)\n",
|
|
"\n",
|
|
"OUTPUT_PATH, dataset_to_save.shape, dataset_to_save.head()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.15"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|