jarno/trading-bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prediction and model training work

Browse Source
This commit is contained in:
jarno
2026-08-06 17:22:57 +03:00
parent fddcc9190a
commit 465e09fc82
13 changed files with 1689 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/spy_direction_dataset.ipynb
+151 -151
View File
@@ -20,14 +20,14 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(PosixPath('/home/jarno/repos/trading-bot'),\n",
" PosixPath('/home/jarno/repos/trading-bot/data/ibkr/daily'),\n",
" PosixPath('/home/jarno/repos/trading-bot/data/alpaca/daily'),\n",
" PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'))"
]
},
@@ -87,16 +87,16 @@
"output_type": "stream",
"text": [
"<class 'pandas.DataFrame'>\n",
"DatetimeIndex: 1251 entries, 2021-07-30 to 2026-07-24\n",
"DatetimeIndex: 1255 entries, 2021-08-02 to 2026-07-31\n",
"Data columns (total 4 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 SPY_close 1251 non-null float64\n",
" 1 VIX_close 1251 non-null float64\n",
" 2 TLT_close 1251 non-null float64\n",
" 3 USO_close 1251 non-null float64\n",
" 0 SPY_close 1255 non-null float64\n",
" 1 VIX_close 1255 non-null float64\n",
" 2 TLT_close 1255 non-null float64\n",
" 3 USO_close 1255 non-null float64\n",
"dtypes: float64(4)\n",
"memory usage: 48.9 KB\n"
"memory usage: 49.0 KB\n"
]
},
{
@@ -135,40 +135,40 @@
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2021-07-30</th>\n",
" <td>438.51</td>\n",
" <td>495.4</td>\n",
" <td>149.52</td>\n",
" <td>50.66</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-08-02</th>\n",
" <td>437.59</td>\n",
" <td>513.6</td>\n",
" <td>25.68</td>\n",
" <td>150.67</td>\n",
" <td>49.18</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-08-03</th>\n",
" <td>441.15</td>\n",
" <td>488.0</td>\n",
" <td>24.40</td>\n",
" <td>150.75</td>\n",
" <td>48.85</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-08-04</th>\n",
" <td>438.98</td>\n",
" <td>487.6</td>\n",
" <td>24.38</td>\n",
" <td>151.06</td>\n",
" <td>47.20</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-08-05</th>\n",
" <td>441.76</td>\n",
" <td>474.8</td>\n",
" <td>23.74</td>\n",
" <td>150.29</td>\n",
" <td>48.10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-08-06</th>\n",
" <td>442.49</td>\n",
" <td>23.14</td>\n",
" <td>147.78</td>\n",
" <td>47.57</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
@@ -176,11 +176,11 @@
"text/plain": [
" SPY_close VIX_close TLT_close USO_close\n",
"date \n",
"2021-07-30 438.51 495.4 149.52 50.66\n",
"2021-08-02 437.59 513.6 150.67 49.18\n",
"2021-08-03 441.15 488.0 150.75 48.85\n",
"2021-08-04 438.98 487.6 151.06 47.20\n",
"2021-08-05 441.76 474.8 150.29 48.10"
"2021-08-02 437.59 25.68 150.67 49.18\n",
"2021-08-03 441.15 24.40 150.75 48.85\n",
"2021-08-04 438.98 24.38 151.06 47.20\n",
"2021-08-05 441.76 23.74 150.29 48.10\n",
"2021-08-06 442.49 23.14 147.78 47.57"
]
},
"execution_count": 2,
@@ -234,7 +234,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -283,24 +283,12 @@
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2021-10-08</th>\n",
" <td>0.008336</td>\n",
" <td>-0.017017</td>\n",
" <td>-0.011155</td>\n",
" <td>-0.075239</td>\n",
" <td>0.125</td>\n",
" <td>-0.034239</td>\n",
" <td>0.041495</td>\n",
" <td>0.032998</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-10-11</th>\n",
" <td>0.014114</td>\n",
" <td>-0.026625</td>\n",
" <td>-0.018145</td>\n",
" <td>-0.090869</td>\n",
" <td>0.200</td>\n",
" <td>0.20</td>\n",
" <td>-0.033135</td>\n",
" <td>0.031015</td>\n",
" <td>0.038908</td>\n",
@@ -312,7 +300,7 @@
" <td>-0.023752</td>\n",
" <td>-0.020386</td>\n",
" <td>-0.074091</td>\n",
" <td>0.100</td>\n",
" <td>0.10</td>\n",
" <td>-0.001041</td>\n",
" <td>0.008628</td>\n",
" <td>-0.001303</td>\n",
@@ -324,7 +312,7 @@
" <td>-0.028356</td>\n",
" <td>-0.016597</td>\n",
" <td>-0.081006</td>\n",
" <td>0.050</td>\n",
" <td>0.05</td>\n",
" <td>0.006928</td>\n",
" <td>0.036928</td>\n",
" <td>-0.005897</td>\n",
@@ -336,12 +324,24 @@
" <td>-0.010443</td>\n",
" <td>-0.000214</td>\n",
" <td>-0.096759</td>\n",
" <td>0.050</td>\n",
" <td>0.05</td>\n",
" <td>0.010809</td>\n",
" <td>0.026192</td>\n",
" <td>-0.011991</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2021-10-15</th>\n",
" <td>0.018294</td>\n",
" <td>0.010127</td>\n",
" <td>0.007213</td>\n",
" <td>-0.079882</td>\n",
" <td>0.05</td>\n",
" <td>-0.002202</td>\n",
" <td>0.030287</td>\n",
" <td>-0.003823</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
@@ -349,22 +349,22 @@
"text/plain": [
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
"date \n",
"2021-10-08 0.008336 -0.017017 -0.011155 -0.075239 0.125 \n",
"2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 0.200 \n",
"2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 0.100 \n",
"2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 0.050 \n",
"2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 0.050 \n",
"2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 0.20 \n",
"2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 0.10 \n",
"2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 0.05 \n",
"2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 0.05 \n",
"2021-10-15 0.018294 0.010127 0.007213 -0.079882 0.05 \n",
"\n",
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d \n",
"date \n",
"2021-10-08 -0.034239 0.041495 0.032998 1.0 \n",
"2021-10-11 -0.033135 0.031015 0.038908 1.0 \n",
"2021-10-12 -0.001041 0.008628 -0.001303 1.0 \n",
"2021-10-13 0.006928 0.036928 -0.005897 1.0 \n",
"2021-10-14 0.010809 0.026192 -0.011991 1.0 "
"2021-10-14 0.010809 0.026192 -0.011991 1.0 \n",
"2021-10-15 -0.002202 0.030287 -0.003823 1.0 "
]
},
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@@ -384,7 +384,7 @@
"df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
"df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
"df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
"df_model\n",
"\n",
"spy_forward_close = df[\"SPY_close\"].shift(-5)\n",
"df[\"spy_up_5d\"] = np.nan\n",
"df.loc[spy_forward_close > df[\"SPY_close\"], \"spy_up_5d\"] = 1.0\n",
@@ -418,7 +418,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -458,24 +458,24 @@
" <tbody>\n",
" <tr>\n",
" <th>train</th>\n",
" <td>837</td>\n",
" <td>2021-10-08</td>\n",
" <td>2025-02-07</td>\n",
" <td>0.583035</td>\n",
" <td>840</td>\n",
" <td>2021-10-11</td>\n",
" <td>2025-02-13</td>\n",
" <td>0.583333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>validation</th>\n",
" <td>179</td>\n",
" <td>2025-02-10</td>\n",
" <td>2025-10-24</td>\n",
" <td>0.653631</td>\n",
" <td>180</td>\n",
" <td>2025-02-14</td>\n",
" <td>2025-10-31</td>\n",
" <td>0.633333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>test</th>\n",
" <td>181</td>\n",
" <td>2025-10-27</td>\n",
" <td>2026-07-17</td>\n",
" <td>0.558011</td>\n",
" <td>2025-11-03</td>\n",
" <td>2026-07-24</td>\n",
" <td>0.569061</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@@ -484,12 +484,12 @@
"text/plain": [
" rows start_date end_date target_mean\n",
"split \n",
"train 837 2021-10-08 2025-02-07 0.583035\n",
"validation 179 2025-02-10 2025-10-24 0.653631\n",
"test 181 2025-10-27 2026-07-17 0.558011"
"train 840 2021-10-11 2025-02-13 0.583333\n",
"validation 180 2025-02-14 2025-10-31 0.633333\n",
"test 181 2025-11-03 2026-07-24 0.569061"
]
},
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@@ -543,17 +543,17 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Rows: 1,197\n",
"Rows: 1,201\n",
"Feature columns: ['SPY_ret_5', 'SPY_ret_20', 'SPY_dist_sma50', 'VIX_change_5', 'VIX_rank_20', 'TLT_ret_10', 'USO_ret_5', 'SPY_TLT_ratio_ret']\n",
"Target column: spy_up_5d\n",
"Training matrix shape: (837, 8)\n"
"Training matrix shape: (840, 8)\n"
]
},
{
@@ -592,16 +592,16 @@
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197.000000</td>\n",
" <td>1197</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201.000000</td>\n",
" <td>1201</td>\n",
" </tr>\n",
" <tr>\n",
" <th>unique</th>\n",
@@ -640,32 +640,32 @@
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>837</td>\n",
" <td>840</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>0.002556</td>\n",
" <td>0.009841</td>\n",
" <td>0.011067</td>\n",
" <td>-0.008646</td>\n",
" <td>0.387009</td>\n",
" <td>-0.004083</td>\n",
" <td>0.004662</td>\n",
" <td>0.004927</td>\n",
" <td>0.589808</td>\n",
" <td>0.002502</td>\n",
" <td>0.009848</td>\n",
" <td>0.011028</td>\n",
" <td>0.017763</td>\n",
" <td>0.403476</td>\n",
" <td>-0.004099</td>\n",
" <td>0.005024</td>\n",
" <td>0.004880</td>\n",
" <td>0.588676</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>0.023081</td>\n",
" <td>0.043634</td>\n",
" <td>0.037371</td>\n",
" <td>0.092413</td>\n",
" <td>0.333697</td>\n",
" <td>0.028557</td>\n",
" <td>0.052319</td>\n",
" <td>0.027656</td>\n",
" <td>0.492074</td>\n",
" <td>0.023057</td>\n",
" <td>0.043558</td>\n",
" <td>0.037315</td>\n",
" <td>0.299417</td>\n",
" <td>0.337501</td>\n",
" <td>0.028502</td>\n",
" <td>0.052628</td>\n",
" <td>0.027606</td>\n",
" <td>0.492279</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
@@ -683,40 +683,40 @@
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>-0.009535</td>\n",
" <td>-0.017017</td>\n",
" <td>-0.008847</td>\n",
" <td>-0.058376</td>\n",
" <td>0.075000</td>\n",
" <td>-0.023517</td>\n",
" <td>-0.026200</td>\n",
" <td>-0.009601</td>\n",
" <td>-0.016509</td>\n",
" <td>-0.008656</td>\n",
" <td>-0.056615</td>\n",
" <td>0.100000</td>\n",
" <td>-0.023304</td>\n",
" <td>-0.025866</td>\n",
" <td>-0.010013</td>\n",
" <td>0.000000</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>0.003869</td>\n",
" <td>0.015797</td>\n",
" <td>0.017906</td>\n",
" <td>-0.020655</td>\n",
" <td>0.250000</td>\n",
" <td>-0.004339</td>\n",
" <td>0.004817</td>\n",
" <td>0.005710</td>\n",
" <td>0.003798</td>\n",
" <td>0.015725</td>\n",
" <td>0.017802</td>\n",
" <td>-0.018570</td>\n",
" <td>0.300000</td>\n",
" <td>-0.004435</td>\n",
" <td>0.004950</td>\n",
" <td>0.005537</td>\n",
" <td>1.000000</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>0.015957</td>\n",
" <td>0.038385</td>\n",
" <td>0.038332</td>\n",
" <td>0.027778</td>\n",
" <td>0.700000</td>\n",
" <td>0.014272</td>\n",
" <td>0.031883</td>\n",
" <td>0.021406</td>\n",
" <td>0.015955</td>\n",
" <td>0.038179</td>\n",
" <td>0.038282</td>\n",
" <td>0.030814</td>\n",
" <td>0.750000</td>\n",
" <td>0.014227</td>\n",
" <td>0.032486</td>\n",
" <td>0.021390</td>\n",
" <td>1.000000</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
@@ -724,8 +724,8 @@
" <th>max</th>\n",
" <td>0.082843</td>\n",
" <td>0.157566</td>\n",
" <td>0.088103</td>\n",
" <td>0.937828</td>\n",
" <td>0.088105</td>\n",
" <td>3.846154</td>\n",
" <td>1.000000</td>\n",
" <td>0.091322</td>\n",
" <td>0.327273</td>\n",
@@ -739,33 +739,33 @@
],
"text/plain": [
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
"count 1197.000000 1197.000000 1197.000000 1197.000000 1197.000000 \n",
"count 1201.000000 1201.000000 1201.000000 1201.000000 1201.000000 \n",
"unique NaN NaN NaN NaN NaN \n",
"top NaN NaN NaN NaN NaN \n",
"freq NaN NaN NaN NaN NaN \n",
"mean 0.002556 0.009841 0.011067 -0.008646 0.387009 \n",
"std 0.023081 0.043634 0.037371 0.092413 0.333697 \n",
"mean 0.002502 0.009848 0.011028 0.017763 0.403476 \n",
"std 0.023057 0.043558 0.037315 0.299417 0.337501 \n",
"min -0.114962 -0.123975 -0.141995 -0.387709 0.050000 \n",
"25% -0.009535 -0.017017 -0.008847 -0.058376 0.075000 \n",
"50% 0.003869 0.015797 0.017906 -0.020655 0.250000 \n",
"75% 0.015957 0.038385 0.038332 0.027778 0.700000 \n",
"max 0.082843 0.157566 0.088103 0.937828 1.000000 \n",
"25% -0.009601 -0.016509 -0.008656 -0.056615 0.100000 \n",
"50% 0.003798 0.015725 0.017802 -0.018570 0.300000 \n",
"75% 0.015955 0.038179 0.038282 0.030814 0.750000 \n",
"max 0.082843 0.157566 0.088105 3.846154 1.000000 \n",
"\n",
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
"count 1197.000000 1197.000000 1197.000000 1197.000000 1197 \n",
"count 1201.000000 1201.000000 1201.000000 1201.000000 1201 \n",
"unique NaN NaN NaN NaN 3 \n",
"top NaN NaN NaN NaN train \n",
"freq NaN NaN NaN NaN 837 \n",
"mean -0.004083 0.004662 0.004927 0.589808 NaN \n",
"std 0.028557 0.052319 0.027656 0.492074 NaN \n",
"freq NaN NaN NaN NaN 840 \n",
"mean -0.004099 0.005024 0.004880 0.588676 NaN \n",
"std 0.028502 0.052628 0.027606 0.492279 NaN \n",
"min -0.092880 -0.196652 -0.117208 0.000000 NaN \n",
"25% -0.023517 -0.026200 -0.010013 0.000000 NaN \n",
"50% -0.004339 0.004817 0.005710 1.000000 NaN \n",
"75% 0.014272 0.031883 0.021406 1.000000 NaN \n",
"25% -0.023304 -0.025866 -0.010013 0.000000 NaN \n",
"50% -0.004435 0.004950 0.005537 1.000000 NaN \n",
"75% 0.014227 0.032486 0.021390 1.000000 NaN \n",
"max 0.091322 0.327273 0.129204 1.000000 NaN "
]
},
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -787,30 +787,30 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'),\n",
" (1197, 11),\n",
" (1201, 11),\n",
" date SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 \\\n",
" 0 2021-10-08 0.008336 -0.017017 -0.011155 -0.075239 \n",
" 1 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n",
" 2 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n",
" 3 2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 \n",
" 4 2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 \n",
" 0 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n",
" 1 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n",
" 2 2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 \n",
" 3 2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 \n",
" 4 2021-10-15 0.018294 0.010127 0.007213 -0.079882 \n",
" \n",
" VIX_rank_20 TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
" 0 0.125 -0.034239 0.041495 0.032998 1.0 train \n",
" 1 0.200 -0.033135 0.031015 0.038908 1.0 train \n",
" 2 0.100 -0.001041 0.008628 -0.001303 1.0 train \n",
" 3 0.050 0.006928 0.036928 -0.005897 1.0 train \n",
" 4 0.050 0.010809 0.026192 -0.011991 1.0 train )"
" 0 0.20 -0.033135 0.031015 0.038908 1.0 train \n",
" 1 0.10 -0.001041 0.008628 -0.001303 1.0 train \n",
" 2 0.05 0.006928 0.036928 -0.005897 1.0 train \n",
" 3 0.05 0.010809 0.026192 -0.011991 1.0 train \n",
" 4 0.05 -0.002202 0.030287 -0.003823 1.0 train )"
]
},
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}