Prediction and model training work
This commit is contained in:
jarno committed
1 parent
fddcc9190a
commit
465e09fc82
13 files changed
+1689
-403
No files matched your search
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"symbols": {
|
||||
"SPY": "SPY",
|
||||
"VIX": "VIXY",
|
||||
"TLT": "TLT",
|
||||
"USO": "USO"
|
||||
},
|
||||
"fractions": {
|
||||
"train": 0.7,
|
||||
"validation": 0.15,
|
||||
"test": 0.15
|
||||
},
|
||||
"model": {
|
||||
"n_estimators": 300,
|
||||
"max_depth": 1,
|
||||
"learning_rate": 0.01,
|
||||
"subsample": 0.7,
|
||||
"colsample_bytree": 0.7,
|
||||
"early_stopping_rounds": 20,
|
||||
"random_state": 42,
|
||||
"eval_metric": "logloss"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "c8a08105",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from dotenv import load_dotenv\n",
|
||||
"from alpaca.trading.client import TradingClient\n",
|
||||
"\n",
|
||||
"load_dotenv()\n",
|
||||
"\n",
|
||||
"# Set paper=True for paper trading (sandbox), paper=False for live trading\n",
|
||||
"trading_client = TradingClient(\n",
|
||||
" api_key=os.getenv(\"ALPACA_API_KEY\"),\n",
|
||||
" secret_key=os.getenv(\"ALPACA_SECRET_KEY\"),\n",
|
||||
" paper=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "49b14380",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"No open positions.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"positions = trading_client.get_all_positions()\n",
|
||||
"\n",
|
||||
"if not positions:\n",
|
||||
" print(\"No open positions.\")\n",
|
||||
"else:\n",
|
||||
" print(\"Current Portfolio Positions:\")\n",
|
||||
" for pos in positions:\n",
|
||||
" print(\n",
|
||||
" f\"Symbol: {pos.symbol:<5} | \"\n",
|
||||
" f\"Qty: {pos.qty:<5} | \"\n",
|
||||
" f\"Avg Entry Price: ${float(pos.avg_entry_price):.2f} | \"\n",
|
||||
" f\"Current Price: ${float(pos.current_price):.2f} | \"\n",
|
||||
" f\"Unrealized P/L: ${float(pos.unrealized_pl):.2f}\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "c032239c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"99997.95 99997.95\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"account = trading_client.get_account()\n",
|
||||
"print(\n",
|
||||
" account.cash,\n",
|
||||
" account.equity\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "c759e40f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"No open orders found.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from alpaca.trading.requests import GetOrdersRequest\n",
|
||||
"from alpaca.trading.enums import QueryOrderStatus\n",
|
||||
"\n",
|
||||
"# Request only open orders\n",
|
||||
"request_params = GetOrdersRequest(status=QueryOrderStatus.OPEN)\n",
|
||||
"open_orders = trading_client.get_orders(filter=request_params)\n",
|
||||
"\n",
|
||||
"if not open_orders:\n",
|
||||
" print(\"No open orders found.\")\n",
|
||||
"else:\n",
|
||||
" print(f\"Found {len(open_orders)} open order(s):\")\n",
|
||||
" for order in open_orders:\n",
|
||||
" print(\n",
|
||||
" f\"ID: {order.id} | Symbol: {order.symbol} | \"\n",
|
||||
" f\"Side: {order.side} | Qty: {order.qty} | Status: {order.status}\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "091cf061",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Submitted Order ID: 4838c16d-7a12-4dc3-80cb-7c5d1dbecda3 | Status: OrderStatus.ACCEPTED\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from alpaca.trading.requests import MarketOrderRequest\n",
|
||||
"from alpaca.trading.enums import OrderSide, TimeInForce\n",
|
||||
"\n",
|
||||
"# Define a market buy order for 10 shares of SPY\n",
|
||||
"market_order_data = MarketOrderRequest(\n",
|
||||
" symbol=\"SPY\",\n",
|
||||
" notional=100.0,\n",
|
||||
" side=OrderSide.BUY,\n",
|
||||
" time_in_force=TimeInForce.DAY,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Submit the order\n",
|
||||
"order = trading_client.submit_order(order_data=market_order_data)\n",
|
||||
"\n",
|
||||
"print(f\"Submitted Order ID: {order.id} | Status: {order.status}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "038bf2ce",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# Source of truth for model inputs (must match training order exactly)\n",
|
||||
"FEATURE_COLUMNS = [\n",
|
||||
" \"SPY_ret_5\",\n",
|
||||
" \"SPY_ret_20\",\n",
|
||||
" \"SPY_dist_sma50\",\n",
|
||||
" \"VIX_change_5\",\n",
|
||||
" \"VIX_rank_20\",\n",
|
||||
" \"TLT_ret_10\",\n",
|
||||
" \"USO_ret_5\",\n",
|
||||
" \"SPY_TLT_ratio_ret\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"DEFAULT_SYMBOLS = {\n",
|
||||
" \"SPY\": \"SPY\",\n",
|
||||
" \"VIXY\": \"VIX\", # Change to \"VIX\": \"VIX\" if using raw VIX Parquet\n",
|
||||
" \"TLT\": \"TLT\",\n",
|
||||
" \"USO\": \"USO\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_raw_close_prices(\n",
|
||||
" raw_data_dir: Path, symbols: dict[str, str]\n",
|
||||
") -> pd.DataFrame:\n",
|
||||
" \"\"\"Reads raw Parquet files and merges close prices into a single inner-joined DataFrame.\"\"\"\n",
|
||||
" frames = []\n",
|
||||
" for symbol, alias in symbols.items():\n",
|
||||
" path = raw_data_dir / f\"{symbol}.parquet\"\n",
|
||||
" if not path.exists():\n",
|
||||
" raise FileNotFoundError(f\"Missing raw data file: {path}\")\n",
|
||||
"\n",
|
||||
" frame = pd.read_parquet(path)\n",
|
||||
" if \"date\" in frame.columns:\n",
|
||||
" frame = frame.set_index(\"date\")\n",
|
||||
"\n",
|
||||
" frame.index = pd.to_datetime(frame.index)\n",
|
||||
" frame = frame.sort_index()\n",
|
||||
" frames.append(frame[[\"close\"]].rename(columns={\"close\": f\"{alias}_close\"}))\n",
|
||||
"\n",
|
||||
" return pd.concat(frames, axis=1, join=\"inner\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def compute_features(prices: pd.DataFrame) -> pd.DataFrame:\n",
|
||||
" \"\"\"Computes engineered features from raw merged price history.\"\"\"\n",
|
||||
" df = prices.copy()\n",
|
||||
"\n",
|
||||
" # Target asset features\n",
|
||||
" df[\"SPY_ret_5\"] = df[\"SPY_close\"].pct_change(5)\n",
|
||||
" df[\"SPY_ret_20\"] = df[\"SPY_close\"].pct_change(20)\n",
|
||||
"\n",
|
||||
" sma_50 = df[\"SPY_close\"].rolling(50).mean()\n",
|
||||
" df[\"SPY_dist_sma50\"] = (df[\"SPY_close\"] - sma_50) / sma_50\n",
|
||||
"\n",
|
||||
" # Volatility / Market Stress\n",
|
||||
" df[\"VIX_change_5\"] = df[\"VIX_close\"].pct_change(5)\n",
|
||||
" df[\"VIX_rank_20\"] = df[\"VIX_close\"].rolling(20).rank(pct=True)\n",
|
||||
"\n",
|
||||
" # Macro & Relative ratios\n",
|
||||
" df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
|
||||
" df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
|
||||
" df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
|
||||
"\n",
|
||||
" return df[FEATURE_COLUMNS]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_latest_inference_features(\n",
|
||||
" raw_data_dir: Path,\n",
|
||||
" symbols: dict[str, str] | None = None,\n",
|
||||
" max_age_days: int = 1,\n",
|
||||
") -> pd.DataFrame:\n",
|
||||
" \"\"\"Loads raw prices, computes features, verifies date freshness,\n",
|
||||
"\n",
|
||||
" and returns the latest single row for model prediction.\n",
|
||||
" \"\"\"\n",
|
||||
" symbols = symbols or DEFAULT_SYMBOLS\n",
|
||||
"\n",
|
||||
" # 1. Load prices & compute rolling features\n",
|
||||
" prices = load_raw_close_prices(raw_data_dir, symbols)\n",
|
||||
" features = compute_features(prices).dropna()\n",
|
||||
"\n",
|
||||
" if features.empty:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Not enough historical rows to compute 50-day rolling window features.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # 2. Extract latest available row as a 1-row DataFrame\n",
|
||||
" latest_row = features.iloc[[-1]]\n",
|
||||
" latest_date = latest_row.index[0]\n",
|
||||
"\n",
|
||||
" # 3. Check data freshness and raise a warning if stale\n",
|
||||
" now = pd.Timestamp.now()\n",
|
||||
" latest_date_naive = (\n",
|
||||
" latest_date.tz_localize(None)\n",
|
||||
" if latest_date.tz is not None\n",
|
||||
" else latest_date\n",
|
||||
" )\n",
|
||||
" days_old = (now.floor(\"D\") - latest_date_naive.floor(\"D\")).days\n",
|
||||
"\n",
|
||||
" if days_old > max_age_days:\n",
|
||||
" warnings.warn(\n",
|
||||
" f\"STALE DATA WARNING: Latest feature row is from {latest_date.strftime('%Y-%m-%d')} \"\n",
|
||||
" f\"({days_old} day(s) old). Update raw Parquet files before executing trades.\",\n",
|
||||
" UserWarning,\n",
|
||||
" stacklevel=2,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" return latest_row\n",
|
||||
"\n",
|
||||
"def get_target_exposure(\n",
|
||||
" p_pred: float, p_base: float, sensitivity: float = 5.0\n",
|
||||
") -> float:\n",
|
||||
" \"\"\"Maps predicted probability to a target portfolio equity allocation (0.0 to 1.0).\n",
|
||||
"\n",
|
||||
" - p_pred == p_base --> 50% Target Exposure (Neutral)\n",
|
||||
" - p_pred > p_base --> Scale up toward 100% (Bullish)\n",
|
||||
" - p_pred < p_base --> Scale down toward 0% (Bearish / Cash)\n",
|
||||
" \"\"\"\n",
|
||||
" # Calculate deviation from the historical average\n",
|
||||
" delta = p_pred - p_base\n",
|
||||
"\n",
|
||||
" # Base target allocation is 50% equity / 50% cash\n",
|
||||
" base_allocation = 0.50\n",
|
||||
"\n",
|
||||
" # Sensitivity controls how aggressively probability changes alter allocation\n",
|
||||
" # e.g., a +0.08 delta * 5.0 = +0.40 -> 90% Equity Allocation\n",
|
||||
" target_allocation = base_allocation + (delta * sensitivity)\n",
|
||||
"\n",
|
||||
" # Clamp bounds strictly between 0% (full cash) and 100% (full SPY)\n",
|
||||
" return float(np.clip(target_allocation, 0.0, 1.0))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "0b3d2c25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Date: 2026-07-27 | Prob: 0.5843 | Base: 0.5830\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"0.5064256139268726"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import xgboost as xgb\n",
|
||||
"\n",
|
||||
"# 1. Load model and metadata\n",
|
||||
"model = xgb.XGBClassifier()\n",
|
||||
"model.load_model(\"models/spy_xgb_v1.json\")\n",
|
||||
"\n",
|
||||
"with open(\"models/spy_xgb_v1_meta.json\", \"r\") as f:\n",
|
||||
" meta = json.load(f)\n",
|
||||
"\n",
|
||||
"# 2. Fetch latest features from raw parquet files\n",
|
||||
"RAW_DATA_DIR = Path(\"../data/ibkr/daily\")\n",
|
||||
"X_latest = get_latest_inference_features(RAW_DATA_DIR, max_age_days=1)\n",
|
||||
"\n",
|
||||
"# 3. Predict probability\n",
|
||||
"p_pred = float(model.predict_proba(X_latest[meta[\"feature_cols\"]])[0, 1])\n",
|
||||
"p_base = meta[\"p_base\"]\n",
|
||||
"\n",
|
||||
"print(\n",
|
||||
" f\"Date: {X_latest.index[0].date()} | Prob: {p_pred:.4f} | Base: {p_base:.4f}\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"get_target_exposure(p_pred, p_base, sensitivity=5.0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3f823fa6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>SPY_ret_5</th>\n",
|
||||
" <th>SPY_ret_20</th>\n",
|
||||
" <th>SPY_dist_sma50</th>\n",
|
||||
" <th>VIX_change_5</th>\n",
|
||||
" <th>VIX_rank_20</th>\n",
|
||||
" <th>TLT_ret_10</th>\n",
|
||||
" <th>USO_ret_5</th>\n",
|
||||
" <th>SPY_TLT_ratio_ret</th>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>date</th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" <th></th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>2026-07-27</th>\n",
|
||||
" <td>-0.004043</td>\n",
|
||||
" <td>0.013855</td>\n",
|
||||
" <td>-0.007938</td>\n",
|
||||
" <td>0.007065</td>\n",
|
||||
" <td>0.75</td>\n",
|
||||
" <td>-0.00262</td>\n",
|
||||
" <td>-0.005976</td>\n",
|
||||
" <td>-0.002378</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
||||
"date \n",
|
||||
"2026-07-27 -0.004043 0.013855 -0.007938 0.007065 0.75 \n",
|
||||
"\n",
|
||||
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret \n",
|
||||
"date \n",
|
||||
"2026-07-27 -0.00262 -0.005976 -0.002378 "
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"X_latest"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "476fd7fd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
+141
-20
@@ -17,7 +17,26 @@
|
||||
"execution_count": null,
|
||||
"id": "543426a2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Connecting to IBKR Gateway at 127.0.0.1:4002 with client id 101...\n",
|
||||
"Failed to connect to IBKR Gateway: This event loop is already running\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Error 200, reqId 9: No security definition has been found for the request, contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n",
|
||||
"Error 200, reqId 10: No security definition has been found for the request\n",
|
||||
"Canceled order: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')\n",
|
||||
"Peer closed connection.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"from pathlib import Path\n",
|
||||
@@ -35,36 +54,55 @@
|
||||
"if str(src_path) not in sys.path:\n",
|
||||
" sys.path.insert(0, str(src_path))\n",
|
||||
"\n",
|
||||
"from trading_bot.data.fetch_ibkr_daily import (\n",
|
||||
" IBKR_CLIENT_ID,\n",
|
||||
" IBKR_CONNECT_TIMEOUT_SECONDS,\n",
|
||||
" IBKR_HOST,\n",
|
||||
" IBKR_PORT,\n",
|
||||
")\n",
|
||||
"IBKR_HOST = \"127.0.0.1\"\n",
|
||||
"IBKR_PORT = 4002\n",
|
||||
"IBKR_CLIENT_ID = 101\n",
|
||||
"IBKR_CONNECT_TIMEOUT_SECONDS = 10\n",
|
||||
"\n",
|
||||
"ib = IB()\n",
|
||||
"print(f\"Connecting to IBKR Gateway at {IBKR_HOST}:{IBKR_PORT} with client id {IBKR_CLIENT_ID}...\")\n",
|
||||
"ib.connect(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID, timeout=IBKR_CONNECT_TIMEOUT_SECONDS)\n",
|
||||
"print(\"Connection established.\")\n"
|
||||
"try:\n",
|
||||
" ib.connect(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID, timeout=IBKR_CONNECT_TIMEOUT_SECONDS)\n",
|
||||
" print(\"Connection established.\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"Failed to connect to IBKR Gateway: {e}\")\n",
|
||||
" ib.disconnect()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"id": "ecdc721f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Connected: True\n",
|
||||
"Client ID: 101\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(f\"Connected: {ib.isConnected()}\")\n",
|
||||
"print(f\"Client ID: {ib.clientId}\")\n"
|
||||
"print(f\"Client ID: {ib.client.clientId}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"id": "10ebc240",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"No open portfolio positions were returned.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"portfolio = ib.portfolio()\n",
|
||||
"if not portfolio:\n",
|
||||
@@ -79,27 +117,110 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 23,
|
||||
"id": "fed654ec",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Unknown contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Submitted order for VUAA: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='PendingSubmit', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0)], advancedError='')\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from ib_insync import MarketOrder\n",
|
||||
"\n",
|
||||
"contract = Stock(\"SPY\", \"SMART\", \"USD\")\n",
|
||||
"ib.qualifyContracts(contract)\n",
|
||||
"contract = Stock(\"VUAA\", \"SMART\", \"USD\")\n",
|
||||
"await ib.qualifyContractsAsync(contract)\n",
|
||||
"\n",
|
||||
"# Adjust the quantity as needed before running this cell.\n",
|
||||
"order = MarketOrder(\"BUY\", 1)\n",
|
||||
"order = MarketOrder(\"BUY\", 0.70)\n",
|
||||
"trade = ib.placeOrder(contract, order)\n",
|
||||
"\n",
|
||||
"print(f\"Submitted order for {contract.symbol}: {trade}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 34,
|
||||
"id": "06881101",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trds = ib.trades()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 42,
|
||||
"id": "08663602",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['Cancelled', 'Cancelled', 'Cancelled']"
|
||||
]
|
||||
},
|
||||
"execution_count": 42,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"list(map(lambda x: x.orderStatus.status, trds))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 43,
|
||||
"id": "2c7abf23",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=1578393268, action='BUY', totalQuantity=1.0, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n",
|
||||
" Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=24474350, action='BUY', totalQuantity=0.1345, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n",
|
||||
" Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')]"
|
||||
]
|
||||
},
|
||||
"execution_count": 43,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"trds"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"learner":{"attributes":{"best_iteration":"41","best_score":"0.6593948980172475","scikit_learn":"{\"_estimator_type\": \"classifier\"}"},"feature_names":["VIX_rank_20","TLT_ret_10","USO_ret_5","SPY_TLT_ratio_ret","SPY_ret_5","SPY_ret_20","SPY_dist_sma50"],"feature_types":["float","float","float","float","float","float","float"],"gradient_booster":{"model":{"cats":{"enc":[],"feature_segments":[],"sorted_idx":[]},"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"62"},"iteration_indptr":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62],"tree_info":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"trees":[{"base_weights":[1.1503149E-2,3.602216E-4,-1.4237287E-2],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":0,"left_children":[1,-1,-1],"loss_changes":[5.1499934E0,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[6.892705E-2,3.602216E-4,-1.4237287E-2],"split_indices":[6,0,0],"split_type":[0,0,0],"sum_hessian":[1.4388889E2,1.4243056E2,1.4583333E0],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[2.2381552E-2,2.2380585E-3,-2.9987623E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":1,"left_children":[1,-1,-1],"loss_changes":[9.187705E0,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[1.4205446E-2,2.2380585E-3,-2.9987623E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.3951015E2,8.603824E1,5.3471912E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[9.584328E-2,-1.7863275E-3,6.1051417E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":2,"left_children":[1,-1,-1],"loss_changes":[2.0163263E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[7.100592E-3,-1.7863275E-3,6.1051417E-3],"split_indices":[1,0,0],"split_type":[0,0,0],"sum_hessian":[1.4096178E2,9.2359184E1,4.86026E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[4.7446564E-2,7.0196665E-3,-1.0298543E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":3,"left_children":[1,-1,-1],"loss_changes":[1.4477959E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-1.7963655E-2,7.0196665E-3,-1.0298543E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.4530872E2,2.6470573E1,1.1883814E2],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[6.2338286E-3,3.326677E-3,-1.7981239E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":4,"left_children":[1,-1,-1],"loss_changes":[8.945065E0,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-1.1166078E-2,3.326677E-3,-1.7981239E-3],"split_indices":[2,0,0],"split_type":[0,0,0],"sum_hessian":[1.4530305E2,5.2463432E1,9.283963E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[-3.0314475E-2,6.923221E-3,-1.2672357E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":5,"left_children":[1,-1,-1],"loss_changes":[1.0347677E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-2.858291E-2,6.923221E-3,-1.2672357E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.4626752E2,1.648944E1,1.2977808E2],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[2.5649436E-2,-3.922325E-3,4.0787407E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":6,"left_children":[1,-1,-1],"loss_changes":[2.3413443E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-8.8626295E-3,-3.922325E-3,4.0787407E-3],"split_indices":[1,0,0],"split_type":[0,0,0],"sum_hessian":[1.4458455E2,6.905801E1,7.5526535E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[-3.350957E-2,2.346545E-3,-3.7299043E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":7,"left_children":[1,-1,-1],"loss_changes":[1.3207154E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[1.2120422E-2,2.346545E-3,-3.7299043E-3],"split_indices":[2,0,0],"split_type":[0,0,0],"sum_hessian":[1.431Line truncated
|
||||
{"learner":{"attributes":{"best_iteration":"131","best_score":"0.6530751986636056","scikit_learn":"{\"_estimator_type\": \"classifier\"}"},"feature_names":["VIX_rank_20","TLT_ret_10","USO_ret_5","SPY_TLT_ratio_ret","SPY_ret_5","SPY_ret_20","SPY_dist_sma50"],"feature_types":["float","float","float","float","float","float","float"],"gradient_booster":{"model":{"cats":{"enc":[],"feature_segments":[],"sorted_idx":[]},"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"152"},"iteration_indptr":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152],"tree_info":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"trees":[{"base_weights":[9.09237E-3,3.342369E-4,-1.4195536E-2],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":0,"left_children":[1,-1,-1],"loss_changes":[5.105532E0,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[6.892705E-2,3.342369E-4,-1.4195536E-2],"split_indices":[6,0,0],"split_type":[0,0,0],"sum_hessian":[1.4451103E2,1.4305133E2,1.4597074E0],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[1.9695353E-2,2.3339752E-3,-3.2156436E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":1,"left_children":[1,-1,-1],"loss_changes":[1.0367284E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[1.4354015E-2,2.3339752E-3,-3.2156436E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.4012877E2,8.636299E1,5.376578E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[9.294399E-2,-1.785295E-3,6.0087293E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":2,"left_children":[1,-1,-1],"loss_changes":[1.9767847E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[7.100592E-3,-1.785295E-3,6.0087293E-3],"split_indices":[1,0,0],"split_type":[0,0,0],"sum_hessian":[1.4158235E2,9.269047E1,4.8891888E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[4.908758E-2,7.0679425E-3,-1.0177689E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":3,"left_children":[1,-1,-1],"loss_changes":[1.4627343E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-1.7963655E-2,7.0679425E-3,-1.0177689E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.4569156E2,2.6495813E1,1.1919575E2],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[3.9804457E-3,3.4415158E-3,-1.9024174E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":4,"left_children":[1,-1,-1],"loss_changes":[9.772864E0,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-1.0558252E-2,3.4415158E-3,-1.9024174E-3],"split_indices":[2,0,0],"split_type":[0,0,0],"sum_hessian":[1.4592857E2,5.2756344E1,9.317223E1],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[-2.8463421E-2,6.9694594E-3,-1.250735E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":5,"left_children":[1,-1,-1],"loss_changes":[1.0434825E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-2.858291E-2,6.9694594E-3,-1.250735E-3],"split_indices":[3,0,0],"split_type":[0,0,0],"sum_hessian":[1.4665097E2,1.6505238E1,1.3014574E2],"tree_param":{"num_deleted":"0","num_feature":"7","num_nodes":"3","size_leaf_vector":"1"}},{"base_weights":[2.73039E-2,-3.935056E-3,4.084229E-3],"categories":[],"categories_nodes":[],"categories_segments":[],"categories_sizes":[],"default_left":[0,0,0],"id":6,"left_children":[1,-1,-1],"loss_changes":[2.3570513E1,0E0,0E0],"parents":[2147483647,0,0],"right_children":[2,-1,-1],"split_conditions":[-9.210526E-3,-3.935056E-3,4.084229E-3],"split_indices":[1,0,0],"split_type":[0,0,0],"sum_hessian":[1.4496594E2,6.888007E1,7.608588E1],"tree_paLine truncated
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"p_base": 0.5833333333333334,
|
||||
"p_base": 0.5819477434679335,
|
||||
"feature_cols": [
|
||||
"VIX_rank_20",
|
||||
"TLT_ret_10",
|
||||
@@ -9,5 +9,5 @@
|
||||
"SPY_ret_20",
|
||||
"SPY_dist_sma50"
|
||||
],
|
||||
"last_trained_date": "2026-07-24 00:00:00"
|
||||
"last_trained_date": "2026-07-29 00:00:00"
|
||||
}
|
||||
@@ -20,14 +20,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(PosixPath('/home/jarno/repos/trading-bot'),\n",
|
||||
" PosixPath('/home/jarno/repos/trading-bot/data/ibkr/daily'),\n",
|
||||
" PosixPath('/home/jarno/repos/trading-bot/data/alpaca/daily'),\n",
|
||||
" PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'))"
|
||||
]
|
||||
},
|
||||
@@ -87,16 +87,16 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"<class 'pandas.DataFrame'>\n",
|
||||
"DatetimeIndex: 1251 entries, 2021-07-30 to 2026-07-24\n",
|
||||
"DatetimeIndex: 1255 entries, 2021-08-02 to 2026-07-31\n",
|
||||
"Data columns (total 4 columns):\n",
|
||||
" # Column Non-Null Count Dtype \n",
|
||||
"--- ------ -------------- ----- \n",
|
||||
" 0 SPY_close 1251 non-null float64\n",
|
||||
" 1 VIX_close 1251 non-null float64\n",
|
||||
" 2 TLT_close 1251 non-null float64\n",
|
||||
" 3 USO_close 1251 non-null float64\n",
|
||||
" 0 SPY_close 1255 non-null float64\n",
|
||||
" 1 VIX_close 1255 non-null float64\n",
|
||||
" 2 TLT_close 1255 non-null float64\n",
|
||||
" 3 USO_close 1255 non-null float64\n",
|
||||
"dtypes: float64(4)\n",
|
||||
"memory usage: 48.9 KB\n"
|
||||
"memory usage: 49.0 KB\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -135,40 +135,40 @@
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-07-30</th>\n",
|
||||
" <td>438.51</td>\n",
|
||||
" <td>495.4</td>\n",
|
||||
" <td>149.52</td>\n",
|
||||
" <td>50.66</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-08-02</th>\n",
|
||||
" <td>437.59</td>\n",
|
||||
" <td>513.6</td>\n",
|
||||
" <td>25.68</td>\n",
|
||||
" <td>150.67</td>\n",
|
||||
" <td>49.18</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-08-03</th>\n",
|
||||
" <td>441.15</td>\n",
|
||||
" <td>488.0</td>\n",
|
||||
" <td>24.40</td>\n",
|
||||
" <td>150.75</td>\n",
|
||||
" <td>48.85</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-08-04</th>\n",
|
||||
" <td>438.98</td>\n",
|
||||
" <td>487.6</td>\n",
|
||||
" <td>24.38</td>\n",
|
||||
" <td>151.06</td>\n",
|
||||
" <td>47.20</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-08-05</th>\n",
|
||||
" <td>441.76</td>\n",
|
||||
" <td>474.8</td>\n",
|
||||
" <td>23.74</td>\n",
|
||||
" <td>150.29</td>\n",
|
||||
" <td>48.10</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-08-06</th>\n",
|
||||
" <td>442.49</td>\n",
|
||||
" <td>23.14</td>\n",
|
||||
" <td>147.78</td>\n",
|
||||
" <td>47.57</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
@@ -176,11 +176,11 @@
|
||||
"text/plain": [
|
||||
" SPY_close VIX_close TLT_close USO_close\n",
|
||||
"date \n",
|
||||
"2021-07-30 438.51 495.4 149.52 50.66\n",
|
||||
"2021-08-02 437.59 513.6 150.67 49.18\n",
|
||||
"2021-08-03 441.15 488.0 150.75 48.85\n",
|
||||
"2021-08-04 438.98 487.6 151.06 47.20\n",
|
||||
"2021-08-05 441.76 474.8 150.29 48.10"
|
||||
"2021-08-02 437.59 25.68 150.67 49.18\n",
|
||||
"2021-08-03 441.15 24.40 150.75 48.85\n",
|
||||
"2021-08-04 438.98 24.38 151.06 47.20\n",
|
||||
"2021-08-05 441.76 23.74 150.29 48.10\n",
|
||||
"2021-08-06 442.49 23.14 147.78 47.57"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@@ -234,7 +234,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -283,24 +283,12 @@
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-10-08</th>\n",
|
||||
" <td>0.008336</td>\n",
|
||||
" <td>-0.017017</td>\n",
|
||||
" <td>-0.011155</td>\n",
|
||||
" <td>-0.075239</td>\n",
|
||||
" <td>0.125</td>\n",
|
||||
" <td>-0.034239</td>\n",
|
||||
" <td>0.041495</td>\n",
|
||||
" <td>0.032998</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-10-11</th>\n",
|
||||
" <td>0.014114</td>\n",
|
||||
" <td>-0.026625</td>\n",
|
||||
" <td>-0.018145</td>\n",
|
||||
" <td>-0.090869</td>\n",
|
||||
" <td>0.200</td>\n",
|
||||
" <td>0.20</td>\n",
|
||||
" <td>-0.033135</td>\n",
|
||||
" <td>0.031015</td>\n",
|
||||
" <td>0.038908</td>\n",
|
||||
@@ -312,7 +300,7 @@
|
||||
" <td>-0.023752</td>\n",
|
||||
" <td>-0.020386</td>\n",
|
||||
" <td>-0.074091</td>\n",
|
||||
" <td>0.100</td>\n",
|
||||
" <td>0.10</td>\n",
|
||||
" <td>-0.001041</td>\n",
|
||||
" <td>0.008628</td>\n",
|
||||
" <td>-0.001303</td>\n",
|
||||
@@ -324,7 +312,7 @@
|
||||
" <td>-0.028356</td>\n",
|
||||
" <td>-0.016597</td>\n",
|
||||
" <td>-0.081006</td>\n",
|
||||
" <td>0.050</td>\n",
|
||||
" <td>0.05</td>\n",
|
||||
" <td>0.006928</td>\n",
|
||||
" <td>0.036928</td>\n",
|
||||
" <td>-0.005897</td>\n",
|
||||
@@ -336,12 +324,24 @@
|
||||
" <td>-0.010443</td>\n",
|
||||
" <td>-0.000214</td>\n",
|
||||
" <td>-0.096759</td>\n",
|
||||
" <td>0.050</td>\n",
|
||||
" <td>0.05</td>\n",
|
||||
" <td>0.010809</td>\n",
|
||||
" <td>0.026192</td>\n",
|
||||
" <td>-0.011991</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2021-10-15</th>\n",
|
||||
" <td>0.018294</td>\n",
|
||||
" <td>0.010127</td>\n",
|
||||
" <td>0.007213</td>\n",
|
||||
" <td>-0.079882</td>\n",
|
||||
" <td>0.05</td>\n",
|
||||
" <td>-0.002202</td>\n",
|
||||
" <td>0.030287</td>\n",
|
||||
" <td>-0.003823</td>\n",
|
||||
" <td>1.0</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
@@ -349,22 +349,22 @@
|
||||
"text/plain": [
|
||||
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
||||
"date \n",
|
||||
"2021-10-08 0.008336 -0.017017 -0.011155 -0.075239 0.125 \n",
|
||||
"2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 0.200 \n",
|
||||
"2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 0.100 \n",
|
||||
"2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 0.050 \n",
|
||||
"2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 0.050 \n",
|
||||
"2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 0.20 \n",
|
||||
"2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 0.10 \n",
|
||||
"2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 0.05 \n",
|
||||
"2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 0.05 \n",
|
||||
"2021-10-15 0.018294 0.010127 0.007213 -0.079882 0.05 \n",
|
||||
"\n",
|
||||
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d \n",
|
||||
"date \n",
|
||||
"2021-10-08 -0.034239 0.041495 0.032998 1.0 \n",
|
||||
"2021-10-11 -0.033135 0.031015 0.038908 1.0 \n",
|
||||
"2021-10-12 -0.001041 0.008628 -0.001303 1.0 \n",
|
||||
"2021-10-13 0.006928 0.036928 -0.005897 1.0 \n",
|
||||
"2021-10-14 0.010809 0.026192 -0.011991 1.0 "
|
||||
"2021-10-14 0.010809 0.026192 -0.011991 1.0 \n",
|
||||
"2021-10-15 -0.002202 0.030287 -0.003823 1.0 "
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -384,7 +384,7 @@
|
||||
"df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
|
||||
"df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
|
||||
"df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
|
||||
"df_model\n",
|
||||
"\n",
|
||||
"spy_forward_close = df[\"SPY_close\"].shift(-5)\n",
|
||||
"df[\"spy_up_5d\"] = np.nan\n",
|
||||
"df.loc[spy_forward_close > df[\"SPY_close\"], \"spy_up_5d\"] = 1.0\n",
|
||||
@@ -418,7 +418,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -458,24 +458,24 @@
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>train</th>\n",
|
||||
" <td>837</td>\n",
|
||||
" <td>2021-10-08</td>\n",
|
||||
" <td>2025-02-07</td>\n",
|
||||
" <td>0.583035</td>\n",
|
||||
" <td>840</td>\n",
|
||||
" <td>2021-10-11</td>\n",
|
||||
" <td>2025-02-13</td>\n",
|
||||
" <td>0.583333</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>validation</th>\n",
|
||||
" <td>179</td>\n",
|
||||
" <td>2025-02-10</td>\n",
|
||||
" <td>2025-10-24</td>\n",
|
||||
" <td>0.653631</td>\n",
|
||||
" <td>180</td>\n",
|
||||
" <td>2025-02-14</td>\n",
|
||||
" <td>2025-10-31</td>\n",
|
||||
" <td>0.633333</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>test</th>\n",
|
||||
" <td>181</td>\n",
|
||||
" <td>2025-10-27</td>\n",
|
||||
" <td>2026-07-17</td>\n",
|
||||
" <td>0.558011</td>\n",
|
||||
" <td>2025-11-03</td>\n",
|
||||
" <td>2026-07-24</td>\n",
|
||||
" <td>0.569061</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
@@ -484,12 +484,12 @@
|
||||
"text/plain": [
|
||||
" rows start_date end_date target_mean\n",
|
||||
"split \n",
|
||||
"train 837 2021-10-08 2025-02-07 0.583035\n",
|
||||
"validation 179 2025-02-10 2025-10-24 0.653631\n",
|
||||
"test 181 2025-10-27 2026-07-17 0.558011"
|
||||
"train 840 2021-10-11 2025-02-13 0.583333\n",
|
||||
"validation 180 2025-02-14 2025-10-31 0.633333\n",
|
||||
"test 181 2025-11-03 2026-07-24 0.569061"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -543,17 +543,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Rows: 1,197\n",
|
||||
"Rows: 1,201\n",
|
||||
"Feature columns: ['SPY_ret_5', 'SPY_ret_20', 'SPY_dist_sma50', 'VIX_change_5', 'VIX_rank_20', 'TLT_ret_10', 'USO_ret_5', 'SPY_TLT_ratio_ret']\n",
|
||||
"Target column: spy_up_5d\n",
|
||||
"Training matrix shape: (837, 8)\n"
|
||||
"Training matrix shape: (840, 8)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -592,16 +592,16 @@
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>count</th>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197.000000</td>\n",
|
||||
" <td>1197</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201.000000</td>\n",
|
||||
" <td>1201</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>unique</th>\n",
|
||||
@@ -640,32 +640,32 @@
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>837</td>\n",
|
||||
" <td>840</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>mean</th>\n",
|
||||
" <td>0.002556</td>\n",
|
||||
" <td>0.009841</td>\n",
|
||||
" <td>0.011067</td>\n",
|
||||
" <td>-0.008646</td>\n",
|
||||
" <td>0.387009</td>\n",
|
||||
" <td>-0.004083</td>\n",
|
||||
" <td>0.004662</td>\n",
|
||||
" <td>0.004927</td>\n",
|
||||
" <td>0.589808</td>\n",
|
||||
" <td>0.002502</td>\n",
|
||||
" <td>0.009848</td>\n",
|
||||
" <td>0.011028</td>\n",
|
||||
" <td>0.017763</td>\n",
|
||||
" <td>0.403476</td>\n",
|
||||
" <td>-0.004099</td>\n",
|
||||
" <td>0.005024</td>\n",
|
||||
" <td>0.004880</td>\n",
|
||||
" <td>0.588676</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>std</th>\n",
|
||||
" <td>0.023081</td>\n",
|
||||
" <td>0.043634</td>\n",
|
||||
" <td>0.037371</td>\n",
|
||||
" <td>0.092413</td>\n",
|
||||
" <td>0.333697</td>\n",
|
||||
" <td>0.028557</td>\n",
|
||||
" <td>0.052319</td>\n",
|
||||
" <td>0.027656</td>\n",
|
||||
" <td>0.492074</td>\n",
|
||||
" <td>0.023057</td>\n",
|
||||
" <td>0.043558</td>\n",
|
||||
" <td>0.037315</td>\n",
|
||||
" <td>0.299417</td>\n",
|
||||
" <td>0.337501</td>\n",
|
||||
" <td>0.028502</td>\n",
|
||||
" <td>0.052628</td>\n",
|
||||
" <td>0.027606</td>\n",
|
||||
" <td>0.492279</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
@@ -683,40 +683,40 @@
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>25%</th>\n",
|
||||
" <td>-0.009535</td>\n",
|
||||
" <td>-0.017017</td>\n",
|
||||
" <td>-0.008847</td>\n",
|
||||
" <td>-0.058376</td>\n",
|
||||
" <td>0.075000</td>\n",
|
||||
" <td>-0.023517</td>\n",
|
||||
" <td>-0.026200</td>\n",
|
||||
" <td>-0.009601</td>\n",
|
||||
" <td>-0.016509</td>\n",
|
||||
" <td>-0.008656</td>\n",
|
||||
" <td>-0.056615</td>\n",
|
||||
" <td>0.100000</td>\n",
|
||||
" <td>-0.023304</td>\n",
|
||||
" <td>-0.025866</td>\n",
|
||||
" <td>-0.010013</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>50%</th>\n",
|
||||
" <td>0.003869</td>\n",
|
||||
" <td>0.015797</td>\n",
|
||||
" <td>0.017906</td>\n",
|
||||
" <td>-0.020655</td>\n",
|
||||
" <td>0.250000</td>\n",
|
||||
" <td>-0.004339</td>\n",
|
||||
" <td>0.004817</td>\n",
|
||||
" <td>0.005710</td>\n",
|
||||
" <td>0.003798</td>\n",
|
||||
" <td>0.015725</td>\n",
|
||||
" <td>0.017802</td>\n",
|
||||
" <td>-0.018570</td>\n",
|
||||
" <td>0.300000</td>\n",
|
||||
" <td>-0.004435</td>\n",
|
||||
" <td>0.004950</td>\n",
|
||||
" <td>0.005537</td>\n",
|
||||
" <td>1.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>75%</th>\n",
|
||||
" <td>0.015957</td>\n",
|
||||
" <td>0.038385</td>\n",
|
||||
" <td>0.038332</td>\n",
|
||||
" <td>0.027778</td>\n",
|
||||
" <td>0.700000</td>\n",
|
||||
" <td>0.014272</td>\n",
|
||||
" <td>0.031883</td>\n",
|
||||
" <td>0.021406</td>\n",
|
||||
" <td>0.015955</td>\n",
|
||||
" <td>0.038179</td>\n",
|
||||
" <td>0.038282</td>\n",
|
||||
" <td>0.030814</td>\n",
|
||||
" <td>0.750000</td>\n",
|
||||
" <td>0.014227</td>\n",
|
||||
" <td>0.032486</td>\n",
|
||||
" <td>0.021390</td>\n",
|
||||
" <td>1.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
@@ -724,8 +724,8 @@
|
||||
" <th>max</th>\n",
|
||||
" <td>0.082843</td>\n",
|
||||
" <td>0.157566</td>\n",
|
||||
" <td>0.088103</td>\n",
|
||||
" <td>0.937828</td>\n",
|
||||
" <td>0.088105</td>\n",
|
||||
" <td>3.846154</td>\n",
|
||||
" <td>1.000000</td>\n",
|
||||
" <td>0.091322</td>\n",
|
||||
" <td>0.327273</td>\n",
|
||||
@@ -739,33 +739,33 @@
|
||||
],
|
||||
"text/plain": [
|
||||
" SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n",
|
||||
"count 1197.000000 1197.000000 1197.000000 1197.000000 1197.000000 \n",
|
||||
"count 1201.000000 1201.000000 1201.000000 1201.000000 1201.000000 \n",
|
||||
"unique NaN NaN NaN NaN NaN \n",
|
||||
"top NaN NaN NaN NaN NaN \n",
|
||||
"freq NaN NaN NaN NaN NaN \n",
|
||||
"mean 0.002556 0.009841 0.011067 -0.008646 0.387009 \n",
|
||||
"std 0.023081 0.043634 0.037371 0.092413 0.333697 \n",
|
||||
"mean 0.002502 0.009848 0.011028 0.017763 0.403476 \n",
|
||||
"std 0.023057 0.043558 0.037315 0.299417 0.337501 \n",
|
||||
"min -0.114962 -0.123975 -0.141995 -0.387709 0.050000 \n",
|
||||
"25% -0.009535 -0.017017 -0.008847 -0.058376 0.075000 \n",
|
||||
"50% 0.003869 0.015797 0.017906 -0.020655 0.250000 \n",
|
||||
"75% 0.015957 0.038385 0.038332 0.027778 0.700000 \n",
|
||||
"max 0.082843 0.157566 0.088103 0.937828 1.000000 \n",
|
||||
"25% -0.009601 -0.016509 -0.008656 -0.056615 0.100000 \n",
|
||||
"50% 0.003798 0.015725 0.017802 -0.018570 0.300000 \n",
|
||||
"75% 0.015955 0.038179 0.038282 0.030814 0.750000 \n",
|
||||
"max 0.082843 0.157566 0.088105 3.846154 1.000000 \n",
|
||||
"\n",
|
||||
" TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
|
||||
"count 1197.000000 1197.000000 1197.000000 1197.000000 1197 \n",
|
||||
"count 1201.000000 1201.000000 1201.000000 1201.000000 1201 \n",
|
||||
"unique NaN NaN NaN NaN 3 \n",
|
||||
"top NaN NaN NaN NaN train \n",
|
||||
"freq NaN NaN NaN NaN 837 \n",
|
||||
"mean -0.004083 0.004662 0.004927 0.589808 NaN \n",
|
||||
"std 0.028557 0.052319 0.027656 0.492074 NaN \n",
|
||||
"freq NaN NaN NaN NaN 840 \n",
|
||||
"mean -0.004099 0.005024 0.004880 0.588676 NaN \n",
|
||||
"std 0.028502 0.052628 0.027606 0.492279 NaN \n",
|
||||
"min -0.092880 -0.196652 -0.117208 0.000000 NaN \n",
|
||||
"25% -0.023517 -0.026200 -0.010013 0.000000 NaN \n",
|
||||
"50% -0.004339 0.004817 0.005710 1.000000 NaN \n",
|
||||
"75% 0.014272 0.031883 0.021406 1.000000 NaN \n",
|
||||
"25% -0.023304 -0.025866 -0.010013 0.000000 NaN \n",
|
||||
"50% -0.004435 0.004950 0.005537 1.000000 NaN \n",
|
||||
"75% 0.014227 0.032486 0.021390 1.000000 NaN \n",
|
||||
"max 0.091322 0.327273 0.129204 1.000000 NaN "
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -787,30 +787,30 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'),\n",
|
||||
" (1197, 11),\n",
|
||||
" (1201, 11),\n",
|
||||
" date SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 \\\n",
|
||||
" 0 2021-10-08 0.008336 -0.017017 -0.011155 -0.075239 \n",
|
||||
" 1 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n",
|
||||
" 2 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n",
|
||||
" 3 2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 \n",
|
||||
" 4 2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 \n",
|
||||
" 0 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n",
|
||||
" 1 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n",
|
||||
" 2 2021-10-13 0.000644 -0.028356 -0.016597 -0.081006 \n",
|
||||
" 3 2021-10-14 0.008754 -0.010443 -0.000214 -0.096759 \n",
|
||||
" 4 2021-10-15 0.018294 0.010127 0.007213 -0.079882 \n",
|
||||
" \n",
|
||||
" VIX_rank_20 TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n",
|
||||
" 0 0.125 -0.034239 0.041495 0.032998 1.0 train \n",
|
||||
" 1 0.200 -0.033135 0.031015 0.038908 1.0 train \n",
|
||||
" 2 0.100 -0.001041 0.008628 -0.001303 1.0 train \n",
|
||||
" 3 0.050 0.006928 0.036928 -0.005897 1.0 train \n",
|
||||
" 4 0.050 0.010809 0.026192 -0.011991 1.0 train )"
|
||||
" 0 0.20 -0.033135 0.031015 0.038908 1.0 train \n",
|
||||
" 1 0.10 -0.001041 0.008628 -0.001303 1.0 train \n",
|
||||
" 2 0.05 0.006928 0.036928 -0.005897 1.0 train \n",
|
||||
" 3 0.05 0.010809 0.026192 -0.011991 1.0 train \n",
|
||||
" 4 0.05 -0.002202 0.030287 -0.003823 1.0 train )"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "882d4907",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import xgboost as xgb\n",
|
||||
"from sklearn.metrics import classification_report, accuracy_score\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "947722b7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"# 1. Load data\n",
|
||||
"df = pd.read_parquet(\"../data/training/spy_direction_5d.parquet\")\n",
|
||||
"\n",
|
||||
"# 2. Define features & target\n",
|
||||
"target_col = \"spy_up_5d\"\n",
|
||||
"feature_cols = [\n",
|
||||
" \"VIX_rank_20\",\n",
|
||||
" \"TLT_ret_10\",\n",
|
||||
" \"USO_ret_5\",\n",
|
||||
" \"SPY_TLT_ratio_ret\",\n",
|
||||
" \"SPY_ret_5\",\n",
|
||||
" \"SPY_ret_20\",\n",
|
||||
" \"SPY_dist_sma50\"\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "5b02ce5e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 3. Create three distinct splits\n",
|
||||
"train_df = df[df[\"split\"] == \"train\"]\n",
|
||||
"val_df = df[df[\"split\"] == \"validation\"] # Adjust string if named \"val\"\n",
|
||||
"test_df = df[df[\"split\"] == \"test\"]\n",
|
||||
"\n",
|
||||
"X_train, y_train = train_df[feature_cols], train_df[target_col]\n",
|
||||
"X_val, y_val = val_df[feature_cols], val_df[target_col]\n",
|
||||
"X_test, y_test = test_df[feature_cols], test_df[target_col]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "93a128a7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Train positive %: 0.5819477434679335\n",
|
||||
"Val positive %: 0.6388888888888888\n",
|
||||
"Test positive %: 0.5769230769230769\n",
|
||||
"\n",
|
||||
"Train Feature Correlations with Target:\n",
|
||||
"VIX_rank_20 -0.004576\n",
|
||||
"TLT_ret_10 0.162370\n",
|
||||
"USO_ret_5 -0.113602\n",
|
||||
"SPY_TLT_ratio_ret -0.111551\n",
|
||||
"SPY_ret_5 -0.001073\n",
|
||||
"SPY_ret_20 0.006523\n",
|
||||
"SPY_dist_sma50 -0.035773\n",
|
||||
"dtype: float64\n",
|
||||
"\n",
|
||||
"Stump Model Best Iteration: 22\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 1. Check target distribution across splits\n",
|
||||
"print(\"Train positive %:\", y_train.mean())\n",
|
||||
"print(\"Val positive %: \", y_val.mean())\n",
|
||||
"print(\"Test positive %: \", y_test.mean())\n",
|
||||
"\n",
|
||||
"# 2. Check simple linear correlation with target\n",
|
||||
"print(\"\\nTrain Feature Correlations with Target:\")\n",
|
||||
"print(train_df[feature_cols].apply(lambda col: col.corr(y_train)))\n",
|
||||
"\n",
|
||||
"# 3. Test a super simple model (Decision Stump)\n",
|
||||
"stump_model = xgb.XGBClassifier(\n",
|
||||
" n_estimators=100,\n",
|
||||
" max_depth=2, # Decision stumps (1 split per tree, minimal overfitting)\n",
|
||||
" learning_rate=0.01,\n",
|
||||
" early_stopping_rounds=15,\n",
|
||||
" eval_metric=\"logloss\"\n",
|
||||
")\n",
|
||||
"stump_model.fit(\n",
|
||||
" X_train, y_train,\n",
|
||||
" eval_set=[(X_train, y_train), (X_val, y_val)],\n",
|
||||
" verbose=False\n",
|
||||
")\n",
|
||||
"print(f\"\\nStump Model Best Iteration: {stump_model.best_iteration}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "2f82ce9d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Optimal number of trees (best iteration): 131\n",
|
||||
"Starting Validation Loss: 0.6606\n",
|
||||
"Final Validation Loss: 0.6537\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"# 4. Initialize XGBoost Classifier with Early Stopping\n",
|
||||
"model = xgb.XGBClassifier(\n",
|
||||
" n_estimators=300, # Set higher; early stopping will truncate training automatically\n",
|
||||
" max_depth=1, # Keep shallow to prevent memorizing noise\n",
|
||||
" learning_rate=0.01,\n",
|
||||
" subsample=0.7,\n",
|
||||
" colsample_bytree=0.7,\n",
|
||||
" early_stopping_rounds=20, # Stop if validation loss stops improving for 15 rounds\n",
|
||||
" eval_metric=\"logloss\",\n",
|
||||
" random_state=42\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 5. Fit using Validation set to monitor performance\n",
|
||||
"model.fit(\n",
|
||||
" X_train, y_train,\n",
|
||||
" eval_set=[(X_train, y_train), (X_val, y_val)],\n",
|
||||
" verbose=False\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Optimal number of trees (best iteration): {model.best_iteration}\")\n",
|
||||
"\n",
|
||||
"evals = model.evals_result()\n",
|
||||
"val_loss = evals['validation_1']['logloss']\n",
|
||||
"print(f\"Starting Validation Loss: {val_loss[0]:.4f}\")\n",
|
||||
"print(f\"Final Validation Loss: {val_loss[-1]:.4f}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "2d54aaee",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"--- Final Holdout Test Performance ---\n",
|
||||
"Test Set Accuracy: 57.69%\n",
|
||||
"\n",
|
||||
"Classification Report:\n",
|
||||
" precision recall f1-score support\n",
|
||||
"\n",
|
||||
" 0.0 0.00 0.00 0.00 77\n",
|
||||
" 1.0 0.58 1.00 0.73 105\n",
|
||||
"\n",
|
||||
" accuracy 0.58 182\n",
|
||||
" macro avg 0.29 0.50 0.37 182\n",
|
||||
"weighted avg 0.33 0.58 0.42 182\n",
|
||||
"\n",
|
||||
"classification_report with custom threshold (0.5819477434679335):\n",
|
||||
" precision recall f1-score support\n",
|
||||
"\n",
|
||||
" 0.0 0.46 0.74 0.57 77\n",
|
||||
" 1.0 0.66 0.37 0.48 105\n",
|
||||
"\n",
|
||||
" accuracy 0.53 182\n",
|
||||
" macro avg 0.56 0.56 0.52 182\n",
|
||||
"weighted avg 0.58 0.53 0.52 182\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
|
||||
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n",
|
||||
"/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
|
||||
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n",
|
||||
"/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
|
||||
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAwMAAAHFCAYAAACuDCWjAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvctoD+AAAAAlwSFlzAAAPYQAAD2EBqD+naQAAd3NJREFUeJzt3Qm8zNX/x/Fj3wnZSZFUKMrWok1ZQlRaRKVU2rRpFaG0KdFGqahUtCutaFeypSyVskXZl1B25v94n35n/t+ZO8t3rnvduff7ej4e98GdO/Ndzsyc7/mc8znnmy8UCoUMAAAAgMDJn9MHAAAAACBnEAwAAAAAAUUwAAAAAAQUwQAAAAAQUAQDAAAAQEARDAAAAAABRTAAAAAABBTBAAAAABBQBAMAAGSDPXv2mIYNG5oBAwbst/I9+OCDTY8ePTL12t27d5s6deqYRx55JMuPC0D6IhgAkGv07t3bFCxYMOaP/pZd+yxZsqRJV2ps1q9f3+R2lStXNj179jR5yXPPPWf++OMPc9NNN2X4WygUMu+8847p2LGjqV69uilRooSpUaOGad68uenTp49ZsGBBphv0CkIyQ98j7XvQoEFmzZo1mdoGgNyHYABArqFGjn4WLVpktm/fHvGTXb2Z2p8aWOlKx5bOx7c/GrHpaO/evebhhx823bp1MwcccEDE3/R5VRBw8cUXm6ZNm5rJkyfbxvf06dPNjTfeaD744APTuHHjTO1XwceoUaMyfdxdu3Y1BQoUME8//XSmtwEgdyEYAJDrqLESPTKQPz/VGdLHJ598YpYuXWob/NHU4P/444/Np59+au6++25z+OGH25GBKlWqmC5dupgffvjBXHnllZn+buzLd6Fw4cLmvPPOs6MaeSHIBJAcV08Aec4vv/xiG1VqXKmR1aBBAzN8+PCI5zzxxBPhQKJQoUKmXLlypmXLlmbixInh55x77rn2eTt27IgIPNT7KurdVZpONLft1atXhx+77rrrzIEHHmg2b95srrjiClOxYsWI9J6xY8ea448/3pQuXdr2JLdt29bMnDkzU+ev16vB+d1335njjjvOblPpJ2pkypQpU+y+SpUqZY9BPdPxtqHn6rVKlVKj9ZlnnsnwXPXoDx482NSrVy/cqFWPuBrDsbb5/fffm5NOOsnuX/n0Kqv169fbHm1Xxqecckr4ddq/e7xo0aLmoIMOMpdffrlZuXJlzO3PmjXLtGjRwm5fOfDPP/98zHIaN26cPY6yZcuaSpUq2ff7559/jnhOZt+X999/374muod/2bJl5oUXXjDnn3++OfHEE2O+Vuf52GOPRTzmtwxizRlItVxOO+00u93Mfv4A5DIhAMglbrzxxpCqreXLl8d9zowZM0IlS5YMtWvXLjRnzpzQhg0bQmPHjg2VKlUq1KdPn/Dz9u7dG9q1a5f92bFjR2jhwoWh6667LlS4cOHQ7Nmz7XP27NkTuuGGG0JFihQJP1c/jvZRr169DMcwdOhQe5wrV64MP9azZ89QmTJlQhdeeGHo3XffDa1ZsyY0cuRI+7e77747VKhQodDgwYNDf/75Z2jZsmWha665JlS0aNHQrFmzEpaJ9l+3bt2Ix0qUKBFq27Zt6Pzzzw8tWLAgtGLFilCHDh1CFStWDE2dOjV09tlnh3755Rf7eKdOnWx5qZyit9GmTZtQ+/btQ/Pnzw+tWrUqdP/994fy5csXGjJkSMRzu3XrZo/1+eefD61du9a+B40aNQpVqFDBnkv0cWmbem8WL14cGj9+vC3T8uXLhy6//PJwGe/evTv8Ov3fPb558+bQN998E2rYsGGoSZMmEe+Htq/3pHPnzqF58+bZMr7lllvsezFlypSIY77jjjtCBQsWDA0YMCC0aNEi+1y9L5deemn4Ofv6vpxyyikZHlcZ6XhefvnlUCr8lkG1atUiziHVcpElS5bYvz388MMpHSOA3IlgAECuCwYKFCiQ4Wf79u32OU2bNg0deuih4d8dNejUsPM20GOpWbNmqFevXhH7VDAQS6rBgB4bPXp0xHN//fXXUP78+UN33XVXxOMKVtTYU+M5M8GAGtdbtmwJP6agQPs/6KCDQps2bQo/riBIjw8fPjzDNhRAbdy4MeJxBRgKHty21fDX69Wo9lq6dKkt7yuvvDJim3pt9DZFx9ujR4+QX2oMa7/Tpk2L2H65cuVsY9nZuXOnDUouueSS8GMKbhTU9O7dO+729/V90Xl26dIlw+P9+vWzx/3ll1+G9lWsMogXDPgpF0fBsbarwAdA3keaEIBcJ9YE4iJFiti0HE3CVPqOfvc644wzzK5du2zqjGzdutX079/fpskoBUapQi4FaOHChdl27J06dYr4XZNFNdlUaSNe+fLls2lLX375Zab2o/QX7ypISgvR+R199NE2fcWpXbu2LSuVaTSl6kRPfj3nnHPMP//8Y6ZNm2Z/dylGSrHxqlmzZnhyrNfJJ5+cYZvJKHVHaV9abUcpMt40ouj3SttXGoyj91XpTYsXLw4/9uGHH9rVfGLl82fF+6K0KZWRt5yT0ecueh6MN2UtlTKIxU+5eOcN6Ofvv//2ffwAci+CAQB5YgKxrFq1yv77+OOP2waTGrn6UcPG5W6vW7fO/quGoJ43cOBA2yD6999/bVChvPedO3fu0/GpoRlL8eLFMzSE3TErJ9wds2uMDR061Gzbts0GLqlS3n50I9bl80dT0LBp06YMjyuPPtYSoN5yVK5/rP25565duzbisWrVqqV0HspdV86+Ar13333Xrrqj90lBn0S/V7GOQ41yb8PWLZuZ6Fj25X3R51NlumXLlgx/U2Ne/vzzzwzBkwtsR44caQMKBSOZKYNY/JSLo+3pJ9WgDUDu9N8VFADygPLly9t/tVZ6v3794jbU1PBVo0oruXh7tNWI1wRP1+BNpkyZMrYHONqKFStiPl+9sfGOec6cOaZWrVoxX+eCnVSo8Z/K47ECmFhrzbtJ0e64NfHaPe4e8z43+rFYZZDI+PHj7fs1YsQIU7du3fDjy5cvz/T5VahQIfw+aVJ3LPv6vmiCb/TkXjn99NPtaj/q9dcynrG2F70aUKplsK/vuztuBSgA8j5GBgDkGbp5k9JgtJKLxLo5mWsUqRFUrFixiNe/9957GXpz1ZuuJRZjNZrUSFSDcuPGjeHH1JurZSX9ateunT0mBSfxbqiWU77++usM5aGy1QiHUoDk1FNPDZed119//WVmzJhhV6bxQ+Ucq4fblXv0ezVmzBiTWVoRyK0UlF3vi1YK0upN0Z+bQw45xI5Kad8qHz+yowwScSMOWnkIQN5HMAAgT9HSl7/99pvN9VavrhqY6ulUnnjr1q1tD6t69NXQ0XPVYFPKh9Z9HzJkiE0T8tLvStnQmvDRDbvu3bvbxv8tt9xiAwI1gK+55hpz2GGH+T5eLXt666232vkL2r+CCx2zzkFpTNdff73JKU2aNDGXXXaZWbJkiS23YcOG2eU4NfLi8uGVRqPRlQceeMA2cDVSovx2rVWvoCHeCE00lbMaodGjEZrroRSdO+64w6Ym6e/33HNP3FQsv2WuuwKrvLUkqj4fCnoUxLm7IO/r+9K+fXtbZm45Vy/d0EtzOjRKoJQjzRfQ50jHoDLQSEB2l0Ein3/+uU0RcwEfgLyNYABAnqLGqdZHVy+qGlFqkKpRq5so3XbbbTYQEDVc9bh6tpU2ohSM1157zeaHe11wwQW20a/Jm9H3GdDkW71Ga/FrG5pYeuaZZ6bco6oG6Ysvvmh7oTXRVw1tTYJWypKOOacoqLn66qvNWWedZdNm1HDVsSq9yktl2bt3b/u4ylf57Uq1mjp1qu0J90N3kFaOukZ3vJNjVR7vvPOODTCU49+oUSPbY3/fffft07npXPSeK7jR2vxKiXnqqafs+WbF+6LPgVKFYvXeaxREwaX299FHH9lzUvqUjkHBl/aj4FSf3+wsg1g0yf7NN9+09yrIyVEpAPtPPi0ptB/3BwCZpt5T/WRnI0WjAGpoxbqLq9/9q1rVdrzP0+v0uOYsZPXxine78c4hlcc1AVZBkBqs+6Nso58nycpK6Vt6jkv9irf97Cr7ZHSjO42MaGQllZWFUuGnDFIpFwUv1157rfn99999z50BkLsxMgAg11BjJrt7K9UwitdY9bt/Nbyin6fXZkdjVNuM3m68c0j18azmdz+xzikW7xyQRNvPrrJP5qqrrrLpNhqFyC5+ysBvuSiw0Opad911F4EAECCMAQIAkB0X2IIFzbx587Ittz87jvfXX38lPQgIGIIBAACyyf4YcclKzBMAgoc5AwCATOf3AwByN4IBAAAAIKDo8gEAAAACijkDSAta4k439SlVqlTEyhgAACB9aYK8bphXtWpV0gpzKYIBpAUFAjVq1MjpwwAAAJmwfPlye9NA5D4EA0gLGhEQ3ZynXLlyOX04aUd3BZ04caJp1aqVvVMpKCM+R3zXqI/SU9Dq682bN9vOPHcdR+5DMIC04FKDVJlk1506c/vFpXjx4rZsgnBxyQzKiDLic8R3LR0EtS4ixTf3YgIxAAAAEFAEAwAAAEBAEQwAAAAAAUUwAAAAAAQUwQAAAAAQUAQDAAAAQEARDAAAAAABRTAAAAAABBTBAAAAABBQBAMAAABAQBEMAAAAAAFFMAAAAAAEFMEAAAAAEFAEAwAAAEBAEQwAAAAAAUUwAAAAAAQUwQAAAAAQUAQDAAAAQEARDAAAAAABRTAAAAAABBTBAAAAABBQBAMAAABAQBEMAAAAAAFFMAAAAAAEFMEAAAAAEFAEAwAAAEBAEQwAAAAAAUUwAAAAAAQUwQAAAAAQUAQDAAAAQEARDAAAAAABRTAAAAAABBTBAAAAABBQBAMAAABAQBEMAAAAAAFFMAAAAAAEFMEAAAAAEFAEAwAAAEBAEQwAAAAAAUUwAAAAAAQUwQAAAAAQUAQDAAAAQEARDAAAAAABVTCnDwAAAADpb+XKlWbLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 640x480 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"# 6. Evaluate strictly on the holdout Test Set\n",
|
||||
"y_pred = model.predict(X_test)\n",
|
||||
"y_proba = model.predict_proba(X_test)[:, 1]\n",
|
||||
"\n",
|
||||
"print(\"\\n--- Final Holdout Test Performance ---\")\n",
|
||||
"print(f\"Test Set Accuracy: {accuracy_score(y_test, y_pred):.2%}\\n\")\n",
|
||||
"print(\"Classification Report:\")\n",
|
||||
"print(classification_report(y_test, y_pred))\n",
|
||||
"threshold = y_train.mean()\n",
|
||||
"y_pred_custom = (y_proba >= threshold).astype(int)\n",
|
||||
"print(f\"classification_report with custom threshold ({threshold}):\")\n",
|
||||
"print(classification_report(y_test, y_pred_custom))\n",
|
||||
"\n",
|
||||
"# 7. Feature Importance\n",
|
||||
"xgb.plot_importance(model, importance_type=\"gain\")\n",
|
||||
"plt.title(\"Feature Importance (Gain)\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "1e4b1067",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Min probability: 0.5154\n",
|
||||
"Max probability: 0.6891\n",
|
||||
"Mean probability: 0.5719\n",
|
||||
"Median probability: 0.5641\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 1. Inspect raw predicted probabilities for the test set\n",
|
||||
"y_proba_test = model.predict_proba(X_test)[:, 1]\n",
|
||||
"\n",
|
||||
"print(f\"Min probability: {y_proba_test.min():.4f}\")\n",
|
||||
"print(f\"Max probability: {y_proba_test.max():.4f}\")\n",
|
||||
"print(f\"Mean probability: {y_proba_test.mean():.4f}\")\n",
|
||||
"print(f\"Median probability: {np.median(y_proba_test):.4f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "0f415ee9",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Successfully saved model to models/spy_xgb_v1.json\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# 1. Create the 'models' directory if it doesn't exist\n",
|
||||
"models_dir = Path(\"models\")\n",
|
||||
"models_dir.mkdir(parents=True, exist_ok=True)\n",
|
||||
"\n",
|
||||
"# 2. Define paths\n",
|
||||
"model_path = models_dir / \"spy_xgb_v1.json\"\n",
|
||||
"meta_path = models_dir / \"spy_xgb_v1_meta.json\"\n",
|
||||
"\n",
|
||||
"# 3. Save XGBoost model (explicitly converted to str)\n",
|
||||
"model.save_model(str(model_path))\n",
|
||||
"\n",
|
||||
"# 4. Save metadata alongside model\n",
|
||||
"metadata = {\n",
|
||||
" \"p_base\": float(y_train.mean()),\n",
|
||||
" \"feature_cols\": feature_cols,\n",
|
||||
" \"last_trained_date\": str(df[\"date\"].iloc[-1]),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"with open(meta_path, \"w\") as f:\n",
|
||||
" json.dump(metadata, f, indent=2)\n",
|
||||
"\n",
|
||||
"print(f\"Successfully saved model to {model_path}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4d73c06a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.15"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Shared Alpaca daily candle helpers used by both the CLI fetcher and inference refreshes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pandas as pd
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file if available
|
||||
load_dotenv()
|
||||
|
||||
DEFAULT_OUTPUT_DIR = Path("data/alpaca/daily")
|
||||
DEFAULT_DURATION = "1 W"
|
||||
EASTERN_TZ = ZoneInfo("America/New_York")
|
||||
DURATION_PATTERN = re.compile(r"^(\d+)\s*([DWMY])$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DailyCandle:
|
||||
"""Daily OHLCV market data for one trading session."""
|
||||
|
||||
trading_day: date
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: int
|
||||
|
||||
|
||||
def default_end_date() -> date:
|
||||
"""Return yesterday's date in the US/Eastern market timezone."""
|
||||
|
||||
return datetime.now(EASTERN_TZ).date() - timedelta(days=1)
|
||||
|
||||
|
||||
def current_market_date() -> date:
|
||||
"""Return today's date in the US/Eastern market timezone."""
|
||||
|
||||
return datetime.now(EASTERN_TZ).date()
|
||||
|
||||
|
||||
def parse_end_date(value: str) -> date:
|
||||
"""Parse an end date in YYYYMMDD format."""
|
||||
|
||||
try:
|
||||
return datetime.strptime(value, "%Y%m%d").date()
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"end date must use YYYYMMDD format, such as 20250605"
|
||||
) from exc
|
||||
|
||||
|
||||
def parse_duration(value: str) -> str:
|
||||
"""Normalize and validate a duration string like '1 W' or '1 M'."""
|
||||
|
||||
normalized = " ".join(value.upper().split())
|
||||
if not DURATION_PATTERN.fullmatch(normalized):
|
||||
raise argparse.ArgumentTypeError(
|
||||
"duration must look like '1 D', '1 W', '1 M', or '1 Y'"
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def duration_to_start_date(end_date: date, duration: str) -> date:
|
||||
"""Calculate the start date based on end date and a duration string."""
|
||||
|
||||
match = DURATION_PATTERN.match(duration)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid duration format: {duration}")
|
||||
|
||||
amount = int(match.group(1))
|
||||
unit = match.group(2)
|
||||
|
||||
if unit == "D":
|
||||
return end_date - timedelta(days=amount)
|
||||
if unit == "W":
|
||||
return end_date - timedelta(weeks=amount)
|
||||
if unit == "M":
|
||||
return end_date - timedelta(days=amount * 30)
|
||||
if unit == "Y":
|
||||
return end_date - timedelta(days=amount * 365)
|
||||
|
||||
raise ValueError(f"Unsupported duration unit: {unit}")
|
||||
|
||||
|
||||
def fetch_daily_candles(
|
||||
symbol: str,
|
||||
end_date: date,
|
||||
duration: str,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
) -> list[DailyCandle]:
|
||||
"""Fetch daily candles for a symbol using Alpaca Market Data API."""
|
||||
|
||||
try:
|
||||
from alpaca.data.historical import StockHistoricalDataClient
|
||||
from alpaca.data.requests import StockBarsRequest
|
||||
from alpaca.data.timeframe import TimeFrame
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
"Missing dependency: alpaca-py. Install it via 'pip install alpaca-py' "
|
||||
"before running the script."
|
||||
) from exc
|
||||
|
||||
start_date = duration_to_start_date(end_date, duration)
|
||||
start_dt = datetime.combine(start_date, time.min, tzinfo=EASTERN_TZ)
|
||||
end_dt = datetime.combine(end_date, time.max, tzinfo=EASTERN_TZ)
|
||||
|
||||
client = StockHistoricalDataClient(api_key=api_key, secret_key=secret_key)
|
||||
|
||||
request_params = StockBarsRequest(
|
||||
symbol_or_symbols=symbol,
|
||||
timeframe=TimeFrame.Day,
|
||||
start=start_dt,
|
||||
end=end_dt,
|
||||
)
|
||||
|
||||
bars = client.get_stock_bars(request_params)
|
||||
|
||||
if not bars or symbol not in bars.data:
|
||||
print(f"Alpaca returned no historical bars for {symbol}.")
|
||||
return []
|
||||
|
||||
candles: list[DailyCandle] = []
|
||||
for bar in bars[symbol]:
|
||||
trading_day = bar.timestamp.astimezone(EASTERN_TZ).date()
|
||||
candles.append(
|
||||
DailyCandle(
|
||||
trading_day=trading_day,
|
||||
open=float(bar.open),
|
||||
high=float(bar.high),
|
||||
low=float(bar.low),
|
||||
close=float(bar.close),
|
||||
volume=int(bar.volume),
|
||||
)
|
||||
)
|
||||
|
||||
return candles
|
||||
|
||||
|
||||
def candles_to_frame(symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
|
||||
"""Convert candles to a date-indexed dataframe ready for Parquet storage."""
|
||||
|
||||
rows = [
|
||||
{
|
||||
"date": candle.trading_day,
|
||||
"symbol": symbol,
|
||||
"open": candle.open,
|
||||
"high": candle.high,
|
||||
"low": candle.low,
|
||||
"close": candle.close,
|
||||
"volume": candle.volume,
|
||||
}
|
||||
for candle in candles
|
||||
]
|
||||
frame = pd.DataFrame.from_records(rows)
|
||||
if frame.empty:
|
||||
return pd.DataFrame(
|
||||
columns=["symbol", "open", "high", "low", "close", "volume"],
|
||||
index=pd.Index([], name="date"),
|
||||
)
|
||||
|
||||
frame["date"] = pd.to_datetime(frame["date"]).dt.date
|
||||
return frame.set_index("date")
|
||||
|
||||
|
||||
def read_existing_candles(path: Path) -> pd.DataFrame:
|
||||
"""Read an existing candle Parquet file as a date-indexed dataframe."""
|
||||
|
||||
if not path.exists():
|
||||
return pd.DataFrame(
|
||||
columns=["symbol", "open", "high", "low", "close", "volume"],
|
||||
index=pd.Index([], name="date"),
|
||||
)
|
||||
|
||||
frame = pd.read_parquet(path)
|
||||
if "date" in frame.columns:
|
||||
frame["date"] = pd.to_datetime(frame["date"]).dt.date
|
||||
frame = frame.set_index("date")
|
||||
|
||||
frame.index = pd.to_datetime(frame.index).date
|
||||
frame.index.name = "date"
|
||||
return frame
|
||||
|
||||
|
||||
def oldest_stored_date_or_today(path: Path) -> date:
|
||||
"""Return the oldest stored candle date, or today if no data exists yet."""
|
||||
|
||||
existing = read_existing_candles(path)
|
||||
if existing.empty:
|
||||
return current_market_date()
|
||||
return min(existing.index)
|
||||
|
||||
|
||||
def write_candles(path: Path, symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
|
||||
"""Append candles to a ticker Parquet file, keeping one row per date."""
|
||||
|
||||
existing = read_existing_candles(path)
|
||||
fetched = candles_to_frame(symbol, candles)
|
||||
combined = pd.concat([existing, fetched])
|
||||
if not combined.empty:
|
||||
combined = combined[~combined.index.duplicated(keep="last")]
|
||||
combined = combined.sort_index()
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
combined.to_parquet(path, index=True)
|
||||
return combined
|
||||
|
||||
|
||||
def print_candles(symbol: str, candles: list[DailyCandle]) -> None:
|
||||
"""Print candles in a compact table."""
|
||||
|
||||
print("date,symbol,open,high,low,close,volume")
|
||||
for candle in candles:
|
||||
print(
|
||||
f"{candle.trading_day.isoformat()},"
|
||||
f"{symbol},"
|
||||
f"{candle.open:.2f},"
|
||||
f"{candle.high:.2f},"
|
||||
f"{candle.low:.2f},"
|
||||
f"{candle.close:.2f},"
|
||||
f"{candle.volume}"
|
||||
)
|
||||
|
||||
|
||||
def refresh_recent_market_data(
|
||||
output_dir: Path | str,
|
||||
symbols: dict[str, str] | None = None,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
duration: str = DEFAULT_DURATION,
|
||||
) -> None:
|
||||
"""Refresh the latest weekly Alpaca candle parquet files for the provided symbols."""
|
||||
|
||||
output_dir = Path(output_dir)
|
||||
symbols = symbols or {}
|
||||
|
||||
for symbol in symbols.keys():
|
||||
output_path = output_dir / f"{symbol}.parquet"
|
||||
candles = fetch_daily_candles(
|
||||
symbol=symbol,
|
||||
end_date=default_end_date(),
|
||||
duration=duration,
|
||||
api_key=api_key,
|
||||
secret_key=secret_key,
|
||||
)
|
||||
print(f"Fetched {len(candles)} daily candles for {symbol}.")
|
||||
write_candles(output_path, symbol, candles)
|
||||
@@ -1,236 +1,31 @@
|
||||
"""Alpaca daily candle fetcher."""
|
||||
"""Alpaca daily candle fetcher CLI wrapper.
|
||||
|
||||
The reusable implementation lives in the shared library module so the same
|
||||
fetching and parquet persistence logic can be reused by the prediction flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pandas as pd
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file if available
|
||||
load_dotenv()
|
||||
|
||||
DEFAULT_OUTPUT_DIR = Path("data/alpaca/daily")
|
||||
DEFAULT_DURATION = "1 W"
|
||||
EASTERN_TZ = ZoneInfo("America/New_York")
|
||||
DURATION_PATTERN = re.compile(r"^(\d+)\s*([DWMY])$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DailyCandle:
|
||||
"""Daily OHLCV market data for one trading session."""
|
||||
|
||||
trading_day: date
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: int
|
||||
|
||||
|
||||
def default_end_date() -> date:
|
||||
"""Return yesterday's date in the US/Eastern market timezone."""
|
||||
|
||||
return datetime.now(EASTERN_TZ).date() - timedelta(days=1)
|
||||
|
||||
|
||||
def current_market_date() -> date:
|
||||
"""Return today's date in the US/Eastern market timezone."""
|
||||
|
||||
return datetime.now(EASTERN_TZ).date()
|
||||
|
||||
|
||||
def parse_end_date(value: str) -> date:
|
||||
"""Parse an end date in YYYYMMDD format."""
|
||||
|
||||
try:
|
||||
return datetime.strptime(value, "%Y%m%d").date()
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"end date must use YYYYMMDD format, such as 20250605"
|
||||
) from exc
|
||||
|
||||
|
||||
def parse_duration(value: str) -> str:
|
||||
"""Normalize and validate a duration string like '1 W' or '1 M'."""
|
||||
|
||||
normalized = " ".join(value.upper().split())
|
||||
if not DURATION_PATTERN.fullmatch(normalized):
|
||||
raise argparse.ArgumentTypeError(
|
||||
"duration must look like '1 D', '1 W', '1 M', or '1 Y'"
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def duration_to_start_date(end_date: date, duration: str) -> date:
|
||||
"""Calculate the start date based on end date and a duration string."""
|
||||
|
||||
match = DURATION_PATTERN.match(duration)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid duration format: {duration}")
|
||||
|
||||
amount = int(match.group(1))
|
||||
unit = match.group(2)
|
||||
|
||||
if unit == "D":
|
||||
return end_date - timedelta(days=amount)
|
||||
if unit == "W":
|
||||
return end_date - timedelta(weeks=amount)
|
||||
if unit == "M":
|
||||
return end_date - timedelta(days=amount * 30)
|
||||
if unit == "Y":
|
||||
return end_date - timedelta(days=amount * 365)
|
||||
|
||||
raise ValueError(f"Unsupported duration unit: {unit}")
|
||||
|
||||
|
||||
def fetch_daily_candles(
|
||||
symbol: str,
|
||||
end_date: date,
|
||||
duration: str,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
) -> list[DailyCandle]:
|
||||
"""Fetch daily candles for a symbol using Alpaca Market Data API."""
|
||||
|
||||
try:
|
||||
from alpaca.data.historical import StockHistoricalDataClient
|
||||
from alpaca.data.requests import StockBarsRequest
|
||||
from alpaca.data.timeframe import TimeFrame
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
"Missing dependency: alpaca-py. Install it via 'pip install alpaca-py' "
|
||||
"before running the script."
|
||||
) from exc
|
||||
|
||||
start_date = duration_to_start_date(end_date, duration)
|
||||
|
||||
start_dt = datetime.combine(start_date, time.min, tzinfo=EASTERN_TZ)
|
||||
end_dt = datetime.combine(end_date, time.max, tzinfo=EASTERN_TZ)
|
||||
|
||||
print(start_dt, end_dt)
|
||||
|
||||
client = StockHistoricalDataClient(api_key=api_key, secret_key=secret_key)
|
||||
|
||||
request_params = StockBarsRequest(
|
||||
symbol_or_symbols=symbol,
|
||||
timeframe=TimeFrame.Day,
|
||||
start=start_dt,
|
||||
end=end_dt,
|
||||
)
|
||||
|
||||
bars = client.get_stock_bars(request_params)
|
||||
|
||||
if not bars or symbol not in bars.data:
|
||||
print(f"Alpaca returned no historical bars for {symbol}.")
|
||||
return []
|
||||
|
||||
candles: list[DailyCandle] = []
|
||||
for bar in bars[symbol]:
|
||||
trading_day = bar.timestamp.astimezone(EASTERN_TZ).date()
|
||||
candles.append(
|
||||
DailyCandle(
|
||||
trading_day=trading_day,
|
||||
open=float(bar.open),
|
||||
high=float(bar.high),
|
||||
low=float(bar.low),
|
||||
close=float(bar.close),
|
||||
volume=int(bar.volume),
|
||||
)
|
||||
)
|
||||
|
||||
return candles
|
||||
|
||||
|
||||
def candles_to_frame(symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
|
||||
"""Convert candles to a date-indexed dataframe ready for Parquet storage."""
|
||||
|
||||
rows = [
|
||||
{
|
||||
"date": candle.trading_day,
|
||||
"symbol": symbol,
|
||||
"open": candle.open,
|
||||
"high": candle.high,
|
||||
"low": candle.low,
|
||||
"close": candle.close,
|
||||
"volume": candle.volume,
|
||||
}
|
||||
for candle in candles
|
||||
]
|
||||
frame = pd.DataFrame.from_records(rows)
|
||||
if frame.empty:
|
||||
return pd.DataFrame(
|
||||
columns=["symbol", "open", "high", "low", "close", "volume"],
|
||||
index=pd.Index([], name="date"),
|
||||
)
|
||||
|
||||
frame["date"] = pd.to_datetime(frame["date"]).dt.date
|
||||
return frame.set_index("date")
|
||||
|
||||
|
||||
def read_existing_candles(path: Path) -> pd.DataFrame:
|
||||
"""Read an existing candle Parquet file as a date-indexed dataframe."""
|
||||
|
||||
if not path.exists():
|
||||
return pd.DataFrame(
|
||||
columns=["symbol", "open", "high", "low", "close", "volume"],
|
||||
index=pd.Index([], name="date"),
|
||||
)
|
||||
|
||||
frame = pd.read_parquet(path)
|
||||
if "date" in frame.columns:
|
||||
frame["date"] = pd.to_datetime(frame["date"]).dt.date
|
||||
frame = frame.set_index("date")
|
||||
|
||||
frame.index = pd.to_datetime(frame.index).date
|
||||
frame.index.name = "date"
|
||||
return frame
|
||||
|
||||
|
||||
def oldest_stored_date_or_today(path: Path) -> date:
|
||||
"""Return the oldest stored candle date, or today if no data exists yet."""
|
||||
|
||||
existing = read_existing_candles(path)
|
||||
if existing.empty:
|
||||
return current_market_date()
|
||||
return min(existing.index)
|
||||
|
||||
|
||||
def write_candles(path: Path, symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
|
||||
"""Append candles to a ticker Parquet file, keeping one row per date."""
|
||||
|
||||
existing = read_existing_candles(path)
|
||||
fetched = candles_to_frame(symbol, candles)
|
||||
combined = pd.concat([existing, fetched])
|
||||
if not combined.empty:
|
||||
combined = combined[~combined.index.duplicated(keep="last")]
|
||||
combined = combined.sort_index()
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
combined.to_parquet(path, index=True)
|
||||
return combined
|
||||
|
||||
|
||||
def print_candles(symbol: str, candles: list[DailyCandle]) -> None:
|
||||
"""Print candles in a compact table."""
|
||||
|
||||
print("date,symbol,open,high,low,close,volume")
|
||||
for candle in candles:
|
||||
print(
|
||||
f"{candle.trading_day.isoformat()},"
|
||||
f"{symbol},"
|
||||
f"{candle.open:.2f},"
|
||||
f"{candle.high:.2f},"
|
||||
f"{candle.low:.2f},"
|
||||
f"{candle.close:.2f},"
|
||||
f"{candle.volume}"
|
||||
)
|
||||
from trading_bot.data.alpaca_daily_lib import (
|
||||
DEFAULT_DURATION,
|
||||
DEFAULT_OUTPUT_DIR,
|
||||
DailyCandle,
|
||||
candles_to_frame,
|
||||
current_market_date,
|
||||
default_end_date,
|
||||
duration_to_start_date,
|
||||
fetch_daily_candles,
|
||||
oldest_stored_date_or_today,
|
||||
parse_duration,
|
||||
parse_end_date,
|
||||
print_candles,
|
||||
read_existing_candles,
|
||||
write_candles,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -292,8 +87,6 @@ def main() -> None:
|
||||
else args.end_date or default_end_date()
|
||||
)
|
||||
|
||||
print(oldest_stored_date_or_today(output_path))
|
||||
|
||||
print(
|
||||
f"Fetching {symbol} daily candles ending {end_date:%Y-%m-%d} "
|
||||
f"for duration {args.duration} via Alpaca API"
|
||||
@@ -305,7 +98,6 @@ def main() -> None:
|
||||
api_key=args.api_key,
|
||||
secret_key=args.secret_key,
|
||||
)
|
||||
#print_candles(symbol, candles)
|
||||
stored = write_candles(output_path, symbol, candles)
|
||||
print(f"Wrote {len(stored)} total daily rows to {output_path}")
|
||||
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate training dataset and train an XGBoost model using a JSON config.
|
||||
|
||||
This script combines the logic from notebooks/spy_direction_dataset.ipynb
|
||||
and notebooks/train-xboost.ipynb into a single runnable script.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import xgboost as xgb
|
||||
from sklearn.metrics import classification_report, accuracy_score
|
||||
|
||||
|
||||
def find_project_root(start: Path | None = None) -> Path:
|
||||
current = (start or Path.cwd()).resolve()
|
||||
for candidate in [current, *current.parents]:
|
||||
if (candidate / "pyproject.toml").exists():
|
||||
return candidate
|
||||
raise RuntimeError("Could not find project root containing pyproject.toml")
|
||||
|
||||
|
||||
def load_close(raw_dir: Path, symbol: str, alias: str | None = None) -> pd.DataFrame:
|
||||
alias = alias or symbol
|
||||
path = raw_dir / f"{symbol}.parquet"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Missing raw data file: {path}")
|
||||
|
||||
frame = pd.read_parquet(path)
|
||||
if "date" in frame.columns:
|
||||
frame = frame.set_index("date")
|
||||
if "close" not in frame.columns:
|
||||
raise ValueError(f"{path} does not contain a close column")
|
||||
|
||||
frame = frame.copy()
|
||||
frame.index = pd.to_datetime(frame.index)
|
||||
frame.index.name = "date"
|
||||
frame = frame.sort_index()
|
||||
return frame[["close"]].rename(columns={"close": f"{alias}_close"})
|
||||
|
||||
|
||||
def assign_chronological_splits(frame: pd.DataFrame, train_fraction: float, validation_fraction: float, test_fraction: float) -> pd.Series:
|
||||
total_fraction = train_fraction + validation_fraction + test_fraction
|
||||
if not np.isclose(total_fraction, 1.0):
|
||||
raise ValueError(f"Split fractions must sum to 1.0, got {total_fraction}")
|
||||
|
||||
n_rows = len(frame)
|
||||
train_end = int(n_rows * train_fraction)
|
||||
validation_end = train_end + int(n_rows * validation_fraction)
|
||||
|
||||
split = pd.Series(index=frame.index, dtype="object")
|
||||
split.iloc[:train_end] = "train"
|
||||
split.iloc[train_end:validation_end] = "validation"
|
||||
split.iloc[validation_end:] = "test"
|
||||
return split
|
||||
|
||||
|
||||
def build_dataset(raw_data_dir: Path, output_path: Path, symbols: dict, fractions: dict) -> pd.DataFrame:
|
||||
prices = pd.concat(
|
||||
[
|
||||
load_close(raw_data_dir, symbols["SPY"], "SPY"),
|
||||
load_close(raw_data_dir, symbols["VIX"] , "VIX"),
|
||||
load_close(raw_data_dir, symbols["TLT"] , "TLT"),
|
||||
load_close(raw_data_dir, symbols["USO"] , "USO"),
|
||||
],
|
||||
axis=1,
|
||||
join="inner",
|
||||
)
|
||||
|
||||
df = prices.copy()
|
||||
df["SPY_ret_5"] = df["SPY_close"].pct_change(5)
|
||||
df["SPY_ret_20"] = df["SPY_close"].pct_change(20)
|
||||
|
||||
sma_50 = df["SPY_close"].rolling(50).mean()
|
||||
df["SPY_dist_sma50"] = (df["SPY_close"] - sma_50) / sma_50
|
||||
|
||||
df["VIX_change_5"] = df["VIX_close"].pct_change(5)
|
||||
df["VIX_rank_20"] = df["VIX_close"].rolling(20).rank(pct=True)
|
||||
|
||||
df["TLT_ret_10"] = df["TLT_close"].pct_change(10)
|
||||
df["USO_ret_5"] = df["USO_close"].pct_change(5)
|
||||
df["SPY_TLT_ratio_ret"] = (df["SPY_close"] / df["TLT_close"]).pct_change(5)
|
||||
|
||||
spy_forward_close = df["SPY_close"].shift(-5)
|
||||
df["spy_up_5d"] = np.nan
|
||||
df.loc[spy_forward_close > df["SPY_close"], "spy_up_5d"] = 1.0
|
||||
df.loc[spy_forward_close < df["SPY_close"], "spy_up_5d"] = 0.0
|
||||
df.loc[spy_forward_close == df["SPY_close"], "spy_up_5d"] = 0.5
|
||||
|
||||
FEATURE_COLUMNS = [
|
||||
"SPY_ret_5",
|
||||
"SPY_ret_20",
|
||||
"SPY_dist_sma50",
|
||||
"VIX_change_5",
|
||||
"VIX_rank_20",
|
||||
"TLT_ret_10",
|
||||
"USO_ret_5",
|
||||
"SPY_TLT_ratio_ret",
|
||||
]
|
||||
TARGET_COLUMN = "spy_up_5d"
|
||||
|
||||
df_model = df[FEATURE_COLUMNS + [TARGET_COLUMN]].dropna().copy()
|
||||
|
||||
# Drop unchanged targets (0.5) to keep binary classification
|
||||
df_model = df_model[df_model[TARGET_COLUMN] != 0.5].copy()
|
||||
df_model[TARGET_COLUMN] = df_model[TARGET_COLUMN].astype(int)
|
||||
|
||||
df_model["split"] = assign_chronological_splits(
|
||||
df_model,
|
||||
fractions["train"],
|
||||
fractions["validation"],
|
||||
fractions["test"],
|
||||
)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
dataset_to_save = df_model.reset_index()
|
||||
dataset_to_save.to_parquet(output_path, index=False)
|
||||
|
||||
return df_model, FEATURE_COLUMNS, TARGET_COLUMN
|
||||
|
||||
|
||||
def train_model(df_model: pd.DataFrame, feature_cols: list, target_col: str, config: dict, models_dir: Path):
|
||||
train_df = df_model[df_model["split"] == "train"]
|
||||
val_df = df_model[df_model["split"] == "validation"]
|
||||
test_df = df_model[df_model["split"] == "test"]
|
||||
|
||||
X_train, y_train = train_df[feature_cols], train_df[target_col]
|
||||
X_val, y_val = val_df[feature_cols], val_df[target_col]
|
||||
X_test, y_test = test_df[feature_cols], test_df[target_col]
|
||||
|
||||
clf_params = dict(config)
|
||||
# Ensure early_stopping_rounds present as int
|
||||
|
||||
model = xgb.XGBClassifier(**clf_params)
|
||||
|
||||
eval_set = [(X_train, y_train), (X_val, y_val)]
|
||||
fit_kwargs = {"eval_set": eval_set, "verbose": False}
|
||||
|
||||
model.fit(X_train, y_train, **fit_kwargs)
|
||||
|
||||
print(f"Best iteration: {getattr(model, 'best_iteration', None)}")
|
||||
evals = model.evals_result()
|
||||
if "validation_1" in evals and "logloss" in evals["validation_1"]:
|
||||
val_loss = evals["validation_1"]["logloss"]
|
||||
print(f"Starting Validation Loss: {val_loss[0]:.4f}")
|
||||
print(f"Final Validation Loss: {val_loss[-1]:.4f}")
|
||||
|
||||
# Evaluate on test set
|
||||
y_pred = model.predict(X_test)
|
||||
print("\n--- Final Holdout Test Performance ---")
|
||||
print(f"Test Set Accuracy: {accuracy_score(y_test, y_pred):.2%}\n")
|
||||
print(classification_report(y_test, y_pred))
|
||||
|
||||
# Save model and metadata
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
model_path = models_dir / "spy_xgb_v1.json"
|
||||
meta_path = models_dir / "spy_xgb_v1_meta.json"
|
||||
model.save_model(str(model_path))
|
||||
|
||||
metadata = {
|
||||
"p_base": float(y_train.mean()),
|
||||
"feature_cols": feature_cols,
|
||||
"last_trained_date": str(df_model.index.max().date()),
|
||||
"config": config,
|
||||
}
|
||||
with open(meta_path, "w") as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
print(f"Saved model to {model_path}")
|
||||
|
||||
|
||||
def main():
|
||||
project_root = find_project_root()
|
||||
|
||||
# Defaults
|
||||
raw_data_dir = project_root / "data" / "alpaca" / "daily"
|
||||
output_path = project_root / "data" / "training" / "spy_direction_5d.parquet"
|
||||
models_dir = project_root / "models"
|
||||
config_path = project_root / "config" / "train_config.json"
|
||||
|
||||
# Load config
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
with open(config_path, "r") as f:
|
||||
config = json.load(f)
|
||||
|
||||
symbols = config.get("symbols", {"SPY": "SPY", "VIX": config.get("vix_symbol", "VIXY"), "TLT": "TLT", "USO": "USO"})
|
||||
fractions = config.get("fractions", {"train": 0.7, "validation": 0.15, "test": 0.15})
|
||||
|
||||
print("Building dataset...")
|
||||
df_model, feature_cols, target_col = build_dataset(raw_data_dir, output_path, symbols, fractions)
|
||||
|
||||
print(f"Rows: {len(df_model):,}")
|
||||
print(f"Feature columns: {feature_cols}")
|
||||
print(f"Target column: {target_col}")
|
||||
|
||||
split_counts = df_model.groupby("split").size()
|
||||
print("Split counts:")
|
||||
print(split_counts.to_string())
|
||||
|
||||
print("\nTraining model...")
|
||||
model_config = config.get("model", {})
|
||||
train_model(df_model, feature_cols, target_col, model_config, models_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,6 +21,8 @@ import pandas as pd
|
||||
import xgboost as xgb
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from trading_bot.data.alpaca_daily_lib import refresh_recent_market_data
|
||||
|
||||
# Load environment variables from .env file if available
|
||||
load_dotenv()
|
||||
|
||||
@@ -158,9 +160,20 @@ def predict_latest_probability(
|
||||
raw_data_dir: Path | str,
|
||||
symbols: dict[str, str] | None = None,
|
||||
max_age_days: int = 1,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
fetch_recent_data: bool = False,
|
||||
) -> PredictionResult:
|
||||
"""Return the latest predicted probability for the next SPY move."""
|
||||
|
||||
if fetch_recent_data:
|
||||
refresh_recent_market_data(
|
||||
output_dir=raw_data_dir,
|
||||
symbols=symbols or DEFAULT_SYMBOLS,
|
||||
api_key=api_key,
|
||||
secret_key=secret_key,
|
||||
)
|
||||
|
||||
model = load_model(model_path)
|
||||
metadata = load_feature_metadata(metadata_path)
|
||||
latest_date, latest_features = get_latest_inference_features(
|
||||
@@ -249,6 +262,9 @@ def main() -> None:
|
||||
model_path=args.model_path,
|
||||
metadata_path=args.metadata_path,
|
||||
raw_data_dir=args.data_dir,
|
||||
api_key=args.api_key,
|
||||
secret_key=args.secret_key,
|
||||
fetch_recent_data=True,
|
||||
)
|
||||
print(
|
||||
f"Date: {result.prediction_date.date()} | Prob: {result.probability:.4f} | "
|
||||
|
||||
@@ -1,8 +1,72 @@
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from trading_bot.models.prediction import FEATURE_COLUMNS, predict_latest_probability
|
||||
|
||||
|
||||
class FakeModel:
|
||||
def predict_proba(self, model_input):
|
||||
return np.array([[0.1, 0.9]])
|
||||
|
||||
|
||||
def test_predict_latest_probability_refreshes_recent_market_data(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
refresh_calls: dict[str, object] = {}
|
||||
|
||||
def fake_refresh_recent_data(
|
||||
output_dir: Path,
|
||||
symbols: dict[str, str] | None = None,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
) -> None:
|
||||
refresh_calls["symbols"] = list((symbols or {}).keys())
|
||||
refresh_calls["output_dir"] = output_dir
|
||||
refresh_calls["api_key"] = api_key
|
||||
refresh_calls["secret_key"] = secret_key
|
||||
|
||||
monkeypatch.setattr(
|
||||
"trading_bot.models.prediction.refresh_recent_market_data",
|
||||
fake_refresh_recent_data,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"trading_bot.models.prediction.load_model",
|
||||
lambda model_path: FakeModel(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"trading_bot.models.prediction.load_feature_metadata",
|
||||
lambda metadata_path: {"feature_cols": FEATURE_COLUMNS, "p_base": 0.5},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"trading_bot.models.prediction.get_latest_inference_features",
|
||||
lambda raw_data_dir, symbols=None, max_age_days=1: (
|
||||
pd.Timestamp("2026-08-03"),
|
||||
pd.DataFrame(
|
||||
[[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08]],
|
||||
columns=FEATURE_COLUMNS,
|
||||
index=[pd.Timestamp("2026-08-03")],
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = predict_latest_probability(
|
||||
model_path=Path("notebooks/models/spy_xgb_v1.json"),
|
||||
metadata_path=Path("notebooks/models/spy_xgb_v1_meta.json"),
|
||||
raw_data_dir=Path("data/alpaca/daily"),
|
||||
api_key="test-key",
|
||||
secret_key="test-secret",
|
||||
fetch_recent_data=True,
|
||||
)
|
||||
|
||||
assert refresh_calls["output_dir"] == Path("data/alpaca/daily")
|
||||
assert refresh_calls["symbols"] == ["SPY", "VIXY", "TLT", "USO"]
|
||||
assert refresh_calls["api_key"] == "test-key"
|
||||
assert refresh_calls["secret_key"] == "test-secret"
|
||||
assert result.probability == 0.9
|
||||
|
||||
|
||||
def test_predict_latest_probability_returns_probability_between_zero_and_one() -> None:
|
||||
result = predict_latest_probability(
|
||||
model_path=Path("notebooks/models/spy_xgb_v1.json"),
|
||||
|
||||
Reference in new issue
Block a user