Prediction and model training work

2026-08-06 17:22:57 +03:00
parent fddcc9190a
commit 465e09fc82
13 changed files with 1689 additions and 403 deletions
@@ -0,0 +1,23 @@
+{
+  "symbols": {
+    "SPY": "SPY",
+    "VIX": "VIXY",
+    "TLT": "TLT",
+    "USO": "USO"
+  },
+  "fractions": {
+    "train": 0.7,
+    "validation": 0.15,
+    "test": 0.15
+  },
+  "model": {
+    "n_estimators": 300,
+    "max_depth": 1,
+    "learning_rate": 0.01,
+    "subsample": 0.7,
+    "colsample_bytree": 0.7,
+    "early_stopping_rounds": 20,
+    "random_state": 42,
+    "eval_metric": "logloss"
+  }
+}
@@ -0,0 +1,164 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "c8a08105",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "from alpaca.trading.client import TradingClient\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Set paper=True for paper trading (sandbox), paper=False for live trading\n",
+    "trading_client = TradingClient(\n",
+    "    api_key=os.getenv(\"ALPACA_API_KEY\"),\n",
+    "    secret_key=os.getenv(\"ALPACA_SECRET_KEY\"),\n",
+    "    paper=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "49b14380",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "No open positions.\n"
+     ]
+    }
+   ],
+   "source": [
+    "positions = trading_client.get_all_positions()\n",
+    "\n",
+    "if not positions:\n",
+    "    print(\"No open positions.\")\n",
+    "else:\n",
+    "    print(\"Current Portfolio Positions:\")\n",
+    "    for pos in positions:\n",
+    "        print(\n",
+    "            f\"Symbol: {pos.symbol:<5} | \"\n",
+    "            f\"Qty: {pos.qty:<5} | \"\n",
+    "            f\"Avg Entry Price: ${float(pos.avg_entry_price):.2f} | \"\n",
+    "            f\"Current Price: ${float(pos.current_price):.2f} | \"\n",
+    "            f\"Unrealized P/L: ${float(pos.unrealized_pl):.2f}\"\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "c032239c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "99997.95 99997.95\n"
+     ]
+    }
+   ],
+   "source": [
+    "account = trading_client.get_account()\n",
+    "print(\n",
+    "    account.cash,\n",
+    "    account.equity\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "c759e40f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "No open orders found.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from alpaca.trading.requests import GetOrdersRequest\n",
+    "from alpaca.trading.enums import QueryOrderStatus\n",
+    "\n",
+    "# Request only open orders\n",
+    "request_params = GetOrdersRequest(status=QueryOrderStatus.OPEN)\n",
+    "open_orders = trading_client.get_orders(filter=request_params)\n",
+    "\n",
+    "if not open_orders:\n",
+    "    print(\"No open orders found.\")\n",
+    "else:\n",
+    "    print(f\"Found {len(open_orders)} open order(s):\")\n",
+    "    for order in open_orders:\n",
+    "        print(\n",
+    "            f\"ID: {order.id} | Symbol: {order.symbol} | \"\n",
+    "            f\"Side: {order.side} | Qty: {order.qty} | Status: {order.status}\"\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "091cf061",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Submitted Order ID: 4838c16d-7a12-4dc3-80cb-7c5d1dbecda3 | Status: OrderStatus.ACCEPTED\n"
+     ]
+    }
+   ],
+   "source": [
+    "from alpaca.trading.requests import MarketOrderRequest\n",
+    "from alpaca.trading.enums import OrderSide, TimeInForce\n",
+    "\n",
+    "# Define a market buy order for 10 shares of SPY\n",
+    "market_order_data = MarketOrderRequest(\n",
+    "    symbol=\"SPY\",\n",
+    "    notional=100.0,\n",
+    "    side=OrderSide.BUY,\n",
+    "    time_in_force=TimeInForce.DAY,\n",
+    ")\n",
+    "\n",
+    "# Submit the order\n",
+    "order = trading_client.submit_order(order_data=market_order_data)\n",
+    "\n",
+    "print(f\"Submitted Order ID: {order.id} | Status: {order.status}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
@@ -0,0 +1,308 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "038bf2ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import warnings\n",
+    "from pathlib import Path\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Source of truth for model inputs (must match training order exactly)\n",
+    "FEATURE_COLUMNS = [\n",
+    "    \"SPY_ret_5\",\n",
+    "    \"SPY_ret_20\",\n",
+    "    \"SPY_dist_sma50\",\n",
+    "    \"VIX_change_5\",\n",
+    "    \"VIX_rank_20\",\n",
+    "    \"TLT_ret_10\",\n",
+    "    \"USO_ret_5\",\n",
+    "    \"SPY_TLT_ratio_ret\",\n",
+    "]\n",
+    "\n",
+    "DEFAULT_SYMBOLS = {\n",
+    "    \"SPY\": \"SPY\",\n",
+    "    \"VIXY\": \"VIX\",  # Change to \"VIX\": \"VIX\" if using raw VIX Parquet\n",
+    "    \"TLT\": \"TLT\",\n",
+    "    \"USO\": \"USO\",\n",
+    "}\n",
+    "\n",
+    "\n",
+    "def load_raw_close_prices(\n",
+    "    raw_data_dir: Path, symbols: dict[str, str]\n",
+    ") -> pd.DataFrame:\n",
+    "    \"\"\"Reads raw Parquet files and merges close prices into a single inner-joined DataFrame.\"\"\"\n",
+    "    frames = []\n",
+    "    for symbol, alias in symbols.items():\n",
+    "        path = raw_data_dir / f\"{symbol}.parquet\"\n",
+    "        if not path.exists():\n",
+    "            raise FileNotFoundError(f\"Missing raw data file: {path}\")\n",
+    "\n",
+    "        frame = pd.read_parquet(path)\n",
+    "        if \"date\" in frame.columns:\n",
+    "            frame = frame.set_index(\"date\")\n",
+    "\n",
+    "        frame.index = pd.to_datetime(frame.index)\n",
+    "        frame = frame.sort_index()\n",
+    "        frames.append(frame[[\"close\"]].rename(columns={\"close\": f\"{alias}_close\"}))\n",
+    "\n",
+    "    return pd.concat(frames, axis=1, join=\"inner\")\n",
+    "\n",
+    "\n",
+    "def compute_features(prices: pd.DataFrame) -> pd.DataFrame:\n",
+    "    \"\"\"Computes engineered features from raw merged price history.\"\"\"\n",
+    "    df = prices.copy()\n",
+    "\n",
+    "    # Target asset features\n",
+    "    df[\"SPY_ret_5\"] = df[\"SPY_close\"].pct_change(5)\n",
+    "    df[\"SPY_ret_20\"] = df[\"SPY_close\"].pct_change(20)\n",
+    "\n",
+    "    sma_50 = df[\"SPY_close\"].rolling(50).mean()\n",
+    "    df[\"SPY_dist_sma50\"] = (df[\"SPY_close\"] - sma_50) / sma_50\n",
+    "\n",
+    "    # Volatility / Market Stress\n",
+    "    df[\"VIX_change_5\"] = df[\"VIX_close\"].pct_change(5)\n",
+    "    df[\"VIX_rank_20\"] = df[\"VIX_close\"].rolling(20).rank(pct=True)\n",
+    "\n",
+    "    # Macro & Relative ratios\n",
+    "    df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
+    "    df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
+    "    df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
+    "\n",
+    "    return df[FEATURE_COLUMNS]\n",
+    "\n",
+    "\n",
+    "def get_latest_inference_features(\n",
+    "    raw_data_dir: Path,\n",
+    "    symbols: dict[str, str] | None = None,\n",
+    "    max_age_days: int = 1,\n",
+    ") -> pd.DataFrame:\n",
+    "    \"\"\"Loads raw prices, computes features, verifies date freshness,\n",
+    "\n",
+    "    and returns the latest single row for model prediction.\n",
+    "    \"\"\"\n",
+    "    symbols = symbols or DEFAULT_SYMBOLS\n",
+    "\n",
+    "    # 1. Load prices & compute rolling features\n",
+    "    prices = load_raw_close_prices(raw_data_dir, symbols)\n",
+    "    features = compute_features(prices).dropna()\n",
+    "\n",
+    "    if features.empty:\n",
+    "        raise ValueError(\n",
+    "            \"Not enough historical rows to compute 50-day rolling window features.\"\n",
+    "        )\n",
+    "\n",
+    "    # 2. Extract latest available row as a 1-row DataFrame\n",
+    "    latest_row = features.iloc[[-1]]\n",
+    "    latest_date = latest_row.index[0]\n",
+    "\n",
+    "    # 3. Check data freshness and raise a warning if stale\n",
+    "    now = pd.Timestamp.now()\n",
+    "    latest_date_naive = (\n",
+    "        latest_date.tz_localize(None)\n",
+    "        if latest_date.tz is not None\n",
+    "        else latest_date\n",
+    "    )\n",
+    "    days_old = (now.floor(\"D\") - latest_date_naive.floor(\"D\")).days\n",
+    "\n",
+    "    if days_old > max_age_days:\n",
+    "        warnings.warn(\n",
+    "            f\"STALE DATA WARNING: Latest feature row is from {latest_date.strftime('%Y-%m-%d')} \"\n",
+    "            f\"({days_old} day(s) old). Update raw Parquet files before executing trades.\",\n",
+    "            UserWarning,\n",
+    "            stacklevel=2,\n",
+    "        )\n",
+    "\n",
+    "    return latest_row\n",
+    "\n",
+    "def get_target_exposure(\n",
+    "    p_pred: float, p_base: float, sensitivity: float = 5.0\n",
+    ") -> float:\n",
+    "    \"\"\"Maps predicted probability to a target portfolio equity allocation (0.0 to 1.0).\n",
+    "\n",
+    "    - p_pred == p_base  --> 50% Target Exposure (Neutral)\n",
+    "    - p_pred > p_base   --> Scale up toward 100% (Bullish)\n",
+    "    - p_pred < p_base   --> Scale down toward 0% (Bearish / Cash)\n",
+    "    \"\"\"\n",
+    "    # Calculate deviation from the historical average\n",
+    "    delta = p_pred - p_base\n",
+    "\n",
+    "    # Base target allocation is 50% equity / 50% cash\n",
+    "    base_allocation = 0.50\n",
+    "\n",
+    "    # Sensitivity controls how aggressively probability changes alter allocation\n",
+    "    # e.g., a +0.08 delta * 5.0 = +0.40 -> 90% Equity Allocation\n",
+    "    target_allocation = base_allocation + (delta * sensitivity)\n",
+    "\n",
+    "    # Clamp bounds strictly between 0% (full cash) and 100% (full SPY)\n",
+    "    return float(np.clip(target_allocation, 0.0, 1.0))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "0b3d2c25",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Date: 2026-07-27 | Prob: 0.5843 | Base: 0.5830\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.5064256139268726"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import xgboost as xgb\n",
+    "\n",
+    "# 1. Load model and metadata\n",
+    "model = xgb.XGBClassifier()\n",
+    "model.load_model(\"models/spy_xgb_v1.json\")\n",
+    "\n",
+    "with open(\"models/spy_xgb_v1_meta.json\", \"r\") as f:\n",
+    "    meta = json.load(f)\n",
+    "\n",
+    "# 2. Fetch latest features from raw parquet files\n",
+    "RAW_DATA_DIR = Path(\"../data/ibkr/daily\")\n",
+    "X_latest = get_latest_inference_features(RAW_DATA_DIR, max_age_days=1)\n",
+    "\n",
+    "# 3. Predict probability\n",
+    "p_pred = float(model.predict_proba(X_latest[meta[\"feature_cols\"]])[0, 1])\n",
+    "p_base = meta[\"p_base\"]\n",
+    "\n",
+    "print(\n",
+    "    f\"Date: {X_latest.index[0].date()} | Prob: {p_pred:.4f} | Base: {p_base:.4f}\"\n",
+    ")\n",
+    "\n",
+    "get_target_exposure(p_pred, p_base, sensitivity=5.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3f823fa6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>SPY_ret_5</th>\n",
+       "      <th>SPY_ret_20</th>\n",
+       "      <th>SPY_dist_sma50</th>\n",
+       "      <th>VIX_change_5</th>\n",
+       "      <th>VIX_rank_20</th>\n",
+       "      <th>TLT_ret_10</th>\n",
+       "      <th>USO_ret_5</th>\n",
+       "      <th>SPY_TLT_ratio_ret</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>date</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2026-07-27</th>\n",
+       "      <td>-0.004043</td>\n",
+       "      <td>0.013855</td>\n",
+       "      <td>-0.007938</td>\n",
+       "      <td>0.007065</td>\n",
+       "      <td>0.75</td>\n",
+       "      <td>-0.00262</td>\n",
+       "      <td>-0.005976</td>\n",
+       "      <td>-0.002378</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            SPY_ret_5  SPY_ret_20  SPY_dist_sma50  VIX_change_5  VIX_rank_20  \\\n",
+       "date                                                                           \n",
+       "2026-07-27  -0.004043    0.013855       -0.007938      0.007065         0.75   \n",
+       "\n",
+       "            TLT_ret_10  USO_ret_5  SPY_TLT_ratio_ret  \n",
+       "date                                                  \n",
+       "2026-07-27    -0.00262  -0.005976          -0.002378  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_latest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "476fd7fd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
@@ -17,7 +17,26 @@
   "execution_count": null,
   "id": "543426a2",
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Connecting to IBKR Gateway at 127.0.0.1:4002 with client id 101...\n",
+      "Failed to connect to IBKR Gateway: This event loop is already running\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Error 200, reqId 9: No security definition has been found for the request, contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n",
+      "Error 200, reqId 10: No security definition has been found for the request\n",
+      "Canceled order: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')\n",
+      "Peer closed connection.\n"
+     ]
+    }
+   ],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
@@ -35,36 +54,55 @@
    "if str(src_path) not in sys.path:\n",
    "    sys.path.insert(0, str(src_path))\n",
    "\n",
-    "from trading_bot.data.fetch_ibkr_daily import (\n",
-    "    IBKR_CLIENT_ID,\n",
-    "    IBKR_CONNECT_TIMEOUT_SECONDS,\n",
-    "    IBKR_HOST,\n",
-    "    IBKR_PORT,\n",
-    ")\n",
+    "IBKR_HOST = \"127.0.0.1\"\n",
+    "IBKR_PORT = 4002\n",
+    "IBKR_CLIENT_ID = 101\n",
+    "IBKR_CONNECT_TIMEOUT_SECONDS = 10\n",
    "\n",
    "ib = IB()\n",
    "print(f\"Connecting to IBKR Gateway at {IBKR_HOST}:{IBKR_PORT} with client id {IBKR_CLIENT_ID}...\")\n",
-    "ib.connect(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID, timeout=IBKR_CONNECT_TIMEOUT_SECONDS)\n",
-    "print(\"Connection established.\")\n"
+    "try:\n",
+    "    ib.connect(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID, timeout=IBKR_CONNECT_TIMEOUT_SECONDS)\n",
+    "    print(\"Connection established.\")\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed to connect to IBKR Gateway: {e}\")\n",
+    "    ib.disconnect()\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
   "id": "ecdc721f",
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Connected: True\n",
+      "Client ID: 101\n"
+     ]
+    }
+   ],
   "source": [
    "print(f\"Connected: {ib.isConnected()}\")\n",
-    "print(f\"Client ID: {ib.clientId}\")\n"
+    "print(f\"Client ID: {ib.client.clientId}\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
   "id": "10ebc240",
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "No open portfolio positions were returned.\n"
+     ]
+    }
+   ],
   "source": [
    "portfolio = ib.portfolio()\n",
    "if not portfolio:\n",
@@ -79,27 +117,110 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
   "id": "fed654ec",
   "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Unknown contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Submitted order for VUAA: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='PendingSubmit', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0)], advancedError='')\n"
+     ]
+    }
+   ],
   "source": [
    "from ib_insync import MarketOrder\n",
    "\n",
-    "contract = Stock(\"SPY\", \"SMART\", \"USD\")\n",
-    "ib.qualifyContracts(contract)\n",
+    "contract = Stock(\"VUAA\", \"SMART\", \"USD\")\n",
+    "await ib.qualifyContractsAsync(contract)\n",
    "\n",
    "# Adjust the quantity as needed before running this cell.\n",
-    "order = MarketOrder(\"BUY\", 1)\n",
+    "order = MarketOrder(\"BUY\", 0.70)\n",
    "trade = ib.placeOrder(contract, order)\n",
    "\n",
    "print(f\"Submitted order for {contract.symbol}: {trade}\")\n"
   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "06881101",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trds = ib.trades()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "08663602",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Cancelled', 'Cancelled', 'Cancelled']"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list(map(lambda x: x.orderStatus.status, trds))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "2c7abf23",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=1578393268, action='BUY', totalQuantity=1.0, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n",
+       " Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=24474350, action='BUY', totalQuantity=0.1345, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n",
+       " Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')]"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trds"
+   ]
  }
 ],
 "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
  "language_info": {
-   "name": "python"
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
  }
 },
 "nbformat": 4,
@@ -1,5 +1,5 @@
 {
-  "p_base": 0.5833333333333334,
+  "p_base": 0.5819477434679335,
  "feature_cols": [
    "VIX_rank_20",
    "TLT_ret_10",
@@ -9,5 +9,5 @@
    "SPY_ret_20",
    "SPY_dist_sma50"
  ],
-  "last_trained_date": "2026-07-24 00:00:00"
+  "last_trained_date": "2026-07-29 00:00:00"
 }
@@ -20,14 +20,14 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PosixPath('/home/jarno/repos/trading-bot'),\n",
-       " PosixPath('/home/jarno/repos/trading-bot/data/ibkr/daily'),\n",
+       " PosixPath('/home/jarno/repos/trading-bot/data/alpaca/daily'),\n",
       " PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'))"
      ]
     },
@@ -87,16 +87,16 @@
     "output_type": "stream",
     "text": [
      "<class 'pandas.DataFrame'>\n",
-      "DatetimeIndex: 1251 entries, 2021-07-30 to 2026-07-24\n",
+      "DatetimeIndex: 1255 entries, 2021-08-02 to 2026-07-31\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype  \n",
      "---  ------     --------------  -----  \n",
-      " 0   SPY_close  1251 non-null   float64\n",
-      " 1   VIX_close  1251 non-null   float64\n",
-      " 2   TLT_close  1251 non-null   float64\n",
-      " 3   USO_close  1251 non-null   float64\n",
+      " 0   SPY_close  1255 non-null   float64\n",
+      " 1   VIX_close  1255 non-null   float64\n",
+      " 2   TLT_close  1255 non-null   float64\n",
+      " 3   USO_close  1255 non-null   float64\n",
      "dtypes: float64(4)\n",
-      "memory usage: 48.9 KB\n"
+      "memory usage: 49.0 KB\n"
     ]
    },
    {
@@ -135,40 +135,40 @@
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
-       "      <th>2021-07-30</th>\n",
-       "      <td>438.51</td>\n",
-       "      <td>495.4</td>\n",
-       "      <td>149.52</td>\n",
-       "      <td>50.66</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
       "      <th>2021-08-02</th>\n",
       "      <td>437.59</td>\n",
-       "      <td>513.6</td>\n",
+       "      <td>25.68</td>\n",
       "      <td>150.67</td>\n",
       "      <td>49.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-03</th>\n",
       "      <td>441.15</td>\n",
-       "      <td>488.0</td>\n",
+       "      <td>24.40</td>\n",
       "      <td>150.75</td>\n",
       "      <td>48.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-04</th>\n",
       "      <td>438.98</td>\n",
-       "      <td>487.6</td>\n",
+       "      <td>24.38</td>\n",
       "      <td>151.06</td>\n",
       "      <td>47.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-05</th>\n",
       "      <td>441.76</td>\n",
-       "      <td>474.8</td>\n",
+       "      <td>23.74</td>\n",
       "      <td>150.29</td>\n",
       "      <td>48.10</td>\n",
       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-08-06</th>\n",
+       "      <td>442.49</td>\n",
+       "      <td>23.14</td>\n",
+       "      <td>147.78</td>\n",
+       "      <td>47.57</td>\n",
+       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
@@ -176,11 +176,11 @@
      "text/plain": [
       "            SPY_close  VIX_close  TLT_close  USO_close\n",
       "date                                                  \n",
-       "2021-07-30     438.51      495.4     149.52      50.66\n",
-       "2021-08-02     437.59      513.6     150.67      49.18\n",
-       "2021-08-03     441.15      488.0     150.75      48.85\n",
-       "2021-08-04     438.98      487.6     151.06      47.20\n",
-       "2021-08-05     441.76      474.8     150.29      48.10"
+       "2021-08-02     437.59      25.68     150.67      49.18\n",
+       "2021-08-03     441.15      24.40     150.75      48.85\n",
+       "2021-08-04     438.98      24.38     151.06      47.20\n",
+       "2021-08-05     441.76      23.74     150.29      48.10\n",
+       "2021-08-06     442.49      23.14     147.78      47.57"
      ]
     },
     "execution_count": 2,
@@ -234,7 +234,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
@@ -283,24 +283,12 @@
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
-       "      <th>2021-10-08</th>\n",
-       "      <td>0.008336</td>\n",
-       "      <td>-0.017017</td>\n",
-       "      <td>-0.011155</td>\n",
-       "      <td>-0.075239</td>\n",
-       "      <td>0.125</td>\n",
-       "      <td>-0.034239</td>\n",
-       "      <td>0.041495</td>\n",
-       "      <td>0.032998</td>\n",
-       "      <td>1.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
       "      <th>2021-10-11</th>\n",
       "      <td>0.014114</td>\n",
       "      <td>-0.026625</td>\n",
       "      <td>-0.018145</td>\n",
       "      <td>-0.090869</td>\n",
-       "      <td>0.200</td>\n",
+       "      <td>0.20</td>\n",
       "      <td>-0.033135</td>\n",
       "      <td>0.031015</td>\n",
       "      <td>0.038908</td>\n",
@@ -312,7 +300,7 @@
       "      <td>-0.023752</td>\n",
       "      <td>-0.020386</td>\n",
       "      <td>-0.074091</td>\n",
-       "      <td>0.100</td>\n",
+       "      <td>0.10</td>\n",
       "      <td>-0.001041</td>\n",
       "      <td>0.008628</td>\n",
       "      <td>-0.001303</td>\n",
@@ -324,7 +312,7 @@
       "      <td>-0.028356</td>\n",
       "      <td>-0.016597</td>\n",
       "      <td>-0.081006</td>\n",
-       "      <td>0.050</td>\n",
+       "      <td>0.05</td>\n",
       "      <td>0.006928</td>\n",
       "      <td>0.036928</td>\n",
       "      <td>-0.005897</td>\n",
@@ -336,12 +324,24 @@
       "      <td>-0.010443</td>\n",
       "      <td>-0.000214</td>\n",
       "      <td>-0.096759</td>\n",
-       "      <td>0.050</td>\n",
+       "      <td>0.05</td>\n",
       "      <td>0.010809</td>\n",
       "      <td>0.026192</td>\n",
       "      <td>-0.011991</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021-10-15</th>\n",
+       "      <td>0.018294</td>\n",
+       "      <td>0.010127</td>\n",
+       "      <td>0.007213</td>\n",
+       "      <td>-0.079882</td>\n",
+       "      <td>0.05</td>\n",
+       "      <td>-0.002202</td>\n",
+       "      <td>0.030287</td>\n",
+       "      <td>-0.003823</td>\n",
+       "      <td>1.0</td>\n",
+       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
@@ -349,22 +349,22 @@
      "text/plain": [
       "            SPY_ret_5  SPY_ret_20  SPY_dist_sma50  VIX_change_5  VIX_rank_20  \\\n",
       "date                                                                           \n",
-       "2021-10-08   0.008336   -0.017017       -0.011155     -0.075239        0.125   \n",
-       "2021-10-11   0.014114   -0.026625       -0.018145     -0.090869        0.200   \n",
-       "2021-10-12   0.001201   -0.023752       -0.020386     -0.074091        0.100   \n",
-       "2021-10-13   0.000644   -0.028356       -0.016597     -0.081006        0.050   \n",
-       "2021-10-14   0.008754   -0.010443       -0.000214     -0.096759        0.050   \n",
+       "2021-10-11   0.014114   -0.026625       -0.018145     -0.090869         0.20   \n",
+       "2021-10-12   0.001201   -0.023752       -0.020386     -0.074091         0.10   \n",
+       "2021-10-13   0.000644   -0.028356       -0.016597     -0.081006         0.05   \n",
+       "2021-10-14   0.008754   -0.010443       -0.000214     -0.096759         0.05   \n",
+       "2021-10-15   0.018294    0.010127        0.007213     -0.079882         0.05   \n",
       "\n",
       "            TLT_ret_10  USO_ret_5  SPY_TLT_ratio_ret  spy_up_5d  \n",
       "date                                                             \n",
-       "2021-10-08   -0.034239   0.041495           0.032998        1.0  \n",
       "2021-10-11   -0.033135   0.031015           0.038908        1.0  \n",
       "2021-10-12   -0.001041   0.008628          -0.001303        1.0  \n",
       "2021-10-13    0.006928   0.036928          -0.005897        1.0  \n",
-       "2021-10-14    0.010809   0.026192          -0.011991        1.0  "
+       "2021-10-14    0.010809   0.026192          -0.011991        1.0  \n",
+       "2021-10-15   -0.002202   0.030287          -0.003823        1.0  "
      ]
     },
-     "execution_count": 3,
+     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -384,7 +384,7 @@
    "df[\"TLT_ret_10\"] = df[\"TLT_close\"].pct_change(10)\n",
    "df[\"USO_ret_5\"] = df[\"USO_close\"].pct_change(5)\n",
    "df[\"SPY_TLT_ratio_ret\"] = (df[\"SPY_close\"] / df[\"TLT_close\"]).pct_change(5)\n",
-    "df_model\n",
+    "\n",
    "spy_forward_close = df[\"SPY_close\"].shift(-5)\n",
    "df[\"spy_up_5d\"] = np.nan\n",
    "df.loc[spy_forward_close > df[\"SPY_close\"], \"spy_up_5d\"] = 1.0\n",
@@ -418,7 +418,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
@@ -458,24 +458,24 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
-       "      <td>837</td>\n",
-       "      <td>2021-10-08</td>\n",
-       "      <td>2025-02-07</td>\n",
-       "      <td>0.583035</td>\n",
+       "      <td>840</td>\n",
+       "      <td>2021-10-11</td>\n",
+       "      <td>2025-02-13</td>\n",
+       "      <td>0.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation</th>\n",
-       "      <td>179</td>\n",
-       "      <td>2025-02-10</td>\n",
-       "      <td>2025-10-24</td>\n",
-       "      <td>0.653631</td>\n",
+       "      <td>180</td>\n",
+       "      <td>2025-02-14</td>\n",
+       "      <td>2025-10-31</td>\n",
+       "      <td>0.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>181</td>\n",
-       "      <td>2025-10-27</td>\n",
-       "      <td>2026-07-17</td>\n",
-       "      <td>0.558011</td>\n",
+       "      <td>2025-11-03</td>\n",
+       "      <td>2026-07-24</td>\n",
+       "      <td>0.569061</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
@@ -484,12 +484,12 @@
      "text/plain": [
       "            rows  start_date    end_date  target_mean\n",
       "split                                                \n",
-       "train        837  2021-10-08  2025-02-07     0.583035\n",
-       "validation   179  2025-02-10  2025-10-24     0.653631\n",
-       "test         181  2025-10-27  2026-07-17     0.558011"
+       "train        840  2021-10-11  2025-02-13     0.583333\n",
+       "validation   180  2025-02-14  2025-10-31     0.633333\n",
+       "test         181  2025-11-03  2026-07-24     0.569061"
      ]
     },
-     "execution_count": 4,
+     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -543,17 +543,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Rows: 1,197\n",
+      "Rows: 1,201\n",
      "Feature columns: ['SPY_ret_5', 'SPY_ret_20', 'SPY_dist_sma50', 'VIX_change_5', 'VIX_rank_20', 'TLT_ret_10', 'USO_ret_5', 'SPY_TLT_ratio_ret']\n",
      "Target column: spy_up_5d\n",
-      "Training matrix shape: (837, 8)\n"
+      "Training matrix shape: (840, 8)\n"
     ]
    },
    {
@@ -592,16 +592,16 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197.000000</td>\n",
-       "      <td>1197</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201.000000</td>\n",
+       "      <td>1201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
@@ -640,32 +640,32 @@
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>837</td>\n",
+       "      <td>840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
-       "      <td>0.002556</td>\n",
-       "      <td>0.009841</td>\n",
-       "      <td>0.011067</td>\n",
-       "      <td>-0.008646</td>\n",
-       "      <td>0.387009</td>\n",
-       "      <td>-0.004083</td>\n",
-       "      <td>0.004662</td>\n",
-       "      <td>0.004927</td>\n",
-       "      <td>0.589808</td>\n",
+       "      <td>0.002502</td>\n",
+       "      <td>0.009848</td>\n",
+       "      <td>0.011028</td>\n",
+       "      <td>0.017763</td>\n",
+       "      <td>0.403476</td>\n",
+       "      <td>-0.004099</td>\n",
+       "      <td>0.005024</td>\n",
+       "      <td>0.004880</td>\n",
+       "      <td>0.588676</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
-       "      <td>0.023081</td>\n",
-       "      <td>0.043634</td>\n",
-       "      <td>0.037371</td>\n",
-       "      <td>0.092413</td>\n",
-       "      <td>0.333697</td>\n",
-       "      <td>0.028557</td>\n",
-       "      <td>0.052319</td>\n",
-       "      <td>0.027656</td>\n",
-       "      <td>0.492074</td>\n",
+       "      <td>0.023057</td>\n",
+       "      <td>0.043558</td>\n",
+       "      <td>0.037315</td>\n",
+       "      <td>0.299417</td>\n",
+       "      <td>0.337501</td>\n",
+       "      <td>0.028502</td>\n",
+       "      <td>0.052628</td>\n",
+       "      <td>0.027606</td>\n",
+       "      <td>0.492279</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
@@ -683,40 +683,40 @@
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
-       "      <td>-0.009535</td>\n",
-       "      <td>-0.017017</td>\n",
-       "      <td>-0.008847</td>\n",
-       "      <td>-0.058376</td>\n",
-       "      <td>0.075000</td>\n",
-       "      <td>-0.023517</td>\n",
-       "      <td>-0.026200</td>\n",
+       "      <td>-0.009601</td>\n",
+       "      <td>-0.016509</td>\n",
+       "      <td>-0.008656</td>\n",
+       "      <td>-0.056615</td>\n",
+       "      <td>0.100000</td>\n",
+       "      <td>-0.023304</td>\n",
+       "      <td>-0.025866</td>\n",
       "      <td>-0.010013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
-       "      <td>0.003869</td>\n",
-       "      <td>0.015797</td>\n",
-       "      <td>0.017906</td>\n",
-       "      <td>-0.020655</td>\n",
-       "      <td>0.250000</td>\n",
-       "      <td>-0.004339</td>\n",
-       "      <td>0.004817</td>\n",
-       "      <td>0.005710</td>\n",
+       "      <td>0.003798</td>\n",
+       "      <td>0.015725</td>\n",
+       "      <td>0.017802</td>\n",
+       "      <td>-0.018570</td>\n",
+       "      <td>0.300000</td>\n",
+       "      <td>-0.004435</td>\n",
+       "      <td>0.004950</td>\n",
+       "      <td>0.005537</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
-       "      <td>0.015957</td>\n",
-       "      <td>0.038385</td>\n",
-       "      <td>0.038332</td>\n",
-       "      <td>0.027778</td>\n",
-       "      <td>0.700000</td>\n",
-       "      <td>0.014272</td>\n",
-       "      <td>0.031883</td>\n",
-       "      <td>0.021406</td>\n",
+       "      <td>0.015955</td>\n",
+       "      <td>0.038179</td>\n",
+       "      <td>0.038282</td>\n",
+       "      <td>0.030814</td>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.014227</td>\n",
+       "      <td>0.032486</td>\n",
+       "      <td>0.021390</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
@@ -724,8 +724,8 @@
       "      <th>max</th>\n",
       "      <td>0.082843</td>\n",
       "      <td>0.157566</td>\n",
-       "      <td>0.088103</td>\n",
-       "      <td>0.937828</td>\n",
+       "      <td>0.088105</td>\n",
+       "      <td>3.846154</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.091322</td>\n",
       "      <td>0.327273</td>\n",
@@ -739,33 +739,33 @@
      ],
      "text/plain": [
       "          SPY_ret_5   SPY_ret_20  SPY_dist_sma50  VIX_change_5  VIX_rank_20  \\\n",
-       "count   1197.000000  1197.000000     1197.000000   1197.000000  1197.000000   \n",
+       "count   1201.000000  1201.000000     1201.000000   1201.000000  1201.000000   \n",
       "unique          NaN          NaN             NaN           NaN          NaN   \n",
       "top             NaN          NaN             NaN           NaN          NaN   \n",
       "freq            NaN          NaN             NaN           NaN          NaN   \n",
-       "mean       0.002556     0.009841        0.011067     -0.008646     0.387009   \n",
-       "std        0.023081     0.043634        0.037371      0.092413     0.333697   \n",
+       "mean       0.002502     0.009848        0.011028      0.017763     0.403476   \n",
+       "std        0.023057     0.043558        0.037315      0.299417     0.337501   \n",
       "min       -0.114962    -0.123975       -0.141995     -0.387709     0.050000   \n",
-       "25%       -0.009535    -0.017017       -0.008847     -0.058376     0.075000   \n",
-       "50%        0.003869     0.015797        0.017906     -0.020655     0.250000   \n",
-       "75%        0.015957     0.038385        0.038332      0.027778     0.700000   \n",
-       "max        0.082843     0.157566        0.088103      0.937828     1.000000   \n",
+       "25%       -0.009601    -0.016509       -0.008656     -0.056615     0.100000   \n",
+       "50%        0.003798     0.015725        0.017802     -0.018570     0.300000   \n",
+       "75%        0.015955     0.038179        0.038282      0.030814     0.750000   \n",
+       "max        0.082843     0.157566        0.088105      3.846154     1.000000   \n",
       "\n",
       "         TLT_ret_10    USO_ret_5  SPY_TLT_ratio_ret    spy_up_5d  split  \n",
-       "count   1197.000000  1197.000000        1197.000000  1197.000000   1197  \n",
+       "count   1201.000000  1201.000000        1201.000000  1201.000000   1201  \n",
       "unique          NaN          NaN                NaN          NaN      3  \n",
       "top             NaN          NaN                NaN          NaN  train  \n",
-       "freq            NaN          NaN                NaN          NaN    837  \n",
-       "mean      -0.004083     0.004662           0.004927     0.589808    NaN  \n",
-       "std        0.028557     0.052319           0.027656     0.492074    NaN  \n",
+       "freq            NaN          NaN                NaN          NaN    840  \n",
+       "mean      -0.004099     0.005024           0.004880     0.588676    NaN  \n",
+       "std        0.028502     0.052628           0.027606     0.492279    NaN  \n",
       "min       -0.092880    -0.196652          -0.117208     0.000000    NaN  \n",
-       "25%       -0.023517    -0.026200          -0.010013     0.000000    NaN  \n",
-       "50%       -0.004339     0.004817           0.005710     1.000000    NaN  \n",
-       "75%        0.014272     0.031883           0.021406     1.000000    NaN  \n",
+       "25%       -0.023304    -0.025866          -0.010013     0.000000    NaN  \n",
+       "50%       -0.004435     0.004950           0.005537     1.000000    NaN  \n",
+       "75%        0.014227     0.032486           0.021390     1.000000    NaN  \n",
       "max        0.091322     0.327273           0.129204     1.000000    NaN  "
      ]
     },
-     "execution_count": 5,
+     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -787,30 +787,30 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'),\n",
-       " (1197, 11),\n",
+       " (1201, 11),\n",
       "         date  SPY_ret_5  SPY_ret_20  SPY_dist_sma50  VIX_change_5  \\\n",
-       " 0 2021-10-08   0.008336   -0.017017       -0.011155     -0.075239   \n",
-       " 1 2021-10-11   0.014114   -0.026625       -0.018145     -0.090869   \n",
-       " 2 2021-10-12   0.001201   -0.023752       -0.020386     -0.074091   \n",
-       " 3 2021-10-13   0.000644   -0.028356       -0.016597     -0.081006   \n",
-       " 4 2021-10-14   0.008754   -0.010443       -0.000214     -0.096759   \n",
+       " 0 2021-10-11   0.014114   -0.026625       -0.018145     -0.090869   \n",
+       " 1 2021-10-12   0.001201   -0.023752       -0.020386     -0.074091   \n",
+       " 2 2021-10-13   0.000644   -0.028356       -0.016597     -0.081006   \n",
+       " 3 2021-10-14   0.008754   -0.010443       -0.000214     -0.096759   \n",
+       " 4 2021-10-15   0.018294    0.010127        0.007213     -0.079882   \n",
       " \n",
       "    VIX_rank_20  TLT_ret_10  USO_ret_5  SPY_TLT_ratio_ret  spy_up_5d  split  \n",
-       " 0        0.125   -0.034239   0.041495           0.032998        1.0  train  \n",
-       " 1        0.200   -0.033135   0.031015           0.038908        1.0  train  \n",
-       " 2        0.100   -0.001041   0.008628          -0.001303        1.0  train  \n",
-       " 3        0.050    0.006928   0.036928          -0.005897        1.0  train  \n",
-       " 4        0.050    0.010809   0.026192          -0.011991        1.0  train  )"
+       " 0         0.20   -0.033135   0.031015           0.038908        1.0  train  \n",
+       " 1         0.10   -0.001041   0.008628          -0.001303        1.0  train  \n",
+       " 2         0.05    0.006928   0.036928          -0.005897        1.0  train  \n",
+       " 3         0.05    0.010809   0.026192          -0.011991        1.0  train  \n",
+       " 4         0.05   -0.002202   0.030287          -0.003823        1.0  train  )"
      ]
     },
-     "execution_count": 6,
+     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -0,0 +1,255 @@
+"""Shared Alpaca daily candle helpers used by both the CLI fetcher and inference refreshes."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+from dotenv import load_dotenv
+
+# Load environment variables from .env file if available
+load_dotenv()
+
+DEFAULT_OUTPUT_DIR = Path("data/alpaca/daily")
+DEFAULT_DURATION = "1 W"
+EASTERN_TZ = ZoneInfo("America/New_York")
+DURATION_PATTERN = re.compile(r"^(\d+)\s*([DWMY])$")
+
+
+@dataclass(frozen=True)
+class DailyCandle:
+    """Daily OHLCV market data for one trading session."""
+
+    trading_day: date
+    open: float
+    high: float
+    low: float
+    close: float
+    volume: int
+
+
+def default_end_date() -> date:
+    """Return yesterday's date in the US/Eastern market timezone."""
+
+    return datetime.now(EASTERN_TZ).date() - timedelta(days=1)
+
+
+def current_market_date() -> date:
+    """Return today's date in the US/Eastern market timezone."""
+
+    return datetime.now(EASTERN_TZ).date()
+
+
+def parse_end_date(value: str) -> date:
+    """Parse an end date in YYYYMMDD format."""
+
+    try:
+        return datetime.strptime(value, "%Y%m%d").date()
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError(
+            "end date must use YYYYMMDD format, such as 20250605"
+        ) from exc
+
+
+def parse_duration(value: str) -> str:
+    """Normalize and validate a duration string like '1 W' or '1 M'."""
+
+    normalized = " ".join(value.upper().split())
+    if not DURATION_PATTERN.fullmatch(normalized):
+        raise argparse.ArgumentTypeError(
+            "duration must look like '1 D', '1 W', '1 M', or '1 Y'"
+        )
+    return normalized
+
+
+def duration_to_start_date(end_date: date, duration: str) -> date:
+    """Calculate the start date based on end date and a duration string."""
+
+    match = DURATION_PATTERN.match(duration)
+    if not match:
+        raise ValueError(f"Invalid duration format: {duration}")
+
+    amount = int(match.group(1))
+    unit = match.group(2)
+
+    if unit == "D":
+        return end_date - timedelta(days=amount)
+    if unit == "W":
+        return end_date - timedelta(weeks=amount)
+    if unit == "M":
+        return end_date - timedelta(days=amount * 30)
+    if unit == "Y":
+        return end_date - timedelta(days=amount * 365)
+
+    raise ValueError(f"Unsupported duration unit: {unit}")
+
+
+def fetch_daily_candles(
+    symbol: str,
+    end_date: date,
+    duration: str,
+    api_key: str | None = None,
+    secret_key: str | None = None,
+) -> list[DailyCandle]:
+    """Fetch daily candles for a symbol using Alpaca Market Data API."""
+
+    try:
+        from alpaca.data.historical import StockHistoricalDataClient
+        from alpaca.data.requests import StockBarsRequest
+        from alpaca.data.timeframe import TimeFrame
+    except ImportError as exc:
+        raise SystemExit(
+            "Missing dependency: alpaca-py. Install it via 'pip install alpaca-py' "
+            "before running the script."
+        ) from exc
+
+    start_date = duration_to_start_date(end_date, duration)
+    start_dt = datetime.combine(start_date, time.min, tzinfo=EASTERN_TZ)
+    end_dt = datetime.combine(end_date, time.max, tzinfo=EASTERN_TZ)
+
+    client = StockHistoricalDataClient(api_key=api_key, secret_key=secret_key)
+
+    request_params = StockBarsRequest(
+        symbol_or_symbols=symbol,
+        timeframe=TimeFrame.Day,
+        start=start_dt,
+        end=end_dt,
+    )
+
+    bars = client.get_stock_bars(request_params)
+
+    if not bars or symbol not in bars.data:
+        print(f"Alpaca returned no historical bars for {symbol}.")
+        return []
+
+    candles: list[DailyCandle] = []
+    for bar in bars[symbol]:
+        trading_day = bar.timestamp.astimezone(EASTERN_TZ).date()
+        candles.append(
+            DailyCandle(
+                trading_day=trading_day,
+                open=float(bar.open),
+                high=float(bar.high),
+                low=float(bar.low),
+                close=float(bar.close),
+                volume=int(bar.volume),
+            )
+        )
+
+    return candles
+
+
+def candles_to_frame(symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
+    """Convert candles to a date-indexed dataframe ready for Parquet storage."""
+
+    rows = [
+        {
+            "date": candle.trading_day,
+            "symbol": symbol,
+            "open": candle.open,
+            "high": candle.high,
+            "low": candle.low,
+            "close": candle.close,
+            "volume": candle.volume,
+        }
+        for candle in candles
+    ]
+    frame = pd.DataFrame.from_records(rows)
+    if frame.empty:
+        return pd.DataFrame(
+            columns=["symbol", "open", "high", "low", "close", "volume"],
+            index=pd.Index([], name="date"),
+        )
+
+    frame["date"] = pd.to_datetime(frame["date"]).dt.date
+    return frame.set_index("date")
+
+
+def read_existing_candles(path: Path) -> pd.DataFrame:
+    """Read an existing candle Parquet file as a date-indexed dataframe."""
+
+    if not path.exists():
+        return pd.DataFrame(
+            columns=["symbol", "open", "high", "low", "close", "volume"],
+            index=pd.Index([], name="date"),
+        )
+
+    frame = pd.read_parquet(path)
+    if "date" in frame.columns:
+        frame["date"] = pd.to_datetime(frame["date"]).dt.date
+        frame = frame.set_index("date")
+
+    frame.index = pd.to_datetime(frame.index).date
+    frame.index.name = "date"
+    return frame
+
+
+def oldest_stored_date_or_today(path: Path) -> date:
+    """Return the oldest stored candle date, or today if no data exists yet."""
+
+    existing = read_existing_candles(path)
+    if existing.empty:
+        return current_market_date()
+    return min(existing.index)
+
+
+def write_candles(path: Path, symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
+    """Append candles to a ticker Parquet file, keeping one row per date."""
+
+    existing = read_existing_candles(path)
+    fetched = candles_to_frame(symbol, candles)
+    combined = pd.concat([existing, fetched])
+    if not combined.empty:
+        combined = combined[~combined.index.duplicated(keep="last")]
+        combined = combined.sort_index()
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    combined.to_parquet(path, index=True)
+    return combined
+
+
+def print_candles(symbol: str, candles: list[DailyCandle]) -> None:
+    """Print candles in a compact table."""
+
+    print("date,symbol,open,high,low,close,volume")
+    for candle in candles:
+        print(
+            f"{candle.trading_day.isoformat()},"
+            f"{symbol},"
+            f"{candle.open:.2f},"
+            f"{candle.high:.2f},"
+            f"{candle.low:.2f},"
+            f"{candle.close:.2f},"
+            f"{candle.volume}"
+        )
+
+
+def refresh_recent_market_data(
+    output_dir: Path | str,
+    symbols: dict[str, str] | None = None,
+    api_key: str | None = None,
+    secret_key: str | None = None,
+    duration: str = DEFAULT_DURATION,
+) -> None:
+    """Refresh the latest weekly Alpaca candle parquet files for the provided symbols."""
+
+    output_dir = Path(output_dir)
+    symbols = symbols or {}
+
+    for symbol in symbols.keys():
+        output_path = output_dir / f"{symbol}.parquet"
+        candles = fetch_daily_candles(
+            symbol=symbol,
+            end_date=default_end_date(),
+            duration=duration,
+            api_key=api_key,
+            secret_key=secret_key,
+        )
+        print(f"Fetched {len(candles)} daily candles for {symbol}.")
+        write_candles(output_path, symbol, candles)
@@ -1,236 +1,31 @@
-"""Alpaca daily candle fetcher."""
+"""Alpaca daily candle fetcher CLI wrapper.
+
+The reusable implementation lives in the shared library module so the same
+fetching and parquet persistence logic can be reused by the prediction flow.
+"""

 from __future__ import annotations

 import argparse
 import os
-import re
-from dataclasses import dataclass
-from datetime import date, datetime, time, timedelta
 from pathlib import Path
-from zoneinfo import ZoneInfo

-import pandas as pd
-from dotenv import load_dotenv
-
-# Load environment variables from .env file if available
-load_dotenv()
-
-DEFAULT_OUTPUT_DIR = Path("data/alpaca/daily")
-DEFAULT_DURATION = "1 W"
-EASTERN_TZ = ZoneInfo("America/New_York")
-DURATION_PATTERN = re.compile(r"^(\d+)\s*([DWMY])$")
-
-
-@dataclass(frozen=True)
-class DailyCandle:
-    """Daily OHLCV market data for one trading session."""
-
-    trading_day: date
-    open: float
-    high: float
-    low: float
-    close: float
-    volume: int
-
-
-def default_end_date() -> date:
-    """Return yesterday's date in the US/Eastern market timezone."""
-
-    return datetime.now(EASTERN_TZ).date() - timedelta(days=1)
-
-
-def current_market_date() -> date:
-    """Return today's date in the US/Eastern market timezone."""
-
-    return datetime.now(EASTERN_TZ).date()
-
-
-def parse_end_date(value: str) -> date:
-    """Parse an end date in YYYYMMDD format."""
-
-    try:
-        return datetime.strptime(value, "%Y%m%d").date()
-    except ValueError as exc:
-        raise argparse.ArgumentTypeError(
-            "end date must use YYYYMMDD format, such as 20250605"
-        ) from exc
-
-
-def parse_duration(value: str) -> str:
-    """Normalize and validate a duration string like '1 W' or '1 M'."""
-
-    normalized = " ".join(value.upper().split())
-    if not DURATION_PATTERN.fullmatch(normalized):
-        raise argparse.ArgumentTypeError(
-            "duration must look like '1 D', '1 W', '1 M', or '1 Y'"
-        )
-    return normalized
-
-
-def duration_to_start_date(end_date: date, duration: str) -> date:
-    """Calculate the start date based on end date and a duration string."""
-
-    match = DURATION_PATTERN.match(duration)
-    if not match:
-        raise ValueError(f"Invalid duration format: {duration}")
-
-    amount = int(match.group(1))
-    unit = match.group(2)
-
-    if unit == "D":
-        return end_date - timedelta(days=amount)
-    if unit == "W":
-        return end_date - timedelta(weeks=amount)
-    if unit == "M":
-        return end_date - timedelta(days=amount * 30)
-    if unit == "Y":
-        return end_date - timedelta(days=amount * 365)
-
-    raise ValueError(f"Unsupported duration unit: {unit}")
-
-
-def fetch_daily_candles(
-    symbol: str,
-    end_date: date,
-    duration: str,
-    api_key: str | None = None,
-    secret_key: str | None = None,
-) -> list[DailyCandle]:
-    """Fetch daily candles for a symbol using Alpaca Market Data API."""
-
-    try:
-        from alpaca.data.historical import StockHistoricalDataClient
-        from alpaca.data.requests import StockBarsRequest
-        from alpaca.data.timeframe import TimeFrame
-    except ImportError as exc:
-        raise SystemExit(
-            "Missing dependency: alpaca-py. Install it via 'pip install alpaca-py' "
-            "before running the script."
-        ) from exc
-
-    start_date = duration_to_start_date(end_date, duration)
-
-    start_dt = datetime.combine(start_date, time.min, tzinfo=EASTERN_TZ)
-    end_dt = datetime.combine(end_date, time.max, tzinfo=EASTERN_TZ)
-
-    print(start_dt, end_dt)
-
-    client = StockHistoricalDataClient(api_key=api_key, secret_key=secret_key)
-
-    request_params = StockBarsRequest(
-        symbol_or_symbols=symbol,
-        timeframe=TimeFrame.Day,
-        start=start_dt,
-        end=end_dt,
-    )
-
-    bars = client.get_stock_bars(request_params)
-
-    if not bars or symbol not in bars.data:
-        print(f"Alpaca returned no historical bars for {symbol}.")
-        return []
-
-    candles: list[DailyCandle] = []
-    for bar in bars[symbol]:
-        trading_day = bar.timestamp.astimezone(EASTERN_TZ).date()
-        candles.append(
-            DailyCandle(
-                trading_day=trading_day,
-                open=float(bar.open),
-                high=float(bar.high),
-                low=float(bar.low),
-                close=float(bar.close),
-                volume=int(bar.volume),
-            )
-        )
-
-    return candles
-
-
-def candles_to_frame(symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
-    """Convert candles to a date-indexed dataframe ready for Parquet storage."""
-
-    rows = [
-        {
-            "date": candle.trading_day,
-            "symbol": symbol,
-            "open": candle.open,
-            "high": candle.high,
-            "low": candle.low,
-            "close": candle.close,
-            "volume": candle.volume,
-        }
-        for candle in candles
-    ]
-    frame = pd.DataFrame.from_records(rows)
-    if frame.empty:
-        return pd.DataFrame(
-            columns=["symbol", "open", "high", "low", "close", "volume"],
-            index=pd.Index([], name="date"),
-        )
-
-    frame["date"] = pd.to_datetime(frame["date"]).dt.date
-    return frame.set_index("date")
-
-
-def read_existing_candles(path: Path) -> pd.DataFrame:
-    """Read an existing candle Parquet file as a date-indexed dataframe."""
-
-    if not path.exists():
-        return pd.DataFrame(
-            columns=["symbol", "open", "high", "low", "close", "volume"],
-            index=pd.Index([], name="date"),
-        )
-
-    frame = pd.read_parquet(path)
-    if "date" in frame.columns:
-        frame["date"] = pd.to_datetime(frame["date"]).dt.date
-        frame = frame.set_index("date")
-
-    frame.index = pd.to_datetime(frame.index).date
-    frame.index.name = "date"
-    return frame
-
-
-def oldest_stored_date_or_today(path: Path) -> date:
-    """Return the oldest stored candle date, or today if no data exists yet."""
-
-    existing = read_existing_candles(path)
-    if existing.empty:
-        return current_market_date()
-    return min(existing.index)
-
-
-def write_candles(path: Path, symbol: str, candles: list[DailyCandle]) -> pd.DataFrame:
-    """Append candles to a ticker Parquet file, keeping one row per date."""
-
-    existing = read_existing_candles(path)
-    fetched = candles_to_frame(symbol, candles)
-    combined = pd.concat([existing, fetched])
-    if not combined.empty:
-        combined = combined[~combined.index.duplicated(keep="last")]
-        combined = combined.sort_index()
-
-    path.parent.mkdir(parents=True, exist_ok=True)
-    combined.to_parquet(path, index=True)
-    return combined
-
-
-def print_candles(symbol: str, candles: list[DailyCandle]) -> None:
-    """Print candles in a compact table."""
-
-    print("date,symbol,open,high,low,close,volume")
-    for candle in candles:
-        print(
-            f"{candle.trading_day.isoformat()},"
-            f"{symbol},"
-            f"{candle.open:.2f},"
-            f"{candle.high:.2f},"
-            f"{candle.low:.2f},"
-            f"{candle.close:.2f},"
-            f"{candle.volume}"
-        )
+from trading_bot.data.alpaca_daily_lib import (
+    DEFAULT_DURATION,
+    DEFAULT_OUTPUT_DIR,
+    DailyCandle,
+    candles_to_frame,
+    current_market_date,
+    default_end_date,
+    duration_to_start_date,
+    fetch_daily_candles,
+    oldest_stored_date_or_today,
+    parse_duration,
+    parse_end_date,
+    print_candles,
+    read_existing_candles,
+    write_candles,
+)


 def parse_args() -> argparse.Namespace:
@@ -292,8 +87,6 @@ def main() -> None:
        else args.end_date or default_end_date()
    )

-    print(oldest_stored_date_or_today(output_path))
-
    print(
        f"Fetching {symbol} daily candles ending {end_date:%Y-%m-%d} "
        f"for duration {args.duration} via Alpaca API"
@@ -305,7 +98,6 @@ def main() -> None:
        api_key=args.api_key,
        secret_key=args.secret_key,
    )
-    #print_candles(symbol, candles)
    stored = write_candles(output_path, symbol, candles)
    print(f"Wrote {len(stored)} total daily rows to {output_path}")

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Generate training dataset and train an XGBoost model using a JSON config.
+
+This script combines the logic from notebooks/spy_direction_dataset.ipynb
+and notebooks/train-xboost.ipynb into a single runnable script.
+"""
+from pathlib import Path
+import json
+import numpy as np
+import pandas as pd
+import xgboost as xgb
+from sklearn.metrics import classification_report, accuracy_score
+
+
+def find_project_root(start: Path | None = None) -> Path:
+    current = (start or Path.cwd()).resolve()
+    for candidate in [current, *current.parents]:
+        if (candidate / "pyproject.toml").exists():
+            return candidate
+    raise RuntimeError("Could not find project root containing pyproject.toml")
+
+
+def load_close(raw_dir: Path, symbol: str, alias: str | None = None) -> pd.DataFrame:
+    alias = alias or symbol
+    path = raw_dir / f"{symbol}.parquet"
+    if not path.exists():
+        raise FileNotFoundError(f"Missing raw data file: {path}")
+
+    frame = pd.read_parquet(path)
+    if "date" in frame.columns:
+        frame = frame.set_index("date")
+    if "close" not in frame.columns:
+        raise ValueError(f"{path} does not contain a close column")
+
+    frame = frame.copy()
+    frame.index = pd.to_datetime(frame.index)
+    frame.index.name = "date"
+    frame = frame.sort_index()
+    return frame[["close"]].rename(columns={"close": f"{alias}_close"})
+
+
+def assign_chronological_splits(frame: pd.DataFrame, train_fraction: float, validation_fraction: float, test_fraction: float) -> pd.Series:
+    total_fraction = train_fraction + validation_fraction + test_fraction
+    if not np.isclose(total_fraction, 1.0):
+        raise ValueError(f"Split fractions must sum to 1.0, got {total_fraction}")
+
+    n_rows = len(frame)
+    train_end = int(n_rows * train_fraction)
+    validation_end = train_end + int(n_rows * validation_fraction)
+
+    split = pd.Series(index=frame.index, dtype="object")
+    split.iloc[:train_end] = "train"
+    split.iloc[train_end:validation_end] = "validation"
+    split.iloc[validation_end:] = "test"
+    return split
+
+
+def build_dataset(raw_data_dir: Path, output_path: Path, symbols: dict, fractions: dict) -> pd.DataFrame:
+    prices = pd.concat(
+        [
+            load_close(raw_data_dir, symbols["SPY"], "SPY"),
+            load_close(raw_data_dir, symbols["VIX"] , "VIX"),
+            load_close(raw_data_dir, symbols["TLT"] , "TLT"),
+            load_close(raw_data_dir, symbols["USO"] , "USO"),
+        ],
+        axis=1,
+        join="inner",
+    )
+
+    df = prices.copy()
+    df["SPY_ret_5"] = df["SPY_close"].pct_change(5)
+    df["SPY_ret_20"] = df["SPY_close"].pct_change(20)
+
+    sma_50 = df["SPY_close"].rolling(50).mean()
+    df["SPY_dist_sma50"] = (df["SPY_close"] - sma_50) / sma_50
+
+    df["VIX_change_5"] = df["VIX_close"].pct_change(5)
+    df["VIX_rank_20"] = df["VIX_close"].rolling(20).rank(pct=True)
+
+    df["TLT_ret_10"] = df["TLT_close"].pct_change(10)
+    df["USO_ret_5"] = df["USO_close"].pct_change(5)
+    df["SPY_TLT_ratio_ret"] = (df["SPY_close"] / df["TLT_close"]).pct_change(5)
+
+    spy_forward_close = df["SPY_close"].shift(-5)
+    df["spy_up_5d"] = np.nan
+    df.loc[spy_forward_close > df["SPY_close"], "spy_up_5d"] = 1.0
+    df.loc[spy_forward_close < df["SPY_close"], "spy_up_5d"] = 0.0
+    df.loc[spy_forward_close == df["SPY_close"], "spy_up_5d"] = 0.5
+
+    FEATURE_COLUMNS = [
+        "SPY_ret_5",
+        "SPY_ret_20",
+        "SPY_dist_sma50",
+        "VIX_change_5",
+        "VIX_rank_20",
+        "TLT_ret_10",
+        "USO_ret_5",
+        "SPY_TLT_ratio_ret",
+    ]
+    TARGET_COLUMN = "spy_up_5d"
+
+    df_model = df[FEATURE_COLUMNS + [TARGET_COLUMN]].dropna().copy()
+
+    # Drop unchanged targets (0.5) to keep binary classification
+    df_model = df_model[df_model[TARGET_COLUMN] != 0.5].copy()
+    df_model[TARGET_COLUMN] = df_model[TARGET_COLUMN].astype(int)
+
+    df_model["split"] = assign_chronological_splits(
+        df_model,
+        fractions["train"],
+        fractions["validation"],
+        fractions["test"],
+    )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    dataset_to_save = df_model.reset_index()
+    dataset_to_save.to_parquet(output_path, index=False)
+
+    return df_model, FEATURE_COLUMNS, TARGET_COLUMN
+
+
+def train_model(df_model: pd.DataFrame, feature_cols: list, target_col: str, config: dict, models_dir: Path):
+    train_df = df_model[df_model["split"] == "train"]
+    val_df = df_model[df_model["split"] == "validation"]
+    test_df = df_model[df_model["split"] == "test"]
+
+    X_train, y_train = train_df[feature_cols], train_df[target_col]
+    X_val, y_val = val_df[feature_cols], val_df[target_col]
+    X_test, y_test = test_df[feature_cols], test_df[target_col]
+
+    clf_params = dict(config)
+    # Ensure early_stopping_rounds present as int
+
+    model = xgb.XGBClassifier(**clf_params)
+
+    eval_set = [(X_train, y_train), (X_val, y_val)]
+    fit_kwargs = {"eval_set": eval_set, "verbose": False}
+
+    model.fit(X_train, y_train, **fit_kwargs)
+
+    print(f"Best iteration: {getattr(model, 'best_iteration', None)}")
+    evals = model.evals_result()
+    if "validation_1" in evals and "logloss" in evals["validation_1"]:
+        val_loss = evals["validation_1"]["logloss"]
+        print(f"Starting Validation Loss: {val_loss[0]:.4f}")
+        print(f"Final Validation Loss:    {val_loss[-1]:.4f}")
+
+    # Evaluate on test set
+    y_pred = model.predict(X_test)
+    print("\n--- Final Holdout Test Performance ---")
+    print(f"Test Set Accuracy: {accuracy_score(y_test, y_pred):.2%}\n")
+    print(classification_report(y_test, y_pred))
+
+    # Save model and metadata
+    models_dir.mkdir(parents=True, exist_ok=True)
+    model_path = models_dir / "spy_xgb_v1.json"
+    meta_path = models_dir / "spy_xgb_v1_meta.json"
+    model.save_model(str(model_path))
+
+    metadata = {
+        "p_base": float(y_train.mean()),
+        "feature_cols": feature_cols,
+        "last_trained_date": str(df_model.index.max().date()),
+        "config": config,
+    }
+    with open(meta_path, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+    print(f"Saved model to {model_path}")
+
+
+def main():
+    project_root = find_project_root()
+
+    # Defaults
+    raw_data_dir = project_root / "data" / "alpaca" / "daily"
+    output_path = project_root / "data" / "training" / "spy_direction_5d.parquet"
+    models_dir = project_root / "models"
+    config_path = project_root / "config" / "train_config.json"
+
+    # Load config
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+    with open(config_path, "r") as f:
+        config = json.load(f)
+
+    symbols = config.get("symbols", {"SPY": "SPY", "VIX": config.get("vix_symbol", "VIXY"), "TLT": "TLT", "USO": "USO"})
+    fractions = config.get("fractions", {"train": 0.7, "validation": 0.15, "test": 0.15})
+
+    print("Building dataset...")
+    df_model, feature_cols, target_col = build_dataset(raw_data_dir, output_path, symbols, fractions)
+
+    print(f"Rows: {len(df_model):,}")
+    print(f"Feature columns: {feature_cols}")
+    print(f"Target column: {target_col}")
+
+    split_counts = df_model.groupby("split").size()
+    print("Split counts:")
+    print(split_counts.to_string())
+
+    print("\nTraining model...")
+    model_config = config.get("model", {})
+    train_model(df_model, feature_cols, target_col, model_config, models_dir)
+
+
+if __name__ == "__main__":
+    main()
@@ -21,6 +21,8 @@ import pandas as pd
 import xgboost as xgb
 from dotenv import load_dotenv

+from trading_bot.data.alpaca_daily_lib import refresh_recent_market_data
+
 # Load environment variables from .env file if available
 load_dotenv()

@@ -158,9 +160,20 @@ def predict_latest_probability(
    raw_data_dir: Path | str,
    symbols: dict[str, str] | None = None,
    max_age_days: int = 1,
+    api_key: str | None = None,
+    secret_key: str | None = None,
+    fetch_recent_data: bool = False,
 ) -> PredictionResult:
    """Return the latest predicted probability for the next SPY move."""

+    if fetch_recent_data:
+        refresh_recent_market_data(
+            output_dir=raw_data_dir,
+            symbols=symbols or DEFAULT_SYMBOLS,
+            api_key=api_key,
+            secret_key=secret_key,
+        )
+
    model = load_model(model_path)
    metadata = load_feature_metadata(metadata_path)
    latest_date, latest_features = get_latest_inference_features(
@@ -249,6 +262,9 @@ def main() -> None:
        model_path=args.model_path,
        metadata_path=args.metadata_path,
        raw_data_dir=args.data_dir,
+        api_key=args.api_key,
+        secret_key=args.secret_key,
+        fetch_recent_data=True,
    )
    print(
        f"Date: {result.prediction_date.date()} | Prob: {result.probability:.4f} | "
@@ -1,8 +1,72 @@
 from pathlib import Path

+import numpy as np
+import pandas as pd
+
 from trading_bot.models.prediction import FEATURE_COLUMNS, predict_latest_probability


+class FakeModel:
+    def predict_proba(self, model_input):
+        return np.array([[0.1, 0.9]])
+
+
+def test_predict_latest_probability_refreshes_recent_market_data(
+    monkeypatch,
+) -> None:
+    refresh_calls: dict[str, object] = {}
+
+    def fake_refresh_recent_data(
+        output_dir: Path,
+        symbols: dict[str, str] | None = None,
+        api_key: str | None = None,
+        secret_key: str | None = None,
+    ) -> None:
+        refresh_calls["symbols"] = list((symbols or {}).keys())
+        refresh_calls["output_dir"] = output_dir
+        refresh_calls["api_key"] = api_key
+        refresh_calls["secret_key"] = secret_key
+
+    monkeypatch.setattr(
+        "trading_bot.models.prediction.refresh_recent_market_data",
+        fake_refresh_recent_data,
+    )
+    monkeypatch.setattr(
+        "trading_bot.models.prediction.load_model",
+        lambda model_path: FakeModel(),
+    )
+    monkeypatch.setattr(
+        "trading_bot.models.prediction.load_feature_metadata",
+        lambda metadata_path: {"feature_cols": FEATURE_COLUMNS, "p_base": 0.5},
+    )
+    monkeypatch.setattr(
+        "trading_bot.models.prediction.get_latest_inference_features",
+        lambda raw_data_dir, symbols=None, max_age_days=1: (
+            pd.Timestamp("2026-08-03"),
+            pd.DataFrame(
+                [[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08]],
+                columns=FEATURE_COLUMNS,
+                index=[pd.Timestamp("2026-08-03")],
+            ),
+        ),
+    )
+
+    result = predict_latest_probability(
+        model_path=Path("notebooks/models/spy_xgb_v1.json"),
+        metadata_path=Path("notebooks/models/spy_xgb_v1_meta.json"),
+        raw_data_dir=Path("data/alpaca/daily"),
+        api_key="test-key",
+        secret_key="test-secret",
+        fetch_recent_data=True,
+    )
+
+    assert refresh_calls["output_dir"] == Path("data/alpaca/daily")
+    assert refresh_calls["symbols"] == ["SPY", "VIXY", "TLT", "USO"]
+    assert refresh_calls["api_key"] == "test-key"
+    assert refresh_calls["secret_key"] == "test-secret"
+    assert result.probability == 0.9
+
+
 def test_predict_latest_probability_returns_probability_between_zero_and_one() -> None:
    result = predict_latest_probability(
        model_path=Path("notebooks/models/spy_xgb_v1.json"),