diff --git a/notebooks/ibkr_scratch.ipynb b/notebooks/ibkr_scratch.ipynb deleted file mode 100644 index 49af44d..0000000 --- a/notebooks/ibkr_scratch.ipynb +++ /dev/null @@ -1,228 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "cd849b8e", - "metadata": {}, - "source": [ - "# IBKR scratch notebook\n", - "\n", - "This notebook is for quick manual testing of the IBKR gateway connection, portfolio lookup, and a simple SPY order flow.\n", - "\n", - "> Use this only with a paper-trading or test setup unless you explicitly intend to submit a live order.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "543426a2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connecting to IBKR Gateway at 127.0.0.1:4002 with client id 101...\n", - "Failed to connect to IBKR Gateway: This event loop is already running\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Error 200, reqId 9: No security definition has been found for the request, contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n", - "Error 200, reqId 10: No security definition has been found for the request\n", - "Canceled order: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')\n", - "Peer closed connection.\n" - ] - } - ], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "\n", - "from ib_insync import IB, Stock\n", - "\n", - "# Allow the notebook to import local source code from the repository.\n", - "repo_root = Path.cwd().resolve()\n", - "if (repo_root / \"src\").exists():\n", - " repo_root = repo_root\n", - "else:\n", - " repo_root = repo_root.parent\n", - "\n", - "src_path = repo_root / \"src\"\n", - "if str(src_path) not in sys.path:\n", - " sys.path.insert(0, str(src_path))\n", - "\n", - "IBKR_HOST = \"127.0.0.1\"\n", - "IBKR_PORT = 4002\n", - "IBKR_CLIENT_ID = 101\n", - "IBKR_CONNECT_TIMEOUT_SECONDS = 10\n", - "\n", - "ib = IB()\n", - "print(f\"Connecting to IBKR Gateway at {IBKR_HOST}:{IBKR_PORT} with client id {IBKR_CLIENT_ID}...\")\n", - "try:\n", - " ib.connect(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID, timeout=IBKR_CONNECT_TIMEOUT_SECONDS)\n", - " print(\"Connection established.\")\n", - "except Exception as e:\n", - " print(f\"Failed to connect to IBKR Gateway: {e}\")\n", - " ib.disconnect()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "ecdc721f", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Connected: True\n", - "Client ID: 101\n" - ] - } - ], - "source": [ - "print(f\"Connected: {ib.isConnected()}\")\n", - "print(f\"Client ID: {ib.client.clientId}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "10ebc240", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No open portfolio positions were returned.\n" - ] - } - ], - "source": [ - "portfolio = ib.portfolio()\n", - "if not portfolio:\n", - " print(\"No open portfolio positions were returned.\")\n", - "else:\n", - " for item in portfolio:\n", - " print(\n", - " f\"{item.contract.symbol}: position={item.position}, \"\n", - " f\"market_value={item.marketValue}, unrealized_pnl={item.unrealizedPNL}\"\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "fed654ec", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Unknown contract: Stock(symbol='VUAA', exchange='SMART', currency='USD')\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Submitted order for VUAA: Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='PendingSubmit', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0)], advancedError='')\n" - ] - } - ], - "source": [ - "from ib_insync import MarketOrder\n", - "\n", - "contract = Stock(\"VUAA\", \"SMART\", \"USD\")\n", - "await ib.qualifyContractsAsync(contract)\n", - "\n", - "# Adjust the quantity as needed before running this cell.\n", - "order = MarketOrder(\"BUY\", 0.70)\n", - "trade = ib.placeOrder(contract, order)\n", - "\n", - "print(f\"Submitted order for {contract.symbol}: {trade}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "06881101", - "metadata": {}, - "outputs": [], - "source": [ - "trds = ib.trades()" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "08663602", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Cancelled', 'Cancelled', 'Cancelled']" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(map(lambda x: x.orderStatus.status, trds))" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "2c7abf23", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=1578393268, action='BUY', totalQuantity=1.0, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n", - " Trade(contract=Stock(conId=756733, symbol='SPY', right='?', exchange='SMART', currency='USD', localSymbol='SPY', tradingClass='SPY'), order=Order(permId=24474350, action='BUY', totalQuantity=0.1345, orderType='MKT', lmtPrice=0.0, auxPrice=0.0, tif='DAY', ocaType=3, displaySize=2147483647, rule80A='0', openClose='', volatilityType=0, deltaNeutralOrderType='None', referencePriceType=0, account='DUR281921', clearingIntent='IB', cashQty=0.0, dontUseAutoPriceForHedge=True, filledQuantity=0.0, refFuturesConId=2147483647, shareholder='Not an insider or substantial shareholder'), orderStatus=OrderStatus(orderId=0, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[], advancedError=''),\n", - " Trade(contract=Stock(symbol='VUAA', exchange='SMART', currency='USD'), order=MarketOrder(orderId=10, clientId=101, action='BUY', totalQuantity=0.7), orderStatus=OrderStatus(orderId=10, status='Cancelled', filled=0.0, remaining=0.0, avgFillPrice=0.0, permId=0, parentId=0, lastFillPrice=0.0, clientId=0, whyHeld='', mktCapPrice=0.0), fills=[], log=[TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 539380, tzinfo=datetime.timezone.utc), status='PendingSubmit', message='', errorCode=0), TradeLogEntry(time=datetime.datetime(2026, 7, 30, 19, 36, 57, 743208, tzinfo=datetime.timezone.utc), status='Cancelled', message='Error 200, reqId 10: No security definition has been found for the request', errorCode=200)], advancedError='')]" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trds" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.15" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/spy_direction_dataset.ipynb b/notebooks/spy_direction_dataset.ipynb index ec932d3..578aded 100644 --- a/notebooks/spy_direction_dataset.ipynb +++ b/notebooks/spy_direction_dataset.ipynb @@ -87,16 +87,16 @@ "output_type": "stream", "text": [ "\n", - "DatetimeIndex: 1255 entries, 2021-08-02 to 2026-07-31\n", + "DatetimeIndex: 1258 entries, 2021-08-02 to 2026-08-05\n", "Data columns (total 4 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", - " 0 SPY_close 1255 non-null float64\n", - " 1 VIX_close 1255 non-null float64\n", - " 2 TLT_close 1255 non-null float64\n", - " 3 USO_close 1255 non-null float64\n", + " 0 SPY_close 1258 non-null float64\n", + " 1 VIX_close 1258 non-null float64\n", + " 2 TLT_close 1258 non-null float64\n", + " 3 USO_close 1258 non-null float64\n", "dtypes: float64(4)\n", - "memory usage: 49.0 KB\n" + "memory usage: 49.1 KB\n" ] }, { @@ -234,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -364,7 +364,7 @@ "2021-10-15 -0.002202 0.030287 -0.003823 1.0 " ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -418,7 +418,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -458,24 +458,24 @@ " \n", " \n", " train\n", - " 840\n", + " 842\n", " 2021-10-11\n", - " 2025-02-13\n", - " 0.583333\n", + " 2025-02-18\n", + " 0.581948\n", " \n", " \n", " validation\n", " 180\n", - " 2025-02-14\n", - " 2025-10-31\n", - " 0.633333\n", + " 2025-02-19\n", + " 2025-11-04\n", + " 0.638889\n", " \n", " \n", " test\n", - " 181\n", - " 2025-11-03\n", - " 2026-07-24\n", - " 0.569061\n", + " 182\n", + " 2025-11-05\n", + " 2026-07-29\n", + " 0.576923\n", " \n", " \n", "\n", @@ -484,12 +484,12 @@ "text/plain": [ " rows start_date end_date target_mean\n", "split \n", - "train 840 2021-10-11 2025-02-13 0.583333\n", - "validation 180 2025-02-14 2025-10-31 0.633333\n", - "test 181 2025-11-03 2026-07-24 0.569061" + "train 842 2021-10-11 2025-02-18 0.581948\n", + "validation 180 2025-02-19 2025-11-04 0.638889\n", + "test 182 2025-11-05 2026-07-29 0.576923" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -543,17 +543,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Rows: 1,201\n", + "Rows: 1,204\n", "Feature columns: ['SPY_ret_5', 'SPY_ret_20', 'SPY_dist_sma50', 'VIX_change_5', 'VIX_rank_20', 'TLT_ret_10', 'USO_ret_5', 'SPY_TLT_ratio_ret']\n", "Target column: spy_up_5d\n", - "Training matrix shape: (840, 8)\n" + "Training matrix shape: (842, 8)\n" ] }, { @@ -592,16 +592,16 @@ " \n", " \n", " count\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201.000000\n", - " 1201\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204.000000\n", + " 1204\n", " \n", " \n", " unique\n", @@ -640,32 +640,32 @@ " NaN\n", " NaN\n", " NaN\n", - " 840\n", + " 842\n", " \n", " \n", " mean\n", - " 0.002502\n", - " 0.009848\n", - " 0.011028\n", - " 0.017763\n", - " 0.403476\n", - " -0.004099\n", - " 0.005024\n", - " 0.004880\n", - " 0.588676\n", + " 0.002464\n", + " 0.009816\n", + " 0.010973\n", + " 0.017820\n", + " 0.404506\n", + " -0.004103\n", + " 0.004938\n", + " 0.004838\n", + " 0.589701\n", " NaN\n", " \n", " \n", " std\n", - " 0.023057\n", - " 0.043558\n", - " 0.037315\n", - " 0.299417\n", - " 0.337501\n", - " 0.028502\n", - " 0.052628\n", - " 0.027606\n", - " 0.492279\n", + " 0.023044\n", + " 0.043515\n", + " 0.037287\n", + " 0.299050\n", + " 0.337773\n", + " 0.028469\n", + " 0.052606\n", + " 0.027586\n", + " 0.492092\n", " NaN\n", " \n", " \n", @@ -683,40 +683,40 @@ " \n", " \n", " 25%\n", - " -0.009601\n", - " -0.016509\n", - " -0.008656\n", - " -0.056615\n", + " -0.009749\n", + " -0.016557\n", + " -0.008704\n", + " -0.056606\n", " 0.100000\n", - " -0.023304\n", - " -0.025866\n", - " -0.010013\n", + " -0.023139\n", + " -0.025950\n", + " -0.010029\n", " 0.000000\n", " NaN\n", " \n", " \n", " 50%\n", - " 0.003798\n", - " 0.015725\n", - " 0.017802\n", - " -0.018570\n", + " 0.003749\n", + " 0.015639\n", + " 0.017755\n", + " -0.018430\n", " 0.300000\n", - " -0.004435\n", - " 0.004950\n", - " 0.005537\n", + " -0.004429\n", + " 0.004869\n", + " 0.005508\n", " 1.000000\n", " NaN\n", " \n", " \n", " 75%\n", - " 0.015955\n", - " 0.038179\n", - " 0.038282\n", - " 0.030814\n", + " 0.015950\n", + " 0.038167\n", + " 0.038124\n", + " 0.030815\n", " 0.750000\n", - " 0.014227\n", - " 0.032486\n", - " 0.021390\n", + " 0.014188\n", + " 0.032295\n", + " 0.021358\n", " 1.000000\n", " NaN\n", " \n", @@ -739,33 +739,33 @@ ], "text/plain": [ " SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 VIX_rank_20 \\\n", - "count 1201.000000 1201.000000 1201.000000 1201.000000 1201.000000 \n", + "count 1204.000000 1204.000000 1204.000000 1204.000000 1204.000000 \n", "unique NaN NaN NaN NaN NaN \n", "top NaN NaN NaN NaN NaN \n", "freq NaN NaN NaN NaN NaN \n", - "mean 0.002502 0.009848 0.011028 0.017763 0.403476 \n", - "std 0.023057 0.043558 0.037315 0.299417 0.337501 \n", + "mean 0.002464 0.009816 0.010973 0.017820 0.404506 \n", + "std 0.023044 0.043515 0.037287 0.299050 0.337773 \n", "min -0.114962 -0.123975 -0.141995 -0.387709 0.050000 \n", - "25% -0.009601 -0.016509 -0.008656 -0.056615 0.100000 \n", - "50% 0.003798 0.015725 0.017802 -0.018570 0.300000 \n", - "75% 0.015955 0.038179 0.038282 0.030814 0.750000 \n", + "25% -0.009749 -0.016557 -0.008704 -0.056606 0.100000 \n", + "50% 0.003749 0.015639 0.017755 -0.018430 0.300000 \n", + "75% 0.015950 0.038167 0.038124 0.030815 0.750000 \n", "max 0.082843 0.157566 0.088105 3.846154 1.000000 \n", "\n", " TLT_ret_10 USO_ret_5 SPY_TLT_ratio_ret spy_up_5d split \n", - "count 1201.000000 1201.000000 1201.000000 1201.000000 1201 \n", + "count 1204.000000 1204.000000 1204.000000 1204.000000 1204 \n", "unique NaN NaN NaN NaN 3 \n", "top NaN NaN NaN NaN train \n", - "freq NaN NaN NaN NaN 840 \n", - "mean -0.004099 0.005024 0.004880 0.588676 NaN \n", - "std 0.028502 0.052628 0.027606 0.492279 NaN \n", + "freq NaN NaN NaN NaN 842 \n", + "mean -0.004103 0.004938 0.004838 0.589701 NaN \n", + "std 0.028469 0.052606 0.027586 0.492092 NaN \n", "min -0.092880 -0.196652 -0.117208 0.000000 NaN \n", - "25% -0.023304 -0.025866 -0.010013 0.000000 NaN \n", - "50% -0.004435 0.004950 0.005537 1.000000 NaN \n", - "75% 0.014227 0.032486 0.021390 1.000000 NaN \n", + "25% -0.023139 -0.025950 -0.010029 0.000000 NaN \n", + "50% -0.004429 0.004869 0.005508 1.000000 NaN \n", + "75% 0.014188 0.032295 0.021358 1.000000 NaN \n", "max 0.091322 0.327273 0.129204 1.000000 NaN " ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -787,14 +787,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(PosixPath('/home/jarno/repos/trading-bot/data/training/spy_direction_5d.parquet'),\n", - " (1201, 11),\n", + " (1204, 11),\n", " date SPY_ret_5 SPY_ret_20 SPY_dist_sma50 VIX_change_5 \\\n", " 0 2021-10-11 0.014114 -0.026625 -0.018145 -0.090869 \n", " 1 2021-10-12 0.001201 -0.023752 -0.020386 -0.074091 \n", @@ -810,7 +810,7 @@ " 4 0.05 -0.002202 0.030287 -0.003823 1.0 train )" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } diff --git a/pyproject.toml b/pyproject.toml index 0c06c84..2895872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,6 @@ readme = "README.md" requires-python = ">=3.11,<3.12" dependencies = [ "alpaca-py>=0.43.5", - "ib-insync>=0.9.86", "matplotlib>=3.11.1", "pandas>=2.3.0", "pyarrow>=20.0.0", diff --git a/src/trading_bot/data/fetch_alpaca_daily.py b/src/trading_bot/data/fetch_alpaca_daily.py index ec15763..5af5b9c 100644 --- a/src/trading_bot/data/fetch_alpaca_daily.py +++ b/src/trading_bot/data/fetch_alpaca_daily.py @@ -98,9 +98,10 @@ def main() -> None: api_key=args.api_key, secret_key=args.secret_key, ) + print(f"Fetched {len(candles)} daily candles for {symbol}.") stored = write_candles(output_path, symbol, candles) print(f"Wrote {len(stored)} total daily rows to {output_path}") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/trading_bot/data/fetch_ibkr_daily.py b/src/trading_bot/data/fetch_ibkr_daily.py deleted file mode 100644 index b5ef94e..0000000 --- a/src/trading_bot/data/fetch_ibkr_daily.py +++ /dev/null @@ -1,293 +0,0 @@ -"""IBKR daily candle fetcher.""" - -from __future__ import annotations - -import argparse -import re -from dataclasses import dataclass -from datetime import date, datetime, timedelta -from pathlib import Path -from zoneinfo import ZoneInfo - -import pandas as pd - -DEFAULT_OUTPUT_DIR = Path("data/ibkr/daily") -DEFAULT_DURATION = "1 W" -EASTERN_TZ = ZoneInfo("America/New_York") -DURATION_PATTERN = re.compile(r"^\d+\s+[SDWMY]$") - -IBKR_HOST = "127.0.0.1" -IBKR_PORT = 4002 -IBKR_CLIENT_ID = 101 -IBKR_CONNECT_TIMEOUT_SECONDS = 10 - - -@dataclass(frozen=True) -class DailyCandle: - """Daily OHLCV market data for one trading session.""" - - trading_day: date - open: float - high: float - low: float - close: float - volume: int - - -def default_end_date() -> date: - """Return yesterday's date in the US/Eastern market timezone.""" - - return datetime.now(EASTERN_TZ).date() - timedelta(days=1) - - -def current_market_date() -> date: - """Return today's date in the US/Eastern market timezone.""" - - return datetime.now(EASTERN_TZ).date() - - -def parse_end_date(value: str) -> date: - """Parse an end date in IBKR-friendly YYYYMMDD form.""" - - try: - return datetime.strptime(value, "%Y%m%d").date() - except ValueError as exc: - raise argparse.ArgumentTypeError( - "end date must use YYYYMMDD format, such as 20250605" - ) from exc - - -def parse_duration(value: str) -> str: - """Normalize and validate an IBKR duration string like '1 W' or '1 M'.""" - - normalized = " ".join(value.upper().split()) - if not DURATION_PATTERN.fullmatch(normalized): - raise argparse.ArgumentTypeError( - "duration must look like an IBKR value, such as '1 W' or '1 M'" - ) - return normalized - - -def format_ibkr_end_datetime(end_date: date) -> str: - """Convert a date to the US/Eastern end datetime string IBKR expects.""" - - return f"{end_date:%Y%m%d} 23:59:59 US/Eastern" - - -def normalize_bar_date(value: date | datetime | str) -> date: - """Normalize an IBKR historical bar date value.""" - - if isinstance(value, datetime): - return value.date() - if isinstance(value, date): - return value - return parse_end_date(value) - - -def fetch_daily_candles( - symbol: str, end_date: date, duration: str -) -> list[DailyCandle]: - """Fetch daily candles for a symbol from IBKR.""" - - try: - from ib_insync import IB, Stock # type: ignore[import-not-found] - except ImportError as exc: - raise SystemExit( - "Missing dependency: ib_insync. Install it before running the " - "IBKR fetcher." - ) from exc - - ib = IB() - try: - print( - f"Connecting to IBKR Gateway at {IBKR_HOST}:{IBKR_PORT} " - f"with client id {IBKR_CLIENT_ID}" - ) - ib.connect( - IBKR_HOST, - IBKR_PORT, - clientId=IBKR_CLIENT_ID, - timeout=IBKR_CONNECT_TIMEOUT_SECONDS, - ) - - contract = Stock(symbol, "SMART", "USD") - ib.qualifyContracts(contract) - bars = ib.reqHistoricalData( - contract, - endDateTime=format_ibkr_end_datetime(end_date), - durationStr=duration, - barSizeSetting="1 day", - whatToShow="TRADES", - useRTH=True, - formatDate=1, - ) - - if not bars: - print(f"IBKR returned no historical bars for {symbol}.") - - candles: list[DailyCandle] = [] - for bar in bars: - trading_day = normalize_bar_date(bar.date) - candles.append( - DailyCandle( - trading_day=trading_day, - open=float(bar.open), - high=float(bar.high), - low=float(bar.low), - close=float(bar.close), - volume=int(bar.volume), - ) - ) - - return candles - finally: - if ib.isConnected(): - ib.disconnect() - - -def candles_to_frame(symbol: str, candles: list[DailyCandle]) -> pd.DataFrame: - """Convert candles to a date-indexed dataframe ready for Parquet storage.""" - - rows = [ - { - "date": candle.trading_day, - "symbol": symbol, - "open": candle.open, - "high": candle.high, - "low": candle.low, - "close": candle.close, - "volume": candle.volume, - } - for candle in candles - ] - frame = pd.DataFrame.from_records(rows) - if frame.empty: - return pd.DataFrame( - columns=["symbol", "open", "high", "low", "close", "volume"], - index=pd.Index([], name="date"), - ) - - frame["date"] = pd.to_datetime(frame["date"]).dt.date - return frame.set_index("date") - - -def read_existing_candles(path: Path) -> pd.DataFrame: - """Read an existing candle Parquet file as a date-indexed dataframe.""" - - if not path.exists(): - return pd.DataFrame( - columns=["symbol", "open", "high", "low", "close", "volume"], - index=pd.Index([], name="date"), - ) - - frame = pd.read_parquet(path) - if "date" in frame.columns: - frame["date"] = pd.to_datetime(frame["date"]).dt.date - frame = frame.set_index("date") - - frame.index = pd.to_datetime(frame.index).date - frame.index.name = "date" - return frame - - -def oldest_stored_date_or_today(path: Path) -> date: - """Return the oldest stored candle date, or today if no data exists yet.""" - - existing = read_existing_candles(path) - if existing.empty: - return current_market_date() - return min(existing.index) - - -def write_candles(path: Path, symbol: str, candles: list[DailyCandle]) -> pd.DataFrame: - """Append candles to a ticker Parquet file, keeping one row per date.""" - - existing = read_existing_candles(path) - fetched = candles_to_frame(symbol, candles) - combined = pd.concat([existing, fetched]) - if not combined.empty: - combined = combined[~combined.index.duplicated(keep="last")] - combined = combined.sort_index() - - path.parent.mkdir(parents=True, exist_ok=True) - combined.to_parquet(path, index=True) - return combined - - -def print_candles(symbol: str, candles: list[DailyCandle]) -> None: - """Print candles in a compact table.""" - - print("date,symbol,open,high,low,close,volume") - for candle in candles: - print( - f"{candle.trading_day.isoformat()}," - f"{symbol}," - f"{candle.open:.2f}," - f"{candle.high:.2f}," - f"{candle.low:.2f}," - f"{candle.close:.2f}," - f"{candle.volume}" - ) - - -def parse_args() -> argparse.Namespace: - """Parse command line arguments.""" - - parser = argparse.ArgumentParser(description="Fetch daily IBKR candles.") - parser.add_argument("symbol", help="Ticker symbol to fetch, such as SPY.") - end_date_group = parser.add_mutually_exclusive_group() - end_date_group.add_argument( - "--end-date", - type=parse_end_date, - help="Request end date in YYYYMMDD format. Defaults to yesterday.", - ) - end_date_group.add_argument( - "--end-date-from-parquet", - action="store_true", - help=( - "Use the oldest date from the symbol Parquet file as the request " - "end date. Defaults to today if the file is missing or empty." - ), - ) - parser.add_argument( - "--duration", - type=parse_duration, - default=DEFAULT_DURATION, - help=( - "IBKR duration string, such as '1 W' or '1 M'. " - f"Defaults to {DEFAULT_DURATION}." - ), - ) - parser.add_argument( - "--output-dir", - type=Path, - default=DEFAULT_OUTPUT_DIR, - help=f"Directory for Parquet files. Defaults to {DEFAULT_OUTPUT_DIR}.", - ) - return parser.parse_args() - - -def main() -> None: - """Run the daily candle fetcher.""" - - args = parse_args() - symbol = args.symbol.upper() - output_path = args.output_dir / f"{symbol}.parquet" - end_date = ( - oldest_stored_date_or_today(output_path) - if args.end_date_from_parquet - else args.end_date or default_end_date() - ) - - print( - f"Fetching {symbol} daily candles ending {end_date:%Y-%m-%d} " - f"for duration {args.duration}" - ) - candles = fetch_daily_candles(symbol, end_date, args.duration) - print_candles(symbol, candles) - stored = write_candles(output_path, symbol, candles) - print(f"Wrote {len(stored)} total daily rows to {output_path}") - - -if __name__ == "__main__": - main() diff --git a/uv.lock b/uv.lock index 5338f3b..940fec8 100644 --- a/uv.lock +++ b/uv.lock @@ -183,18 +183,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, ] -[[package]] -name = "eventkit" -version = "1.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/16/1e/0fac4e45d71ace143a2673ec642701c3cd16f833a0e77a57fa6a40472696/eventkit-1.0.3.tar.gz", hash = "sha256:99497f6f3c638a50ff7616f2f8cd887b18bbff3765dc1bd8681554db1467c933", size = 28320, upload-time = "2023-12-11T11:41:35.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/d9/7497d650b69b420e1a913329a843e16c715dac883750679240ef00a921e2/eventkit-1.0.3-py3-none-any.whl", hash = "sha256:0e199527a89aff9d195b9671ad45d2cc9f79ecda0900de8ecfb4c864d67ad6a2", size = 31837, upload-time = "2023-12-11T11:41:33.358Z" }, -] - [[package]] name = "executing" version = "2.2.1" @@ -221,19 +209,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, ] -[[package]] -name = "ib-insync" -version = "0.9.86" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "eventkit" }, - { name = "nest-asyncio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/bb/733d5c81c8c2f54e90898afc7ff3a99f4d53619e6917c848833f9cc1ab56/ib_insync-0.9.86.tar.gz", hash = "sha256:73af602ca2463f260999970c5bd937b1c4325e383686eff301743a4de08d381e", size = 69859, upload-time = "2023-07-02T12:43:31.968Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/f3/28ea87be30570f4d6b8fd24380d12fa74e59467ee003755e76aeb29082b8/ib_insync-0.9.86-py3-none-any.whl", hash = "sha256:a61fbe56ff405d93d211dad8238d7300de76dd6399eafc04c320470edec9a4a4", size = 72980, upload-time = "2023-07-02T12:43:29.928Z" }, -] - [[package]] name = "idna" version = "3.18" @@ -459,15 +434,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, ] -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - [[package]] name = "nest-asyncio2" version = "1.7.2" @@ -946,7 +912,6 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "alpaca-py" }, - { name = "ib-insync" }, { name = "matplotlib" }, { name = "pandas" }, { name = "pyarrow" }, @@ -964,7 +929,6 @@ dev = [ [package.metadata] requires-dist = [ { name = "alpaca-py", specifier = ">=0.43.5" }, - { name = "ib-insync", specifier = ">=0.9.86" }, { name = "matplotlib", specifier = ">=3.11.1" }, { name = "pandas", specifier = ">=2.3.0" }, { name = "pyarrow", specifier = ">=20.0.0" },