trading-bot/notebooks/train-xboost.ipynb

In [1]:

import pandas as pd
import xgboost as xgb
from sklearn.metrics import classification_report, accuracy_score
import matplotlib.pyplot as plt
import numpy as np

In [2]:

# 1. Load data
df = pd.read_parquet("../data/training/spy_direction_5d.parquet")

# 2. Define features & target
target_col = "spy_up_5d"
feature_cols = [
    "VIX_rank_20",
    "TLT_ret_10",
    "USO_ret_5",
    "SPY_TLT_ratio_ret",
    "SPY_ret_5",
    "SPY_ret_20",
    "SPY_dist_sma50"
]

In [3]:

# 3. Create three distinct splits
train_df = df[df["split"] == "train"]
val_df = df[df["split"] == "validation"]  # Adjust string if named "val"
test_df = df[df["split"] == "test"]

X_train, y_train = train_df[feature_cols], train_df[target_col]
X_val, y_val = val_df[feature_cols], val_df[target_col]
X_test, y_test = test_df[feature_cols], test_df[target_col]

In [4]:

# 1. Check target distribution across splits
print("Train positive %:", y_train.mean())
print("Val positive %:  ", y_val.mean())
print("Test positive %: ", y_test.mean())

# 2. Check simple linear correlation with target
print("\nTrain Feature Correlations with Target:")
print(train_df[feature_cols].apply(lambda col: col.corr(y_train)))

# 3. Test a super simple model (Decision Stump)
stump_model = xgb.XGBClassifier(
    n_estimators=100,
    max_depth=2,             # Decision stumps (1 split per tree, minimal overfitting)
    learning_rate=0.01,
    early_stopping_rounds=15,
    eval_metric="logloss"
)
stump_model.fit(
    X_train, y_train,
    eval_set=[(X_train, y_train), (X_val, y_val)],
    verbose=False
)
print(f"\nStump Model Best Iteration: {stump_model.best_iteration}")

Train positive %: 0.5819477434679335
Val positive %:   0.6388888888888888
Test positive %:  0.5769230769230769

Train Feature Correlations with Target:
VIX_rank_20         -0.004576
TLT_ret_10           0.162370
USO_ret_5           -0.113602
SPY_TLT_ratio_ret   -0.111551
SPY_ret_5           -0.001073
SPY_ret_20           0.006523
SPY_dist_sma50      -0.035773
dtype: float64

Stump Model Best Iteration: 22

In [5]:

# 4. Initialize XGBoost Classifier with Early Stopping
model = xgb.XGBClassifier(
    n_estimators=300,          # Set higher; early stopping will truncate training automatically
    max_depth=1,              # Keep shallow to prevent memorizing noise
    learning_rate=0.01,
    subsample=0.7,
    colsample_bytree=0.7,
    early_stopping_rounds=20, # Stop if validation loss stops improving for 15 rounds
    eval_metric="logloss",
    random_state=42
)

# 5. Fit using Validation set to monitor performance
model.fit(
    X_train, y_train,
    eval_set=[(X_train, y_train), (X_val, y_val)],
    verbose=False
)

print(f"Optimal number of trees (best iteration): {model.best_iteration}")

evals = model.evals_result()
val_loss = evals['validation_1']['logloss']
print(f"Starting Validation Loss: {val_loss[0]:.4f}")
print(f"Final Validation Loss:    {val_loss[-1]:.4f}")

Optimal number of trees (best iteration): 131
Starting Validation Loss: 0.6606
Final Validation Loss:    0.6537

In [6]:

# 6. Evaluate strictly on the holdout Test Set
y_pred = model.predict(X_test)
y_proba = model.predict_proba(X_test)[:, 1]

print("\n--- Final Holdout Test Performance ---")
print(f"Test Set Accuracy: {accuracy_score(y_test, y_pred):.2%}\n")
print("Classification Report:")
print(classification_report(y_test, y_pred))
threshold = y_train.mean()
y_pred_custom = (y_proba >= threshold).astype(int)
print(f"classification_report with custom threshold ({threshold}):")
print(classification_report(y_test, y_pred_custom))

# 7. Feature Importance
xgb.plot_importance(model, importance_type="gain")
plt.title("Feature Importance (Gain)")
plt.show()

--- Final Holdout Test Performance ---
Test Set Accuracy: 57.69%

Classification Report:
              precision    recall  f1-score   support

         0.0       0.00      0.00      0.00        77
         1.0       0.58      1.00      0.73       105

    accuracy                           0.58       182
   macro avg       0.29      0.50      0.37       182
weighted avg       0.33      0.58      0.42       182

classification_report with custom threshold (0.5819477434679335):
              precision    recall  f1-score   support

         0.0       0.46      0.74      0.57        77
         1.0       0.66      0.37      0.48       105

    accuracy                           0.53       182
   macro avg       0.56      0.56      0.52       182
weighted avg       0.58      0.53      0.52       182

/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", result.shape[0])
/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", result.shape[0])
/home/jarno/repos/trading-bot/.venv/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1879: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", result.shape[0])

In [7]:

# 1. Inspect raw predicted probabilities for the test set
y_proba_test = model.predict_proba(X_test)[:, 1]

print(f"Min probability:    {y_proba_test.min():.4f}")
print(f"Max probability:    {y_proba_test.max():.4f}")
print(f"Mean probability:   {y_proba_test.mean():.4f}")
print(f"Median probability: {np.median(y_proba_test):.4f}")

Min probability:    0.5154
Max probability:    0.6891
Mean probability:   0.5719
Median probability: 0.5641

In [8]:

import json
from pathlib import Path

# 1. Create the 'models' directory if it doesn't exist
models_dir = Path("models")
models_dir.mkdir(parents=True, exist_ok=True)

# 2. Define paths
model_path = models_dir / "spy_xgb_v1.json"
meta_path = models_dir / "spy_xgb_v1_meta.json"

# 3. Save XGBoost model (explicitly converted to str)
model.save_model(str(model_path))

# 4. Save metadata alongside model
metadata = {
    "p_base": float(y_train.mean()),
    "feature_cols": feature_cols,
    "last_trained_date": str(df["date"].iloc[-1]),
}

with open(meta_path, "w") as f:
    json.dump(metadata, f, indent=2)

print(f"Successfully saved model to {model_path}")

Successfully saved model to models/spy_xgb_v1.json

In [ ]: