800 lines
31 KiB
Python
800 lines
31 KiB
Python
"""Fair, progress-reporting feature ablation experiments for the 8H model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import joblib
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.ensemble import ExtraTreesRegressor, HistGradientBoostingRegressor
|
|
|
|
from .pipeline import (
|
|
EIGHT_H_MS,
|
|
HORIZON,
|
|
LOWER_QUANTILE,
|
|
TARGET_QUANTILE,
|
|
VOLATILITY_RANK_THRESHOLD,
|
|
VOLATILITY_RANK_WINDOW,
|
|
WINDOW,
|
|
_ASSET_FEATURE_NAMES,
|
|
_BASE_FEATURE_COLUMNS,
|
|
_rank_windows,
|
|
apply_calibration,
|
|
asset_profile,
|
|
compute_indicators,
|
|
evaluate_predictions,
|
|
fit_asset_scaler,
|
|
floor_to_8h,
|
|
load_manifest,
|
|
load_raw_frames,
|
|
months_before,
|
|
recent_split_boundaries,
|
|
scale_asset_profile,
|
|
split_holdout_symbols,
|
|
)
|
|
|
|
|
|
def _progress(message: str) -> None:
|
|
print(f"[cbpred-experiment] {message}", file=sys.stderr, flush=True)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FeatureVariant:
|
|
name: str
|
|
description: str
|
|
bar_columns: tuple[str, ...]
|
|
indicator_columns: tuple[str, ...] = ()
|
|
global_columns: tuple[str, ...] = ()
|
|
cross_columns: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def uses_global(self) -> bool:
|
|
return bool(self.global_columns)
|
|
|
|
@property
|
|
def uses_cross(self) -> bool:
|
|
return bool(self.cross_columns)
|
|
|
|
|
|
_VOLUME_COLUMNS = ("quote_volume_rank", "quote_volume_log_rel_median")
|
|
_ALL_BAR_COLUMNS = _BASE_FEATURE_COLUMNS
|
|
_COMPACT7_COLUMNS = tuple(name for name in _ALL_BAR_COLUMNS if name != "body_abs_range")
|
|
_COMPACT6_COLUMNS = tuple(
|
|
name for name in _COMPACT7_COLUMNS if name != "body_sign"
|
|
)
|
|
_GLOBAL_COLUMNS = (
|
|
"global_median_return_1",
|
|
"global_median_trend_3",
|
|
"global_median_atr_pct",
|
|
"global_return_dispersion",
|
|
"global_up_ratio",
|
|
"global_median_quote_activity",
|
|
)
|
|
_CROSS_COLUMNS = (
|
|
"cross_return_1_rank",
|
|
"cross_trend_3_rank",
|
|
"cross_atr_pct_rank",
|
|
"cross_quote_activity_rank",
|
|
)
|
|
|
|
|
|
def experiment_variants() -> list[FeatureVariant]:
|
|
"""Return independent candidates; baseline is deliberately first."""
|
|
base = tuple(_ALL_BAR_COLUMNS)
|
|
return [
|
|
FeatureVariant("baseline", "current Stage 1 feature set", base),
|
|
FeatureVariant("compact7", "remove absolute body size", _COMPACT7_COLUMNS),
|
|
FeatureVariant(
|
|
"compact6",
|
|
"remove absolute body size and explicit body sign",
|
|
_COMPACT6_COLUMNS,
|
|
),
|
|
FeatureVariant(
|
|
"rsi_sma",
|
|
"add normalized RSI and normalized SMA20/SMA50 distance",
|
|
base,
|
|
("rsi_norm", "sma20_gap", "sma50_gap"),
|
|
),
|
|
FeatureVariant(
|
|
"rsi_bbands",
|
|
"add normalized RSI and Bollinger position/width",
|
|
base,
|
|
("rsi_norm", "bb_position", "bb_width"),
|
|
),
|
|
FeatureVariant(
|
|
"rsi_trend",
|
|
"add normalized RSI, short trends, and rolling volatility",
|
|
base,
|
|
("rsi_norm", "trend_3", "trend_9", "volatility_18"),
|
|
),
|
|
FeatureVariant(
|
|
"full_indicators",
|
|
"add the complete existing indicator group",
|
|
base,
|
|
(
|
|
"atr_pct",
|
|
"atr_rank_200",
|
|
"rsi_norm",
|
|
"bb_position",
|
|
"bb_width",
|
|
"ema9_gap",
|
|
"ema21_gap",
|
|
"trend_3",
|
|
"trend_9",
|
|
"volatility_18",
|
|
"drawdown_18",
|
|
),
|
|
),
|
|
FeatureVariant(
|
|
"global6",
|
|
"add six cross-symbol market-state summaries",
|
|
base,
|
|
global_columns=_GLOBAL_COLUMNS,
|
|
),
|
|
FeatureVariant(
|
|
"cross4",
|
|
"add four current-symbol cross-sectional ranks",
|
|
base,
|
|
cross_columns=_CROSS_COLUMNS,
|
|
),
|
|
FeatureVariant(
|
|
"global_cross",
|
|
"add global market summaries and current-symbol ranks",
|
|
base,
|
|
global_columns=_GLOBAL_COLUMNS,
|
|
cross_columns=_CROSS_COLUMNS,
|
|
),
|
|
]
|
|
|
|
|
|
def feature_names(variant: FeatureVariant) -> list[str]:
|
|
names: list[str] = []
|
|
per_bar = (*variant.bar_columns, *_VOLUME_COLUMNS, *variant.indicator_columns)
|
|
for lag in range(WINDOW - 1, -1, -1):
|
|
names.extend(f"bar_{lag:02d}_{name}" for name in per_bar)
|
|
names.extend(_ASSET_FEATURE_NAMES)
|
|
names.extend(variant.global_columns)
|
|
names.extend(variant.cross_columns)
|
|
return names
|
|
|
|
|
|
def _cross_sectional_context(
|
|
indicators: dict[str, pd.DataFrame],
|
|
raw_profiles: dict[str, dict[str, float]],
|
|
) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
|
|
records: list[pd.DataFrame] = []
|
|
for symbol, frame in indicators.items():
|
|
profile = raw_profiles.get(symbol)
|
|
if profile is None:
|
|
continue
|
|
record = pd.DataFrame(
|
|
{
|
|
"symbol": symbol,
|
|
"timestamp": frame["timestamp"].to_numpy(dtype=np.int64),
|
|
"return_1": frame["log_return"].to_numpy(dtype=float),
|
|
"trend_3": frame["trend_3"].to_numpy(dtype=float),
|
|
"atr_pct": frame["atr_pct"].to_numpy(dtype=float),
|
|
"body_sign": frame["body_sign"].to_numpy(dtype=float),
|
|
"quote_activity": frame["quote_log"].to_numpy(dtype=float)
|
|
- float(profile["asset_quote_volume_top20_median_log"]),
|
|
}
|
|
)
|
|
records.append(record)
|
|
if not records:
|
|
raise ValueError("no cross-sectional records available")
|
|
all_rows = pd.concat(records, ignore_index=True)
|
|
grouped = all_rows.groupby("timestamp", sort=False)
|
|
global_context = grouped.agg(
|
|
global_median_return_1=("return_1", "median"),
|
|
global_median_trend_3=("trend_3", "median"),
|
|
global_median_atr_pct=("atr_pct", "median"),
|
|
global_return_dispersion=("return_1", lambda values: values.std(ddof=0)),
|
|
global_up_ratio=("body_sign", lambda values: float(np.mean(values > 0))),
|
|
global_median_quote_activity=("quote_activity", "median"),
|
|
)
|
|
rank_columns = {
|
|
"return_1": "cross_return_1_rank",
|
|
"trend_3": "cross_trend_3_rank",
|
|
"atr_pct": "cross_atr_pct_rank",
|
|
"quote_activity": "cross_quote_activity_rank",
|
|
}
|
|
for source, target in rank_columns.items():
|
|
all_rows[target] = all_rows.groupby("timestamp")[source].rank(
|
|
method="average", pct=True
|
|
)
|
|
cross_context = {
|
|
symbol: group.set_index("timestamp")[list(rank_columns.values())].sort_index()
|
|
for symbol, group in all_rows.groupby("symbol", sort=False)
|
|
}
|
|
return global_context.sort_index(), cross_context
|
|
|
|
|
|
def _variant_window_features(
|
|
frame: pd.DataFrame,
|
|
variant: FeatureVariant,
|
|
anchor_indices: np.ndarray,
|
|
asset: dict[str, float],
|
|
global_context: pd.DataFrame | None = None,
|
|
cross_context: pd.DataFrame | None = None,
|
|
) -> np.ndarray:
|
|
starts = anchor_indices - (WINDOW - 1)
|
|
quote = frame["quote_log"].to_numpy(dtype=float)
|
|
quote_windows = np.lib.stride_tricks.sliding_window_view(quote, WINDOW)[starts]
|
|
volume_rank = _rank_windows(quote_windows)
|
|
volume_median = np.median(quote_windows, axis=1, keepdims=True)
|
|
columns = [*variant.bar_columns, *variant.indicator_columns]
|
|
values = frame[columns].to_numpy(dtype=float)
|
|
windows = np.lib.stride_tricks.sliding_window_view(values, WINDOW, axis=0)
|
|
windows = windows.transpose(0, 2, 1)[starts]
|
|
base_count = len(variant.bar_columns)
|
|
parts = [windows[:, :, :base_count]]
|
|
parts.extend(
|
|
[
|
|
volume_rank[:, :, None],
|
|
(quote_windows - volume_median)[:, :, None],
|
|
]
|
|
)
|
|
if variant.indicator_columns:
|
|
parts.append(windows[:, :, base_count:])
|
|
feature_matrix = np.concatenate(parts, axis=2).reshape(len(anchor_indices), -1)
|
|
asset_values = np.array([asset[name] for name in _ASSET_FEATURE_NAMES], dtype=float)
|
|
feature_matrix = np.column_stack(
|
|
[feature_matrix, np.repeat(asset_values[None, :], len(anchor_indices), axis=0)]
|
|
)
|
|
timestamps = frame["timestamp"].to_numpy(dtype=np.int64)[anchor_indices]
|
|
if variant.global_columns:
|
|
if global_context is None:
|
|
raise ValueError("global context is required by this variant")
|
|
values = global_context.reindex(timestamps)[list(variant.global_columns)].to_numpy(dtype=float)
|
|
feature_matrix = np.column_stack([feature_matrix, values])
|
|
if variant.cross_columns:
|
|
if cross_context is None:
|
|
raise ValueError("cross-sectional context is required by this variant")
|
|
values = cross_context.reindex(timestamps)[list(variant.cross_columns)].to_numpy(dtype=float)
|
|
feature_matrix = np.column_stack([feature_matrix, values])
|
|
return feature_matrix
|
|
|
|
|
|
def build_variant_dataset(
|
|
frames: dict[str, pd.DataFrame],
|
|
variant: FeatureVariant,
|
|
test_start_ms: int,
|
|
end_ms: int,
|
|
asset_scaler: dict[str, dict[str, float]],
|
|
train_symbols: list[str],
|
|
validation_symbols: list[str],
|
|
test_symbols: list[str],
|
|
validation_end_ms: int,
|
|
calibration_end_ms: int,
|
|
volatility_rank_threshold: float,
|
|
) -> tuple[pd.DataFrame, list[str], dict[str, dict[str, float]]]:
|
|
indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()}
|
|
raw_profiles = {
|
|
symbol: asset_profile(frame, test_start_ms)
|
|
for symbol, frame in indicators.items()
|
|
}
|
|
profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None}
|
|
scaled_profiles = {
|
|
symbol: scale_asset_profile(profile, asset_scaler)
|
|
for symbol, profile in profiles.items()
|
|
}
|
|
global_context = None
|
|
cross_context: dict[str, pd.DataFrame] = {}
|
|
if variant.uses_global or variant.uses_cross:
|
|
global_context, cross_context = _cross_sectional_context(indicators, profiles)
|
|
train_set = set(train_symbols)
|
|
validation_set = set(validation_symbols)
|
|
test_set = set(test_symbols)
|
|
step = EIGHT_H_MS
|
|
rows: list[pd.DataFrame] = []
|
|
candidate_counts = {"train": 0, "validation": 0, "calibration": 0, "test": 0}
|
|
selected_counts = {name: 0 for name in candidate_counts}
|
|
names = feature_names(variant)
|
|
for symbol, frame in indicators.items():
|
|
asset = scaled_profiles.get(symbol)
|
|
if asset is None or len(frame) < WINDOW + HORIZON:
|
|
continue
|
|
frame = frame.reset_index(drop=True)
|
|
timestamps = frame["timestamp"].to_numpy(dtype=np.int64)
|
|
anchor_indices = np.arange(WINDOW - 1, len(frame) - HORIZON, dtype=np.int64)
|
|
continuity = np.convolve(
|
|
(np.diff(timestamps) == step).astype(np.int8),
|
|
np.ones(WINDOW + HORIZON - 1, dtype=np.int8),
|
|
mode="valid",
|
|
) == WINDOW + HORIZON - 1
|
|
anchor_ms = timestamps[anchor_indices]
|
|
future_end_ms = timestamps[anchor_indices + HORIZON] + step
|
|
split_labels = np.full(len(anchor_indices), "", dtype=object)
|
|
if symbol in train_set:
|
|
mask = (anchor_ms < test_start_ms) & (future_end_ms <= test_start_ms)
|
|
split_labels[mask] = "train"
|
|
if symbol in validation_set:
|
|
mask = (
|
|
(test_start_ms <= anchor_ms)
|
|
& (anchor_ms < validation_end_ms)
|
|
& (future_end_ms <= validation_end_ms)
|
|
)
|
|
split_labels[mask] = "validation"
|
|
mask = (
|
|
(validation_end_ms <= anchor_ms)
|
|
& (anchor_ms < calibration_end_ms)
|
|
& (future_end_ms <= calibration_end_ms)
|
|
)
|
|
split_labels[mask] = "calibration"
|
|
if symbol in test_set:
|
|
mask = (
|
|
(calibration_end_ms <= anchor_ms)
|
|
& (anchor_ms < end_ms)
|
|
& (future_end_ms <= end_ms)
|
|
)
|
|
split_labels[mask] = "test"
|
|
candidate = continuity & (split_labels != "")
|
|
for split_name in candidate_counts:
|
|
candidate_counts[split_name] += int(
|
|
np.count_nonzero(candidate & (split_labels == split_name))
|
|
)
|
|
volatility_rank = frame["atr_rank_200"].to_numpy(dtype=float)[anchor_indices]
|
|
if volatility_rank_threshold == 0.0:
|
|
selected = candidate
|
|
else:
|
|
selected = candidate & np.isfinite(volatility_rank) & (volatility_rank >= volatility_rank_threshold)
|
|
for split_name in selected_counts:
|
|
selected_counts[split_name] += int(
|
|
np.count_nonzero(selected & (split_labels == split_name))
|
|
)
|
|
if not np.any(selected):
|
|
continue
|
|
selected_indices = anchor_indices[selected]
|
|
feature_matrix = _variant_window_features(
|
|
frame,
|
|
variant,
|
|
selected_indices,
|
|
asset,
|
|
global_context,
|
|
cross_context.get(symbol),
|
|
)
|
|
low = frame["low"].to_numpy(dtype=float)
|
|
close = frame["close"].to_numpy(dtype=float)[selected_indices]
|
|
future_lows = np.column_stack(
|
|
[low[selected_indices + offset] for offset in range(1, HORIZON + 1)]
|
|
)
|
|
future_reference = np.quantile(future_lows, 0.20, axis=1)
|
|
future_return = future_reference / np.maximum(close, 1e-12) - 1.0
|
|
target_scale = max(float(profiles[symbol]["asset_atr_price_top20_median"]), 1e-9)
|
|
target = future_return / target_scale
|
|
sample_atr = frame["atr_pct"].to_numpy(dtype=float)[selected_indices]
|
|
finite = (
|
|
np.isfinite(target)
|
|
& np.isfinite(sample_atr)
|
|
& np.isfinite(feature_matrix).all(axis=1)
|
|
)
|
|
if not np.any(finite):
|
|
continue
|
|
block = pd.DataFrame(feature_matrix[finite], columns=names)
|
|
block["symbol"] = symbol
|
|
block["timestamp"] = anchor_ms[selected][finite]
|
|
block["target"] = target[finite]
|
|
block["target_scale"] = target_scale
|
|
block["split"] = split_labels[selected][finite]
|
|
block["sample_atr_pct"] = sample_atr[finite]
|
|
rows.append(block)
|
|
dataset = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame()
|
|
if dataset.empty:
|
|
raise ValueError(f"variant {variant.name} produced no rows")
|
|
dataset.attrs["volatility_filter"] = {
|
|
"window": VOLATILITY_RANK_WINDOW,
|
|
"threshold": volatility_rank_threshold,
|
|
"candidate_counts": candidate_counts,
|
|
"selected_counts": selected_counts,
|
|
}
|
|
return dataset, names, profiles
|
|
|
|
|
|
def _quantile_model(alpha: float, seed: int, n_estimators: int) -> HistGradientBoostingRegressor:
|
|
return HistGradientBoostingRegressor(
|
|
quantile=alpha,
|
|
max_iter=max(1, n_estimators),
|
|
learning_rate=0.04,
|
|
max_depth=2,
|
|
min_samples_leaf=8,
|
|
random_state=seed,
|
|
verbose=0,
|
|
)
|
|
|
|
|
|
def _fit_variant(
|
|
dataset: pd.DataFrame,
|
|
names: list[str],
|
|
seed: int,
|
|
n_estimators: int,
|
|
n_jobs: int,
|
|
) -> dict[str, Any]:
|
|
train = dataset[dataset["split"] == "train"]
|
|
validation = dataset[dataset["split"] == "validation"]
|
|
if train.empty or validation.empty:
|
|
raise ValueError("variant train or validation rows are empty")
|
|
train_x = train[names]
|
|
train_y = train["target"].to_numpy(dtype=float)
|
|
counts = train["symbol"].value_counts()
|
|
weights = 1.0 / train["symbol"].map(counts).to_numpy(dtype=float)
|
|
weights /= weights.mean()
|
|
lower_model = _quantile_model(LOWER_QUANTILE, seed, n_estimators)
|
|
upper_model = _quantile_model(TARGET_QUANTILE, seed, n_estimators)
|
|
_progress(" fitting q20 quantile model")
|
|
lower_model.fit(train_x, train_y)
|
|
_progress(" fitting q80 quantile model")
|
|
upper_model.fit(train_x, train_y)
|
|
point_model = ExtraTreesRegressor(
|
|
criterion="squared_error",
|
|
n_estimators=n_estimators,
|
|
max_features=0.7,
|
|
min_samples_leaf=8,
|
|
random_state=seed,
|
|
n_jobs=n_jobs,
|
|
verbose=0,
|
|
)
|
|
_progress(" fitting ExtraTrees point model")
|
|
point_model.fit(train_x, train_y, sample_weight=weights)
|
|
validation_x = validation[names]
|
|
validation_y = validation["target"].to_numpy(dtype=float)
|
|
point = point_model.predict(validation_x)
|
|
lower = lower_model.predict(validation_x)
|
|
upper = upper_model.predict(validation_x)
|
|
scale = validation["target_scale"].to_numpy(dtype=float)
|
|
metrics = evaluate_predictions(validation_y, point, lower, upper, scale)
|
|
score = (
|
|
metrics["pinball_q20"]
|
|
+ metrics["pinball_q80"]
|
|
+ 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"])
|
|
+ 0.05 * metrics["mae_point"]
|
|
)
|
|
calibration = dataset[dataset["split"] == "calibration"]
|
|
if calibration.empty:
|
|
raise ValueError("variant calibration rows are empty")
|
|
calibration_x = calibration[names]
|
|
calibration_y = calibration["target"].to_numpy(dtype=float)
|
|
cal_lower = lower_model.predict(calibration_x)
|
|
cal_upper = upper_model.predict(calibration_x)
|
|
offsets = {
|
|
"q20_offset": float(np.quantile(calibration_y - cal_lower, LOWER_QUANTILE)),
|
|
"q80_offset": float(np.quantile(calibration_y - cal_upper, TARGET_QUANTILE)),
|
|
}
|
|
return {
|
|
"point_model": point_model,
|
|
"lower_model": lower_model,
|
|
"upper_model": upper_model,
|
|
"feature_names": names,
|
|
"validation_metrics": metrics,
|
|
"selection_score": float(score),
|
|
"calibration": offsets,
|
|
}
|
|
|
|
|
|
def _evaluate_test(dataset: pd.DataFrame, fitted: dict[str, Any]) -> dict[str, Any]:
|
|
test = dataset[dataset["split"] == "test"]
|
|
features = test[fitted["feature_names"]]
|
|
point = fitted["point_model"].predict(features)
|
|
lower = fitted["lower_model"].predict(features)
|
|
upper = fitted["upper_model"].predict(features)
|
|
lower, upper = apply_calibration(lower, upper, fitted["calibration"])
|
|
y = test["target"].to_numpy(dtype=float)
|
|
scale = test["target_scale"].to_numpy(dtype=float)
|
|
metrics = evaluate_predictions(y, point, lower, upper, scale)
|
|
metrics.update(
|
|
{
|
|
"calibrated_q80_coverage": float(np.mean(y <= upper)),
|
|
"calibrated_q20_hit_rate": float(np.mean(y >= lower)),
|
|
"test_rows": int(len(test)),
|
|
}
|
|
)
|
|
return metrics
|
|
|
|
|
|
def _baseline_score(report: dict[str, Any]) -> float:
|
|
metrics = report["selected"]["validation"]
|
|
return float(
|
|
metrics["pinball_q20"]
|
|
+ metrics["pinball_q80"]
|
|
+ 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"])
|
|
+ 0.05 * metrics["mae_point"]
|
|
)
|
|
|
|
|
|
def _artifact(
|
|
fitted: dict[str, Any],
|
|
variant: FeatureVariant,
|
|
profiles: dict[str, dict[str, float]],
|
|
scaler: dict[str, dict[str, float]],
|
|
manifest: dict[str, Any],
|
|
end_ms: int,
|
|
test_start_ms: int,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"version": 4,
|
|
"timeframe": "8h",
|
|
"window": WINDOW,
|
|
"horizon": HORIZON,
|
|
"target_kind": "signed_future_low_q20_return_atr_normalized",
|
|
"target": "quantile(future_6_low, 0.20) / anchor_close - 1",
|
|
"future_low_quantile": 0.20,
|
|
"target_scale": "asset_atr_price_top20_median",
|
|
"lower_quantile": LOWER_QUANTILE,
|
|
"upper_quantile": TARGET_QUANTILE,
|
|
"volatility_rank_window": VOLATILITY_RANK_WINDOW,
|
|
"volatility_rank_threshold": VOLATILITY_RANK_THRESHOLD,
|
|
"feature_variant": {
|
|
"name": variant.name,
|
|
"description": variant.description,
|
|
"bar_columns": list(variant.bar_columns),
|
|
"indicator_columns": list(variant.indicator_columns),
|
|
"global_columns": list(variant.global_columns),
|
|
"cross_columns": list(variant.cross_columns),
|
|
},
|
|
"stage": variant.name,
|
|
"model_name": "extra_trees_squared_error",
|
|
"model_kind": "point_plus_gbdt_quantiles",
|
|
"feature_names": fitted["feature_names"],
|
|
"asset_scaler": scaler,
|
|
"asset_profiles": profiles,
|
|
"symbol_files": {
|
|
item["symbol"]: item["path"]
|
|
for item in manifest.get("symbols", [])
|
|
if item["symbol"] in profiles
|
|
},
|
|
"calibration": fitted["calibration"],
|
|
"point_model": fitted["point_model"],
|
|
"lower_model": fitted["lower_model"],
|
|
"upper_model": fitted["upper_model"],
|
|
"end_ms": end_ms,
|
|
"test_start_ms": test_start_ms,
|
|
"symbols": sorted(profiles),
|
|
}
|
|
|
|
|
|
def run_experiments(
|
|
data_dir: Path,
|
|
output_dir: Path,
|
|
baseline_artifact: Path = Path("artifacts/v1/model.joblib"),
|
|
baseline_report: Path = Path("artifacts/v1/report.json"),
|
|
test_months: int = 10,
|
|
seed: int = 42,
|
|
n_estimators: int = 250,
|
|
n_jobs: int = -1,
|
|
volatility_rank_threshold: float = VOLATILITY_RANK_THRESHOLD,
|
|
variant_names: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
_progress("loading manifest and raw 8H frames")
|
|
manifest = load_manifest(data_dir)
|
|
frames = load_raw_frames(data_dir, manifest)
|
|
end_ms = floor_to_8h(int(manifest["end_ms"]))
|
|
test_start_ms = months_before(end_ms, test_months)
|
|
indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()}
|
|
raw_profiles = {
|
|
symbol: asset_profile(frame, test_start_ms)
|
|
for symbol, frame in indicators.items()
|
|
}
|
|
profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None}
|
|
train_symbols, validation_symbols, test_symbols = split_holdout_symbols(profiles, seed)
|
|
scaler = fit_asset_scaler(profiles, train_symbols)
|
|
validation_end_ms, calibration_end_ms = recent_split_boundaries(test_start_ms, end_ms)
|
|
_progress(
|
|
f"dataset={len(frames)} symbols; train/validation/test symbols="
|
|
f"{len(train_symbols)}/{len(validation_symbols)}/{len(test_symbols)}"
|
|
)
|
|
stored_baseline = json.loads(baseline_report.read_text())
|
|
stored_baseline_score = _baseline_score(stored_baseline)
|
|
_progress(f"stored baseline validation selection score={stored_baseline_score:.8f}")
|
|
results: list[dict[str, Any]] = []
|
|
fitted_by_name: dict[str, tuple[FeatureVariant, dict[str, Any], pd.DataFrame, dict[str, dict[str, float]]]] = {}
|
|
comparison_baseline_score: float | None = None
|
|
variants = experiment_variants()
|
|
if variant_names:
|
|
wanted = set(variant_names) | {"baseline"}
|
|
unknown = wanted - {variant.name for variant in variants}
|
|
if unknown:
|
|
raise ValueError(f"unknown experiment variants: {sorted(unknown)}")
|
|
variants = [variant for variant in variants if variant.name in wanted]
|
|
for index, variant in enumerate(variants, start=1):
|
|
variant_started = time.monotonic()
|
|
_progress(f"variant {index}/{len(variants)} {variant.name}: {variant.description}")
|
|
dataset, names, variant_profiles = build_variant_dataset(
|
|
frames,
|
|
variant,
|
|
test_start_ms,
|
|
end_ms,
|
|
scaler,
|
|
train_symbols,
|
|
validation_symbols,
|
|
test_symbols,
|
|
validation_end_ms,
|
|
calibration_end_ms,
|
|
volatility_rank_threshold,
|
|
)
|
|
filter_stats = dataset.attrs.get("volatility_filter", {})
|
|
_progress(
|
|
f" features={len(names)}, rows="
|
|
f"{int((dataset['split'] == 'train').sum())}/"
|
|
f"{int((dataset['split'] == 'validation').sum())}/"
|
|
f"{int((dataset['split'] == 'calibration').sum())}/"
|
|
f"{int((dataset['split'] == 'test').sum())}"
|
|
)
|
|
fitted = _fit_variant(dataset, names, seed, n_estimators, n_jobs)
|
|
if variant.name == "baseline":
|
|
comparison_baseline_score = fitted["selection_score"]
|
|
_progress(f" rerun baseline validation selection score={comparison_baseline_score:.8f}")
|
|
if comparison_baseline_score is None:
|
|
raise RuntimeError("baseline variant must be evaluated before comparisons")
|
|
improved = (
|
|
variant.name != "baseline"
|
|
and fitted["selection_score"] < comparison_baseline_score
|
|
)
|
|
record = {
|
|
"name": variant.name,
|
|
"description": variant.description,
|
|
"feature_count": len(names),
|
|
"train_rows": int((dataset["split"] == "train").sum()),
|
|
"validation_rows": int((dataset["split"] == "validation").sum()),
|
|
"calibration_rows": int((dataset["split"] == "calibration").sum()),
|
|
"test_rows": int((dataset["split"] == "test").sum()),
|
|
"validation_selection_score": fitted["selection_score"],
|
|
"improves_baseline_validation": improved,
|
|
"validation": fitted["validation_metrics"],
|
|
"volatility_filter": filter_stats,
|
|
"elapsed_seconds": round(time.monotonic() - variant_started, 2),
|
|
}
|
|
results.append(record)
|
|
fitted_by_name[variant.name] = (variant, fitted, dataset, variant_profiles)
|
|
_progress(
|
|
f" result score={fitted['selection_score']:.8f}; "
|
|
f"baseline_delta={fitted['selection_score'] - comparison_baseline_score:+.8f}; "
|
|
f"elapsed={record['elapsed_seconds']:.1f}s"
|
|
)
|
|
if comparison_baseline_score is None:
|
|
raise RuntimeError("baseline result is missing")
|
|
best_record = min(results, key=lambda item: item["validation_selection_score"])
|
|
best_variant, best_fitted, best_dataset, best_profiles = fitted_by_name[best_record["name"]]
|
|
_progress(f"selected by validation: {best_variant.name}")
|
|
_, baseline_fitted, baseline_dataset, _ = fitted_by_name["baseline"]
|
|
baseline_test = _evaluate_test(baseline_dataset, baseline_fitted)
|
|
baseline = {
|
|
"artifact": str(baseline_artifact.resolve()),
|
|
"report": str(baseline_report.resolve()),
|
|
"feature_count": 182,
|
|
"stored_validation_selection_score": stored_baseline_score,
|
|
"validation_selection_score": comparison_baseline_score,
|
|
"validation": next(item["validation"] for item in results if item["name"] == "baseline"),
|
|
"test": baseline_test,
|
|
"stored_test": stored_baseline["test"],
|
|
}
|
|
best_test = _evaluate_test(best_dataset, best_fitted)
|
|
best_dir = output_dir / "best"
|
|
best_dir.mkdir(parents=True, exist_ok=True)
|
|
joblib.dump(
|
|
_artifact(
|
|
best_fitted,
|
|
best_variant,
|
|
best_profiles,
|
|
scaler,
|
|
manifest,
|
|
end_ms,
|
|
test_start_ms,
|
|
),
|
|
best_dir / "model.joblib",
|
|
)
|
|
report = {
|
|
"protocol": {
|
|
"data_dir": str(data_dir.resolve()),
|
|
"end_ms": end_ms,
|
|
"test_start_ms": test_start_ms,
|
|
"validation_end_ms": validation_end_ms,
|
|
"calibration_end_ms": calibration_end_ms,
|
|
"test_months": test_months,
|
|
"seed": seed,
|
|
"n_estimators": n_estimators,
|
|
"n_jobs": n_jobs,
|
|
"volatility_rank_threshold": volatility_rank_threshold,
|
|
"train_symbols": train_symbols,
|
|
"validation_symbols": validation_symbols,
|
|
"test_symbols": test_symbols,
|
|
},
|
|
"baseline": baseline,
|
|
"variants": results,
|
|
"selected": {
|
|
**best_record,
|
|
"test": best_test,
|
|
"artifact": str((best_dir / "model.joblib").resolve()),
|
|
},
|
|
"elapsed_seconds": round(time.monotonic() - started, 2),
|
|
}
|
|
(output_dir / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False))
|
|
_write_report_markdown(output_dir / "report.md", report)
|
|
_progress(f"experiment report written to {output_dir}; total={report['elapsed_seconds']:.1f}s")
|
|
return report
|
|
|
|
|
|
def _write_report_markdown(path: Path, report: dict[str, Any]) -> None:
|
|
baseline = report["baseline"]
|
|
selected = report["selected"]
|
|
lines = [
|
|
"# Feature experiment report",
|
|
"",
|
|
f"- baseline: `{baseline['feature_count']}` features, validation score `{baseline['validation_selection_score']:.8f}`",
|
|
f"- selected: `{selected['name']}`, `{selected['feature_count']}` features, validation score `{selected['validation_selection_score']:.8f}`",
|
|
f"- selected test MAE/RMSE: `{selected['test']['mae_point']:.6f}` / `{selected['test']['rmse_point']:.6f}`",
|
|
f"- baseline test MAE/RMSE: `{baseline['test']['mae_point']:.6f}` / `{baseline['test']['rmse_point']:.6f}`",
|
|
"",
|
|
"## Variant comparison",
|
|
"",
|
|
"| variant | features | validation score | delta vs baseline | validation MAE | q80 coverage |",
|
|
"| --- | ---: | ---: | ---: | ---: | ---: |",
|
|
]
|
|
for item in report["variants"]:
|
|
metrics = item["validation"]
|
|
delta = item["validation_selection_score"] - baseline["validation_selection_score"]
|
|
lines.append(
|
|
f"| `{item['name']}` | {item['feature_count']} | {item['validation_selection_score']:.8f} | {delta:+.8f} | "
|
|
f"{metrics['mae_point']:.6f} | {metrics['q80_coverage']:.4f} |"
|
|
)
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"Selection uses only the validation selection score. The final test block is reported for the baseline and the validation-selected candidate; it is not used to choose features.",
|
|
]
|
|
)
|
|
path.write_text("\n".join(lines) + "\n")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Compare compact, indicator, global, and cross-sectional features")
|
|
parser.add_argument("--data-dir", type=Path, default=Path("data"))
|
|
parser.add_argument("--output-dir", type=Path, default=Path("artifacts/experiments"))
|
|
parser.add_argument("--baseline-artifact", type=Path, default=Path("artifacts/v1/model.joblib"))
|
|
parser.add_argument("--baseline-report", type=Path, default=Path("artifacts/v1/report.json"))
|
|
parser.add_argument("--test-months", type=int, default=10)
|
|
parser.add_argument("--seed", type=int, default=42)
|
|
parser.add_argument("--n-estimators", type=int, default=250)
|
|
parser.add_argument("--n-jobs", type=int, default=-1)
|
|
parser.add_argument("--volatility-rank-threshold", type=float, default=VOLATILITY_RANK_THRESHOLD)
|
|
parser.add_argument(
|
|
"--variants",
|
|
default=None,
|
|
help="comma-separated variants to run; baseline is always included",
|
|
)
|
|
args = parser.parse_args()
|
|
report = run_experiments(
|
|
args.data_dir,
|
|
args.output_dir,
|
|
args.baseline_artifact,
|
|
args.baseline_report,
|
|
args.test_months,
|
|
args.seed,
|
|
args.n_estimators,
|
|
args.n_jobs,
|
|
args.volatility_rank_threshold,
|
|
[item.strip() for item in args.variants.split(",") if item.strip()]
|
|
if args.variants
|
|
else None,
|
|
)
|
|
print(json.dumps({
|
|
"selected": report["selected"]["name"],
|
|
"baseline_score": report["baseline"]["validation_selection_score"],
|
|
"selected_score": report["selected"]["validation_selection_score"],
|
|
"selected_test_mae": report["selected"]["test"]["mae_point"],
|
|
"selected_test_rmse": report["selected"]["test"]["rmse_point"],
|
|
"report": str((args.output_dir / "report.json").resolve()),
|
|
"artifact": report["selected"]["artifact"],
|
|
}, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|