update model version
This commit is contained in:
@@ -35,7 +35,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
train = sub.add_parser("train", help="build three feature stages, tune on validation symbols, test once")
|
||||
train.add_argument("--data-dir", type=Path, default=Path("data"))
|
||||
train.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed"))
|
||||
train.add_argument("--artifact-dir", type=Path, default=Path("artifacts/v1"))
|
||||
train.add_argument("--end", dest="end_ms", default=None)
|
||||
train.add_argument("--test-months", type=int, default=10)
|
||||
train.add_argument("--seed", type=int, default=42)
|
||||
@@ -47,7 +47,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
for action in prepare._actions[1:]:
|
||||
if action.dest not in {"help", "command"}:
|
||||
run._add_action(action)
|
||||
run.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed"))
|
||||
run.add_argument("--artifact-dir", type=Path, default=Path("artifacts/v1"))
|
||||
run.add_argument("--test-months", type=int, default=10)
|
||||
run.add_argument("--seed", type=int, default=42)
|
||||
run.add_argument("--n-estimators", type=int, default=250)
|
||||
@@ -57,7 +57,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
predict = sub.add_parser(
|
||||
"predict", help="predict signed robust future-low q20/q80 from an exported 8H CSV"
|
||||
)
|
||||
predict.add_argument("--artifact", type=Path, default=Path("artifacts_signed/model.joblib"))
|
||||
predict.add_argument("--artifact", type=Path, default=Path("artifacts/v1/model.joblib"))
|
||||
predict.add_argument("--csv", type=Path, required=True)
|
||||
predict.add_argument("--at", dest="at_ms", default=None)
|
||||
predict.add_argument("--symbol", default=None, help="symbol used for the fixed training asset profile")
|
||||
|
||||
799
cbpred/experiments.py
Normal file
799
cbpred/experiments.py
Normal file
@@ -0,0 +1,799 @@
|
||||
"""Fair, progress-reporting feature ablation experiments for the 8H model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.ensemble import ExtraTreesRegressor, HistGradientBoostingRegressor
|
||||
|
||||
from .pipeline import (
|
||||
EIGHT_H_MS,
|
||||
HORIZON,
|
||||
LOWER_QUANTILE,
|
||||
TARGET_QUANTILE,
|
||||
VOLATILITY_RANK_THRESHOLD,
|
||||
VOLATILITY_RANK_WINDOW,
|
||||
WINDOW,
|
||||
_ASSET_FEATURE_NAMES,
|
||||
_BASE_FEATURE_COLUMNS,
|
||||
_rank_windows,
|
||||
apply_calibration,
|
||||
asset_profile,
|
||||
compute_indicators,
|
||||
evaluate_predictions,
|
||||
fit_asset_scaler,
|
||||
floor_to_8h,
|
||||
load_manifest,
|
||||
load_raw_frames,
|
||||
months_before,
|
||||
recent_split_boundaries,
|
||||
scale_asset_profile,
|
||||
split_holdout_symbols,
|
||||
)
|
||||
|
||||
|
||||
def _progress(message: str) -> None:
|
||||
print(f"[cbpred-experiment] {message}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeatureVariant:
|
||||
name: str
|
||||
description: str
|
||||
bar_columns: tuple[str, ...]
|
||||
indicator_columns: tuple[str, ...] = ()
|
||||
global_columns: tuple[str, ...] = ()
|
||||
cross_columns: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def uses_global(self) -> bool:
|
||||
return bool(self.global_columns)
|
||||
|
||||
@property
|
||||
def uses_cross(self) -> bool:
|
||||
return bool(self.cross_columns)
|
||||
|
||||
|
||||
_VOLUME_COLUMNS = ("quote_volume_rank", "quote_volume_log_rel_median")
|
||||
_ALL_BAR_COLUMNS = _BASE_FEATURE_COLUMNS
|
||||
_COMPACT7_COLUMNS = tuple(name for name in _ALL_BAR_COLUMNS if name != "body_abs_range")
|
||||
_COMPACT6_COLUMNS = tuple(
|
||||
name for name in _COMPACT7_COLUMNS if name != "body_sign"
|
||||
)
|
||||
_GLOBAL_COLUMNS = (
|
||||
"global_median_return_1",
|
||||
"global_median_trend_3",
|
||||
"global_median_atr_pct",
|
||||
"global_return_dispersion",
|
||||
"global_up_ratio",
|
||||
"global_median_quote_activity",
|
||||
)
|
||||
_CROSS_COLUMNS = (
|
||||
"cross_return_1_rank",
|
||||
"cross_trend_3_rank",
|
||||
"cross_atr_pct_rank",
|
||||
"cross_quote_activity_rank",
|
||||
)
|
||||
|
||||
|
||||
def experiment_variants() -> list[FeatureVariant]:
|
||||
"""Return independent candidates; baseline is deliberately first."""
|
||||
base = tuple(_ALL_BAR_COLUMNS)
|
||||
return [
|
||||
FeatureVariant("baseline", "current Stage 1 feature set", base),
|
||||
FeatureVariant("compact7", "remove absolute body size", _COMPACT7_COLUMNS),
|
||||
FeatureVariant(
|
||||
"compact6",
|
||||
"remove absolute body size and explicit body sign",
|
||||
_COMPACT6_COLUMNS,
|
||||
),
|
||||
FeatureVariant(
|
||||
"rsi_sma",
|
||||
"add normalized RSI and normalized SMA20/SMA50 distance",
|
||||
base,
|
||||
("rsi_norm", "sma20_gap", "sma50_gap"),
|
||||
),
|
||||
FeatureVariant(
|
||||
"rsi_bbands",
|
||||
"add normalized RSI and Bollinger position/width",
|
||||
base,
|
||||
("rsi_norm", "bb_position", "bb_width"),
|
||||
),
|
||||
FeatureVariant(
|
||||
"rsi_trend",
|
||||
"add normalized RSI, short trends, and rolling volatility",
|
||||
base,
|
||||
("rsi_norm", "trend_3", "trend_9", "volatility_18"),
|
||||
),
|
||||
FeatureVariant(
|
||||
"full_indicators",
|
||||
"add the complete existing indicator group",
|
||||
base,
|
||||
(
|
||||
"atr_pct",
|
||||
"atr_rank_200",
|
||||
"rsi_norm",
|
||||
"bb_position",
|
||||
"bb_width",
|
||||
"ema9_gap",
|
||||
"ema21_gap",
|
||||
"trend_3",
|
||||
"trend_9",
|
||||
"volatility_18",
|
||||
"drawdown_18",
|
||||
),
|
||||
),
|
||||
FeatureVariant(
|
||||
"global6",
|
||||
"add six cross-symbol market-state summaries",
|
||||
base,
|
||||
global_columns=_GLOBAL_COLUMNS,
|
||||
),
|
||||
FeatureVariant(
|
||||
"cross4",
|
||||
"add four current-symbol cross-sectional ranks",
|
||||
base,
|
||||
cross_columns=_CROSS_COLUMNS,
|
||||
),
|
||||
FeatureVariant(
|
||||
"global_cross",
|
||||
"add global market summaries and current-symbol ranks",
|
||||
base,
|
||||
global_columns=_GLOBAL_COLUMNS,
|
||||
cross_columns=_CROSS_COLUMNS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def feature_names(variant: FeatureVariant) -> list[str]:
|
||||
names: list[str] = []
|
||||
per_bar = (*variant.bar_columns, *_VOLUME_COLUMNS, *variant.indicator_columns)
|
||||
for lag in range(WINDOW - 1, -1, -1):
|
||||
names.extend(f"bar_{lag:02d}_{name}" for name in per_bar)
|
||||
names.extend(_ASSET_FEATURE_NAMES)
|
||||
names.extend(variant.global_columns)
|
||||
names.extend(variant.cross_columns)
|
||||
return names
|
||||
|
||||
|
||||
def _cross_sectional_context(
|
||||
indicators: dict[str, pd.DataFrame],
|
||||
raw_profiles: dict[str, dict[str, float]],
|
||||
) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]:
|
||||
records: list[pd.DataFrame] = []
|
||||
for symbol, frame in indicators.items():
|
||||
profile = raw_profiles.get(symbol)
|
||||
if profile is None:
|
||||
continue
|
||||
record = pd.DataFrame(
|
||||
{
|
||||
"symbol": symbol,
|
||||
"timestamp": frame["timestamp"].to_numpy(dtype=np.int64),
|
||||
"return_1": frame["log_return"].to_numpy(dtype=float),
|
||||
"trend_3": frame["trend_3"].to_numpy(dtype=float),
|
||||
"atr_pct": frame["atr_pct"].to_numpy(dtype=float),
|
||||
"body_sign": frame["body_sign"].to_numpy(dtype=float),
|
||||
"quote_activity": frame["quote_log"].to_numpy(dtype=float)
|
||||
- float(profile["asset_quote_volume_top20_median_log"]),
|
||||
}
|
||||
)
|
||||
records.append(record)
|
||||
if not records:
|
||||
raise ValueError("no cross-sectional records available")
|
||||
all_rows = pd.concat(records, ignore_index=True)
|
||||
grouped = all_rows.groupby("timestamp", sort=False)
|
||||
global_context = grouped.agg(
|
||||
global_median_return_1=("return_1", "median"),
|
||||
global_median_trend_3=("trend_3", "median"),
|
||||
global_median_atr_pct=("atr_pct", "median"),
|
||||
global_return_dispersion=("return_1", lambda values: values.std(ddof=0)),
|
||||
global_up_ratio=("body_sign", lambda values: float(np.mean(values > 0))),
|
||||
global_median_quote_activity=("quote_activity", "median"),
|
||||
)
|
||||
rank_columns = {
|
||||
"return_1": "cross_return_1_rank",
|
||||
"trend_3": "cross_trend_3_rank",
|
||||
"atr_pct": "cross_atr_pct_rank",
|
||||
"quote_activity": "cross_quote_activity_rank",
|
||||
}
|
||||
for source, target in rank_columns.items():
|
||||
all_rows[target] = all_rows.groupby("timestamp")[source].rank(
|
||||
method="average", pct=True
|
||||
)
|
||||
cross_context = {
|
||||
symbol: group.set_index("timestamp")[list(rank_columns.values())].sort_index()
|
||||
for symbol, group in all_rows.groupby("symbol", sort=False)
|
||||
}
|
||||
return global_context.sort_index(), cross_context
|
||||
|
||||
|
||||
def _variant_window_features(
|
||||
frame: pd.DataFrame,
|
||||
variant: FeatureVariant,
|
||||
anchor_indices: np.ndarray,
|
||||
asset: dict[str, float],
|
||||
global_context: pd.DataFrame | None = None,
|
||||
cross_context: pd.DataFrame | None = None,
|
||||
) -> np.ndarray:
|
||||
starts = anchor_indices - (WINDOW - 1)
|
||||
quote = frame["quote_log"].to_numpy(dtype=float)
|
||||
quote_windows = np.lib.stride_tricks.sliding_window_view(quote, WINDOW)[starts]
|
||||
volume_rank = _rank_windows(quote_windows)
|
||||
volume_median = np.median(quote_windows, axis=1, keepdims=True)
|
||||
columns = [*variant.bar_columns, *variant.indicator_columns]
|
||||
values = frame[columns].to_numpy(dtype=float)
|
||||
windows = np.lib.stride_tricks.sliding_window_view(values, WINDOW, axis=0)
|
||||
windows = windows.transpose(0, 2, 1)[starts]
|
||||
base_count = len(variant.bar_columns)
|
||||
parts = [windows[:, :, :base_count]]
|
||||
parts.extend(
|
||||
[
|
||||
volume_rank[:, :, None],
|
||||
(quote_windows - volume_median)[:, :, None],
|
||||
]
|
||||
)
|
||||
if variant.indicator_columns:
|
||||
parts.append(windows[:, :, base_count:])
|
||||
feature_matrix = np.concatenate(parts, axis=2).reshape(len(anchor_indices), -1)
|
||||
asset_values = np.array([asset[name] for name in _ASSET_FEATURE_NAMES], dtype=float)
|
||||
feature_matrix = np.column_stack(
|
||||
[feature_matrix, np.repeat(asset_values[None, :], len(anchor_indices), axis=0)]
|
||||
)
|
||||
timestamps = frame["timestamp"].to_numpy(dtype=np.int64)[anchor_indices]
|
||||
if variant.global_columns:
|
||||
if global_context is None:
|
||||
raise ValueError("global context is required by this variant")
|
||||
values = global_context.reindex(timestamps)[list(variant.global_columns)].to_numpy(dtype=float)
|
||||
feature_matrix = np.column_stack([feature_matrix, values])
|
||||
if variant.cross_columns:
|
||||
if cross_context is None:
|
||||
raise ValueError("cross-sectional context is required by this variant")
|
||||
values = cross_context.reindex(timestamps)[list(variant.cross_columns)].to_numpy(dtype=float)
|
||||
feature_matrix = np.column_stack([feature_matrix, values])
|
||||
return feature_matrix
|
||||
|
||||
|
||||
def build_variant_dataset(
|
||||
frames: dict[str, pd.DataFrame],
|
||||
variant: FeatureVariant,
|
||||
test_start_ms: int,
|
||||
end_ms: int,
|
||||
asset_scaler: dict[str, dict[str, float]],
|
||||
train_symbols: list[str],
|
||||
validation_symbols: list[str],
|
||||
test_symbols: list[str],
|
||||
validation_end_ms: int,
|
||||
calibration_end_ms: int,
|
||||
volatility_rank_threshold: float,
|
||||
) -> tuple[pd.DataFrame, list[str], dict[str, dict[str, float]]]:
|
||||
indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()}
|
||||
raw_profiles = {
|
||||
symbol: asset_profile(frame, test_start_ms)
|
||||
for symbol, frame in indicators.items()
|
||||
}
|
||||
profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None}
|
||||
scaled_profiles = {
|
||||
symbol: scale_asset_profile(profile, asset_scaler)
|
||||
for symbol, profile in profiles.items()
|
||||
}
|
||||
global_context = None
|
||||
cross_context: dict[str, pd.DataFrame] = {}
|
||||
if variant.uses_global or variant.uses_cross:
|
||||
global_context, cross_context = _cross_sectional_context(indicators, profiles)
|
||||
train_set = set(train_symbols)
|
||||
validation_set = set(validation_symbols)
|
||||
test_set = set(test_symbols)
|
||||
step = EIGHT_H_MS
|
||||
rows: list[pd.DataFrame] = []
|
||||
candidate_counts = {"train": 0, "validation": 0, "calibration": 0, "test": 0}
|
||||
selected_counts = {name: 0 for name in candidate_counts}
|
||||
names = feature_names(variant)
|
||||
for symbol, frame in indicators.items():
|
||||
asset = scaled_profiles.get(symbol)
|
||||
if asset is None or len(frame) < WINDOW + HORIZON:
|
||||
continue
|
||||
frame = frame.reset_index(drop=True)
|
||||
timestamps = frame["timestamp"].to_numpy(dtype=np.int64)
|
||||
anchor_indices = np.arange(WINDOW - 1, len(frame) - HORIZON, dtype=np.int64)
|
||||
continuity = np.convolve(
|
||||
(np.diff(timestamps) == step).astype(np.int8),
|
||||
np.ones(WINDOW + HORIZON - 1, dtype=np.int8),
|
||||
mode="valid",
|
||||
) == WINDOW + HORIZON - 1
|
||||
anchor_ms = timestamps[anchor_indices]
|
||||
future_end_ms = timestamps[anchor_indices + HORIZON] + step
|
||||
split_labels = np.full(len(anchor_indices), "", dtype=object)
|
||||
if symbol in train_set:
|
||||
mask = (anchor_ms < test_start_ms) & (future_end_ms <= test_start_ms)
|
||||
split_labels[mask] = "train"
|
||||
if symbol in validation_set:
|
||||
mask = (
|
||||
(test_start_ms <= anchor_ms)
|
||||
& (anchor_ms < validation_end_ms)
|
||||
& (future_end_ms <= validation_end_ms)
|
||||
)
|
||||
split_labels[mask] = "validation"
|
||||
mask = (
|
||||
(validation_end_ms <= anchor_ms)
|
||||
& (anchor_ms < calibration_end_ms)
|
||||
& (future_end_ms <= calibration_end_ms)
|
||||
)
|
||||
split_labels[mask] = "calibration"
|
||||
if symbol in test_set:
|
||||
mask = (
|
||||
(calibration_end_ms <= anchor_ms)
|
||||
& (anchor_ms < end_ms)
|
||||
& (future_end_ms <= end_ms)
|
||||
)
|
||||
split_labels[mask] = "test"
|
||||
candidate = continuity & (split_labels != "")
|
||||
for split_name in candidate_counts:
|
||||
candidate_counts[split_name] += int(
|
||||
np.count_nonzero(candidate & (split_labels == split_name))
|
||||
)
|
||||
volatility_rank = frame["atr_rank_200"].to_numpy(dtype=float)[anchor_indices]
|
||||
if volatility_rank_threshold == 0.0:
|
||||
selected = candidate
|
||||
else:
|
||||
selected = candidate & np.isfinite(volatility_rank) & (volatility_rank >= volatility_rank_threshold)
|
||||
for split_name in selected_counts:
|
||||
selected_counts[split_name] += int(
|
||||
np.count_nonzero(selected & (split_labels == split_name))
|
||||
)
|
||||
if not np.any(selected):
|
||||
continue
|
||||
selected_indices = anchor_indices[selected]
|
||||
feature_matrix = _variant_window_features(
|
||||
frame,
|
||||
variant,
|
||||
selected_indices,
|
||||
asset,
|
||||
global_context,
|
||||
cross_context.get(symbol),
|
||||
)
|
||||
low = frame["low"].to_numpy(dtype=float)
|
||||
close = frame["close"].to_numpy(dtype=float)[selected_indices]
|
||||
future_lows = np.column_stack(
|
||||
[low[selected_indices + offset] for offset in range(1, HORIZON + 1)]
|
||||
)
|
||||
future_reference = np.quantile(future_lows, 0.20, axis=1)
|
||||
future_return = future_reference / np.maximum(close, 1e-12) - 1.0
|
||||
target_scale = max(float(profiles[symbol]["asset_atr_price_top20_median"]), 1e-9)
|
||||
target = future_return / target_scale
|
||||
sample_atr = frame["atr_pct"].to_numpy(dtype=float)[selected_indices]
|
||||
finite = (
|
||||
np.isfinite(target)
|
||||
& np.isfinite(sample_atr)
|
||||
& np.isfinite(feature_matrix).all(axis=1)
|
||||
)
|
||||
if not np.any(finite):
|
||||
continue
|
||||
block = pd.DataFrame(feature_matrix[finite], columns=names)
|
||||
block["symbol"] = symbol
|
||||
block["timestamp"] = anchor_ms[selected][finite]
|
||||
block["target"] = target[finite]
|
||||
block["target_scale"] = target_scale
|
||||
block["split"] = split_labels[selected][finite]
|
||||
block["sample_atr_pct"] = sample_atr[finite]
|
||||
rows.append(block)
|
||||
dataset = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame()
|
||||
if dataset.empty:
|
||||
raise ValueError(f"variant {variant.name} produced no rows")
|
||||
dataset.attrs["volatility_filter"] = {
|
||||
"window": VOLATILITY_RANK_WINDOW,
|
||||
"threshold": volatility_rank_threshold,
|
||||
"candidate_counts": candidate_counts,
|
||||
"selected_counts": selected_counts,
|
||||
}
|
||||
return dataset, names, profiles
|
||||
|
||||
|
||||
def _quantile_model(alpha: float, seed: int, n_estimators: int) -> HistGradientBoostingRegressor:
|
||||
return HistGradientBoostingRegressor(
|
||||
quantile=alpha,
|
||||
max_iter=max(1, n_estimators),
|
||||
learning_rate=0.04,
|
||||
max_depth=2,
|
||||
min_samples_leaf=8,
|
||||
random_state=seed,
|
||||
verbose=0,
|
||||
)
|
||||
|
||||
|
||||
def _fit_variant(
|
||||
dataset: pd.DataFrame,
|
||||
names: list[str],
|
||||
seed: int,
|
||||
n_estimators: int,
|
||||
n_jobs: int,
|
||||
) -> dict[str, Any]:
|
||||
train = dataset[dataset["split"] == "train"]
|
||||
validation = dataset[dataset["split"] == "validation"]
|
||||
if train.empty or validation.empty:
|
||||
raise ValueError("variant train or validation rows are empty")
|
||||
train_x = train[names]
|
||||
train_y = train["target"].to_numpy(dtype=float)
|
||||
counts = train["symbol"].value_counts()
|
||||
weights = 1.0 / train["symbol"].map(counts).to_numpy(dtype=float)
|
||||
weights /= weights.mean()
|
||||
lower_model = _quantile_model(LOWER_QUANTILE, seed, n_estimators)
|
||||
upper_model = _quantile_model(TARGET_QUANTILE, seed, n_estimators)
|
||||
_progress(" fitting q20 quantile model")
|
||||
lower_model.fit(train_x, train_y)
|
||||
_progress(" fitting q80 quantile model")
|
||||
upper_model.fit(train_x, train_y)
|
||||
point_model = ExtraTreesRegressor(
|
||||
criterion="squared_error",
|
||||
n_estimators=n_estimators,
|
||||
max_features=0.7,
|
||||
min_samples_leaf=8,
|
||||
random_state=seed,
|
||||
n_jobs=n_jobs,
|
||||
verbose=0,
|
||||
)
|
||||
_progress(" fitting ExtraTrees point model")
|
||||
point_model.fit(train_x, train_y, sample_weight=weights)
|
||||
validation_x = validation[names]
|
||||
validation_y = validation["target"].to_numpy(dtype=float)
|
||||
point = point_model.predict(validation_x)
|
||||
lower = lower_model.predict(validation_x)
|
||||
upper = upper_model.predict(validation_x)
|
||||
scale = validation["target_scale"].to_numpy(dtype=float)
|
||||
metrics = evaluate_predictions(validation_y, point, lower, upper, scale)
|
||||
score = (
|
||||
metrics["pinball_q20"]
|
||||
+ metrics["pinball_q80"]
|
||||
+ 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"])
|
||||
+ 0.05 * metrics["mae_point"]
|
||||
)
|
||||
calibration = dataset[dataset["split"] == "calibration"]
|
||||
if calibration.empty:
|
||||
raise ValueError("variant calibration rows are empty")
|
||||
calibration_x = calibration[names]
|
||||
calibration_y = calibration["target"].to_numpy(dtype=float)
|
||||
cal_lower = lower_model.predict(calibration_x)
|
||||
cal_upper = upper_model.predict(calibration_x)
|
||||
offsets = {
|
||||
"q20_offset": float(np.quantile(calibration_y - cal_lower, LOWER_QUANTILE)),
|
||||
"q80_offset": float(np.quantile(calibration_y - cal_upper, TARGET_QUANTILE)),
|
||||
}
|
||||
return {
|
||||
"point_model": point_model,
|
||||
"lower_model": lower_model,
|
||||
"upper_model": upper_model,
|
||||
"feature_names": names,
|
||||
"validation_metrics": metrics,
|
||||
"selection_score": float(score),
|
||||
"calibration": offsets,
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_test(dataset: pd.DataFrame, fitted: dict[str, Any]) -> dict[str, Any]:
|
||||
test = dataset[dataset["split"] == "test"]
|
||||
features = test[fitted["feature_names"]]
|
||||
point = fitted["point_model"].predict(features)
|
||||
lower = fitted["lower_model"].predict(features)
|
||||
upper = fitted["upper_model"].predict(features)
|
||||
lower, upper = apply_calibration(lower, upper, fitted["calibration"])
|
||||
y = test["target"].to_numpy(dtype=float)
|
||||
scale = test["target_scale"].to_numpy(dtype=float)
|
||||
metrics = evaluate_predictions(y, point, lower, upper, scale)
|
||||
metrics.update(
|
||||
{
|
||||
"calibrated_q80_coverage": float(np.mean(y <= upper)),
|
||||
"calibrated_q20_hit_rate": float(np.mean(y >= lower)),
|
||||
"test_rows": int(len(test)),
|
||||
}
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
def _baseline_score(report: dict[str, Any]) -> float:
|
||||
metrics = report["selected"]["validation"]
|
||||
return float(
|
||||
metrics["pinball_q20"]
|
||||
+ metrics["pinball_q80"]
|
||||
+ 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"])
|
||||
+ 0.05 * metrics["mae_point"]
|
||||
)
|
||||
|
||||
|
||||
def _artifact(
|
||||
fitted: dict[str, Any],
|
||||
variant: FeatureVariant,
|
||||
profiles: dict[str, dict[str, float]],
|
||||
scaler: dict[str, dict[str, float]],
|
||||
manifest: dict[str, Any],
|
||||
end_ms: int,
|
||||
test_start_ms: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"version": 4,
|
||||
"timeframe": "8h",
|
||||
"window": WINDOW,
|
||||
"horizon": HORIZON,
|
||||
"target_kind": "signed_future_low_q20_return_atr_normalized",
|
||||
"target": "quantile(future_6_low, 0.20) / anchor_close - 1",
|
||||
"future_low_quantile": 0.20,
|
||||
"target_scale": "asset_atr_price_top20_median",
|
||||
"lower_quantile": LOWER_QUANTILE,
|
||||
"upper_quantile": TARGET_QUANTILE,
|
||||
"volatility_rank_window": VOLATILITY_RANK_WINDOW,
|
||||
"volatility_rank_threshold": VOLATILITY_RANK_THRESHOLD,
|
||||
"feature_variant": {
|
||||
"name": variant.name,
|
||||
"description": variant.description,
|
||||
"bar_columns": list(variant.bar_columns),
|
||||
"indicator_columns": list(variant.indicator_columns),
|
||||
"global_columns": list(variant.global_columns),
|
||||
"cross_columns": list(variant.cross_columns),
|
||||
},
|
||||
"stage": variant.name,
|
||||
"model_name": "extra_trees_squared_error",
|
||||
"model_kind": "point_plus_gbdt_quantiles",
|
||||
"feature_names": fitted["feature_names"],
|
||||
"asset_scaler": scaler,
|
||||
"asset_profiles": profiles,
|
||||
"symbol_files": {
|
||||
item["symbol"]: item["path"]
|
||||
for item in manifest.get("symbols", [])
|
||||
if item["symbol"] in profiles
|
||||
},
|
||||
"calibration": fitted["calibration"],
|
||||
"point_model": fitted["point_model"],
|
||||
"lower_model": fitted["lower_model"],
|
||||
"upper_model": fitted["upper_model"],
|
||||
"end_ms": end_ms,
|
||||
"test_start_ms": test_start_ms,
|
||||
"symbols": sorted(profiles),
|
||||
}
|
||||
|
||||
|
||||
def run_experiments(
|
||||
data_dir: Path,
|
||||
output_dir: Path,
|
||||
baseline_artifact: Path = Path("artifacts/v1/model.joblib"),
|
||||
baseline_report: Path = Path("artifacts/v1/report.json"),
|
||||
test_months: int = 10,
|
||||
seed: int = 42,
|
||||
n_estimators: int = 250,
|
||||
n_jobs: int = -1,
|
||||
volatility_rank_threshold: float = VOLATILITY_RANK_THRESHOLD,
|
||||
variant_names: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
started = time.monotonic()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
_progress("loading manifest and raw 8H frames")
|
||||
manifest = load_manifest(data_dir)
|
||||
frames = load_raw_frames(data_dir, manifest)
|
||||
end_ms = floor_to_8h(int(manifest["end_ms"]))
|
||||
test_start_ms = months_before(end_ms, test_months)
|
||||
indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()}
|
||||
raw_profiles = {
|
||||
symbol: asset_profile(frame, test_start_ms)
|
||||
for symbol, frame in indicators.items()
|
||||
}
|
||||
profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None}
|
||||
train_symbols, validation_symbols, test_symbols = split_holdout_symbols(profiles, seed)
|
||||
scaler = fit_asset_scaler(profiles, train_symbols)
|
||||
validation_end_ms, calibration_end_ms = recent_split_boundaries(test_start_ms, end_ms)
|
||||
_progress(
|
||||
f"dataset={len(frames)} symbols; train/validation/test symbols="
|
||||
f"{len(train_symbols)}/{len(validation_symbols)}/{len(test_symbols)}"
|
||||
)
|
||||
stored_baseline = json.loads(baseline_report.read_text())
|
||||
stored_baseline_score = _baseline_score(stored_baseline)
|
||||
_progress(f"stored baseline validation selection score={stored_baseline_score:.8f}")
|
||||
results: list[dict[str, Any]] = []
|
||||
fitted_by_name: dict[str, tuple[FeatureVariant, dict[str, Any], pd.DataFrame, dict[str, dict[str, float]]]] = {}
|
||||
comparison_baseline_score: float | None = None
|
||||
variants = experiment_variants()
|
||||
if variant_names:
|
||||
wanted = set(variant_names) | {"baseline"}
|
||||
unknown = wanted - {variant.name for variant in variants}
|
||||
if unknown:
|
||||
raise ValueError(f"unknown experiment variants: {sorted(unknown)}")
|
||||
variants = [variant for variant in variants if variant.name in wanted]
|
||||
for index, variant in enumerate(variants, start=1):
|
||||
variant_started = time.monotonic()
|
||||
_progress(f"variant {index}/{len(variants)} {variant.name}: {variant.description}")
|
||||
dataset, names, variant_profiles = build_variant_dataset(
|
||||
frames,
|
||||
variant,
|
||||
test_start_ms,
|
||||
end_ms,
|
||||
scaler,
|
||||
train_symbols,
|
||||
validation_symbols,
|
||||
test_symbols,
|
||||
validation_end_ms,
|
||||
calibration_end_ms,
|
||||
volatility_rank_threshold,
|
||||
)
|
||||
filter_stats = dataset.attrs.get("volatility_filter", {})
|
||||
_progress(
|
||||
f" features={len(names)}, rows="
|
||||
f"{int((dataset['split'] == 'train').sum())}/"
|
||||
f"{int((dataset['split'] == 'validation').sum())}/"
|
||||
f"{int((dataset['split'] == 'calibration').sum())}/"
|
||||
f"{int((dataset['split'] == 'test').sum())}"
|
||||
)
|
||||
fitted = _fit_variant(dataset, names, seed, n_estimators, n_jobs)
|
||||
if variant.name == "baseline":
|
||||
comparison_baseline_score = fitted["selection_score"]
|
||||
_progress(f" rerun baseline validation selection score={comparison_baseline_score:.8f}")
|
||||
if comparison_baseline_score is None:
|
||||
raise RuntimeError("baseline variant must be evaluated before comparisons")
|
||||
improved = (
|
||||
variant.name != "baseline"
|
||||
and fitted["selection_score"] < comparison_baseline_score
|
||||
)
|
||||
record = {
|
||||
"name": variant.name,
|
||||
"description": variant.description,
|
||||
"feature_count": len(names),
|
||||
"train_rows": int((dataset["split"] == "train").sum()),
|
||||
"validation_rows": int((dataset["split"] == "validation").sum()),
|
||||
"calibration_rows": int((dataset["split"] == "calibration").sum()),
|
||||
"test_rows": int((dataset["split"] == "test").sum()),
|
||||
"validation_selection_score": fitted["selection_score"],
|
||||
"improves_baseline_validation": improved,
|
||||
"validation": fitted["validation_metrics"],
|
||||
"volatility_filter": filter_stats,
|
||||
"elapsed_seconds": round(time.monotonic() - variant_started, 2),
|
||||
}
|
||||
results.append(record)
|
||||
fitted_by_name[variant.name] = (variant, fitted, dataset, variant_profiles)
|
||||
_progress(
|
||||
f" result score={fitted['selection_score']:.8f}; "
|
||||
f"baseline_delta={fitted['selection_score'] - comparison_baseline_score:+.8f}; "
|
||||
f"elapsed={record['elapsed_seconds']:.1f}s"
|
||||
)
|
||||
if comparison_baseline_score is None:
|
||||
raise RuntimeError("baseline result is missing")
|
||||
best_record = min(results, key=lambda item: item["validation_selection_score"])
|
||||
best_variant, best_fitted, best_dataset, best_profiles = fitted_by_name[best_record["name"]]
|
||||
_progress(f"selected by validation: {best_variant.name}")
|
||||
_, baseline_fitted, baseline_dataset, _ = fitted_by_name["baseline"]
|
||||
baseline_test = _evaluate_test(baseline_dataset, baseline_fitted)
|
||||
baseline = {
|
||||
"artifact": str(baseline_artifact.resolve()),
|
||||
"report": str(baseline_report.resolve()),
|
||||
"feature_count": 182,
|
||||
"stored_validation_selection_score": stored_baseline_score,
|
||||
"validation_selection_score": comparison_baseline_score,
|
||||
"validation": next(item["validation"] for item in results if item["name"] == "baseline"),
|
||||
"test": baseline_test,
|
||||
"stored_test": stored_baseline["test"],
|
||||
}
|
||||
best_test = _evaluate_test(best_dataset, best_fitted)
|
||||
best_dir = output_dir / "best"
|
||||
best_dir.mkdir(parents=True, exist_ok=True)
|
||||
joblib.dump(
|
||||
_artifact(
|
||||
best_fitted,
|
||||
best_variant,
|
||||
best_profiles,
|
||||
scaler,
|
||||
manifest,
|
||||
end_ms,
|
||||
test_start_ms,
|
||||
),
|
||||
best_dir / "model.joblib",
|
||||
)
|
||||
report = {
|
||||
"protocol": {
|
||||
"data_dir": str(data_dir.resolve()),
|
||||
"end_ms": end_ms,
|
||||
"test_start_ms": test_start_ms,
|
||||
"validation_end_ms": validation_end_ms,
|
||||
"calibration_end_ms": calibration_end_ms,
|
||||
"test_months": test_months,
|
||||
"seed": seed,
|
||||
"n_estimators": n_estimators,
|
||||
"n_jobs": n_jobs,
|
||||
"volatility_rank_threshold": volatility_rank_threshold,
|
||||
"train_symbols": train_symbols,
|
||||
"validation_symbols": validation_symbols,
|
||||
"test_symbols": test_symbols,
|
||||
},
|
||||
"baseline": baseline,
|
||||
"variants": results,
|
||||
"selected": {
|
||||
**best_record,
|
||||
"test": best_test,
|
||||
"artifact": str((best_dir / "model.joblib").resolve()),
|
||||
},
|
||||
"elapsed_seconds": round(time.monotonic() - started, 2),
|
||||
}
|
||||
(output_dir / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
_write_report_markdown(output_dir / "report.md", report)
|
||||
_progress(f"experiment report written to {output_dir}; total={report['elapsed_seconds']:.1f}s")
|
||||
return report
|
||||
|
||||
|
||||
def _write_report_markdown(path: Path, report: dict[str, Any]) -> None:
|
||||
baseline = report["baseline"]
|
||||
selected = report["selected"]
|
||||
lines = [
|
||||
"# Feature experiment report",
|
||||
"",
|
||||
f"- baseline: `{baseline['feature_count']}` features, validation score `{baseline['validation_selection_score']:.8f}`",
|
||||
f"- selected: `{selected['name']}`, `{selected['feature_count']}` features, validation score `{selected['validation_selection_score']:.8f}`",
|
||||
f"- selected test MAE/RMSE: `{selected['test']['mae_point']:.6f}` / `{selected['test']['rmse_point']:.6f}`",
|
||||
f"- baseline test MAE/RMSE: `{baseline['test']['mae_point']:.6f}` / `{baseline['test']['rmse_point']:.6f}`",
|
||||
"",
|
||||
"## Variant comparison",
|
||||
"",
|
||||
"| variant | features | validation score | delta vs baseline | validation MAE | q80 coverage |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for item in report["variants"]:
|
||||
metrics = item["validation"]
|
||||
delta = item["validation_selection_score"] - baseline["validation_selection_score"]
|
||||
lines.append(
|
||||
f"| `{item['name']}` | {item['feature_count']} | {item['validation_selection_score']:.8f} | {delta:+.8f} | "
|
||||
f"{metrics['mae_point']:.6f} | {metrics['q80_coverage']:.4f} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"Selection uses only the validation selection score. The final test block is reported for the baseline and the validation-selected candidate; it is not used to choose features.",
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Compare compact, indicator, global, and cross-sectional features")
|
||||
parser.add_argument("--data-dir", type=Path, default=Path("data"))
|
||||
parser.add_argument("--output-dir", type=Path, default=Path("artifacts/experiments"))
|
||||
parser.add_argument("--baseline-artifact", type=Path, default=Path("artifacts/v1/model.joblib"))
|
||||
parser.add_argument("--baseline-report", type=Path, default=Path("artifacts/v1/report.json"))
|
||||
parser.add_argument("--test-months", type=int, default=10)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--n-estimators", type=int, default=250)
|
||||
parser.add_argument("--n-jobs", type=int, default=-1)
|
||||
parser.add_argument("--volatility-rank-threshold", type=float, default=VOLATILITY_RANK_THRESHOLD)
|
||||
parser.add_argument(
|
||||
"--variants",
|
||||
default=None,
|
||||
help="comma-separated variants to run; baseline is always included",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
report = run_experiments(
|
||||
args.data_dir,
|
||||
args.output_dir,
|
||||
args.baseline_artifact,
|
||||
args.baseline_report,
|
||||
args.test_months,
|
||||
args.seed,
|
||||
args.n_estimators,
|
||||
args.n_jobs,
|
||||
args.volatility_rank_threshold,
|
||||
[item.strip() for item in args.variants.split(",") if item.strip()]
|
||||
if args.variants
|
||||
else None,
|
||||
)
|
||||
print(json.dumps({
|
||||
"selected": report["selected"]["name"],
|
||||
"baseline_score": report["baseline"]["validation_selection_score"],
|
||||
"selected_score": report["selected"]["validation_selection_score"],
|
||||
"selected_test_mae": report["selected"]["test"]["mae_point"],
|
||||
"selected_test_rmse": report["selected"]["test"]["rmse_point"],
|
||||
"report": str((args.output_dir / "report.json").resolve()),
|
||||
"artifact": report["selected"]["artifact"],
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -448,6 +448,9 @@ def compute_indicators(frame: pd.DataFrame) -> pd.DataFrame:
|
||||
df["rsi_norm"] = rsi.where(loss > 0, np.where(gain > 0, 1.0, 0.5))
|
||||
sma20 = close.rolling(20, min_periods=20).mean()
|
||||
std20 = close.rolling(20, min_periods=20).std(ddof=0)
|
||||
sma50 = close.rolling(50, min_periods=50).mean()
|
||||
df["sma20_gap"] = (close / sma20 - 1).clip(-2, 2)
|
||||
df["sma50_gap"] = (close / sma50 - 1).clip(-2, 2)
|
||||
df["bb_position"] = ((close - sma20) / (2 * std20.replace(0, np.nan))).clip(-5, 5)
|
||||
df["bb_width"] = (4 * std20 / sma20.replace(0, np.nan)).clip(0, 5)
|
||||
ema9 = close.ewm(span=9, adjust=False, min_periods=9).mean()
|
||||
@@ -1465,13 +1468,22 @@ def predict_from_csv(
|
||||
symbol: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
artifact = joblib.load(artifact_path)
|
||||
if (
|
||||
artifact.get("version") != 3
|
||||
or artifact.get("target_kind") != TARGET_KIND
|
||||
or artifact.get("target_scale") != "asset_atr_price_top20_median"
|
||||
):
|
||||
is_current_artifact = (
|
||||
artifact.get("version") == 3
|
||||
and artifact.get("target_kind") == TARGET_KIND
|
||||
and artifact.get("target_scale") == "asset_atr_price_top20_median"
|
||||
)
|
||||
variant_config = artifact.get("feature_variant") or {}
|
||||
is_compatible_variant = (
|
||||
artifact.get("version") == 4
|
||||
and artifact.get("target_kind") == TARGET_KIND
|
||||
and artifact.get("target_scale") == "asset_atr_price_top20_median"
|
||||
and not variant_config.get("global_columns")
|
||||
and not variant_config.get("cross_columns")
|
||||
)
|
||||
if not is_current_artifact and not is_compatible_variant:
|
||||
raise ValueError(
|
||||
"artifact uses a legacy target; retrain with the signed robust future-low target"
|
||||
"artifact uses an unsupported target or requires full-universe feature context"
|
||||
)
|
||||
frame = pd.read_csv(csv_path)
|
||||
for column in CSV_COLUMNS:
|
||||
@@ -1500,11 +1512,16 @@ def predict_from_csv(
|
||||
threshold = float(artifact.get("volatility_rank_threshold", VOLATILITY_RANK_THRESHOLD))
|
||||
eligible = np.isfinite(rank_value) and rank_value >= threshold
|
||||
target_scale = max(float(profile["asset_atr_price_top20_median"]), 1e-9)
|
||||
feature_stage = (
|
||||
str(variant_config.get("name"))
|
||||
if is_compatible_variant
|
||||
else artifact["stage"]
|
||||
)
|
||||
response = {
|
||||
"timestamp": int(frame.iloc[anchor_index]["timestamp"]),
|
||||
"close": float(frame.iloc[anchor_index]["close"]),
|
||||
"model": artifact["model_name"],
|
||||
"feature_stage": artifact["stage"],
|
||||
"feature_stage": feature_stage,
|
||||
"symbol": symbol,
|
||||
"volatility_rank_window": VOLATILITY_RANK_WINDOW,
|
||||
"volatility_rank_200": rank_value if np.isfinite(rank_value) else None,
|
||||
@@ -1539,13 +1556,33 @@ def predict_from_csv(
|
||||
}
|
||||
)
|
||||
return response
|
||||
features = build_prediction_features(
|
||||
frame,
|
||||
artifact["stage"],
|
||||
anchor_index,
|
||||
artifact["asset_scaler"],
|
||||
profile,
|
||||
)
|
||||
if is_compatible_variant:
|
||||
from .experiments import FeatureVariant, _variant_window_features
|
||||
|
||||
variant = FeatureVariant(
|
||||
name=str(variant_config["name"]),
|
||||
description=str(variant_config.get("description", "")),
|
||||
bar_columns=tuple(variant_config.get("bar_columns", ())),
|
||||
indicator_columns=tuple(variant_config.get("indicator_columns", ())),
|
||||
global_columns=tuple(variant_config.get("global_columns", ())),
|
||||
cross_columns=tuple(variant_config.get("cross_columns", ())),
|
||||
)
|
||||
scaled_asset = scale_asset_profile(profile, artifact["asset_scaler"])
|
||||
matrix = _variant_window_features(
|
||||
indicators,
|
||||
variant,
|
||||
np.array([anchor_index], dtype=np.int64),
|
||||
scaled_asset,
|
||||
)
|
||||
features = pd.DataFrame(matrix, columns=artifact["feature_names"])
|
||||
else:
|
||||
features = build_prediction_features(
|
||||
frame,
|
||||
artifact["stage"],
|
||||
anchor_index,
|
||||
artifact["asset_scaler"],
|
||||
profile,
|
||||
)
|
||||
point, lower, upper = predict_quantiles(
|
||||
artifact["point_model"],
|
||||
artifact["lower_model"],
|
||||
|
||||
Reference in New Issue
Block a user