From dc65b98568ded22aa6d64b38a2e11e9cd508e8ee Mon Sep 17 00:00:00 2001 From: anyongjin Date: Sun, 23 Aug 2026 13:04:48 +0800 Subject: [PATCH] update model version --- README.md | 8 +- cbpred/__main__.py | 6 +- cbpred/experiments.py | 799 +++++++++++++++++++++++++++++++ cbpred/pipeline.py | 65 ++- docs/{fea_0823.md => v0_0823.md} | 0 docs/v1_0823.md | 438 +++++++++++++++++ tests/test_artifact_paths.py | 21 + tests/test_experiments.py | 54 +++ 8 files changed, 1370 insertions(+), 21 deletions(-) create mode 100644 cbpred/experiments.py rename docs/{fea_0823.md => v0_0823.md} (100%) create mode 100644 docs/v1_0823.md create mode 100644 tests/test_artifact_paths.py create mode 100644 tests/test_experiments.py diff --git a/README.md b/README.md index 4e9bef8..af5b33a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ python3 -m venv .venv ```bash .venv/bin/python -m cbpred train \ --data-dir data \ - --artifact-dir artifacts_signed + --artifact-dir artifacts/v1 ``` 默认保留所有连续样本，把当前 ATR 在过去 200 根 8H K 线中的排名作为特征；可以通过 `--volatility-rank-threshold 0.80` 或 `0.90` 做高波动样本对照实验。训练阶段会把数据集行数、过滤前后数量、阶段和候选模型进度输出到 stderr。 @@ -47,16 +47,16 @@ python3 -m venv .venv 一条命令完成准备和训练： ```bash -.venv/bin/python -m cbpred run --out-dir data --artifact-dir artifacts_signed +.venv/bin/python -m cbpred run --out-dir data --artifact-dir artifacts/v1 ``` -输出在 `artifacts_signed/report.md`、`artifacts_signed/report.json` 和 `artifacts_signed/model.joblib`。报告会列出三轮特征的验证表现、最终模型和测试集 MAE/RMSE、q80 覆盖率、q20 下界未跌破率。 +输出在 `artifacts/v1/report.md`、`artifacts/v1/report.json` 和 `artifacts/v1/model.joblib`。当前 `v1` 是最终特征复核中保留的 `compact6` 模型。 对某个已导出的 8H CSV 预测任意锚点： ```bash .venv/bin/python -m cbpred predict \ - --artifact artifacts_signed/model.joblib \ + --artifact artifacts/v1/model.joblib \ --csv data/raw/123_BTC_USDT_USDT.csv \ --at 2026010100000000 ``` diff --git a/cbpred/__main__.py b/cbpred/__main__.py index 0799313..ad547db 100644 --- a/cbpred/__main__.py +++ b/cbpred/__main__.py @@ -35,7 +35,7 @@ def build_parser() -> argparse.ArgumentParser: train = sub.add_parser("train", help="build three feature stages, tune on validation symbols, test once") train.add_argument("--data-dir", type=Path, default=Path("data")) - train.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed")) + train.add_argument("--artifact-dir", type=Path, default=Path("artifacts/v1")) train.add_argument("--end", dest="end_ms", default=None) train.add_argument("--test-months", type=int, default=10) train.add_argument("--seed", type=int, default=42) @@ -47,7 +47,7 @@ def build_parser() -> argparse.ArgumentParser: for action in prepare._actions[1:]: if action.dest not in {"help", "command"}: run._add_action(action) - run.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed")) + run.add_argument("--artifact-dir", type=Path, default=Path("artifacts/v1")) run.add_argument("--test-months", type=int, default=10) run.add_argument("--seed", type=int, default=42) run.add_argument("--n-estimators", type=int, default=250) @@ -57,7 +57,7 @@ def build_parser() -> argparse.ArgumentParser: predict = sub.add_parser( "predict", help="predict signed robust future-low q20/q80 from an exported 8H CSV" ) - predict.add_argument("--artifact", type=Path, default=Path("artifacts_signed/model.joblib")) + predict.add_argument("--artifact", type=Path, default=Path("artifacts/v1/model.joblib")) predict.add_argument("--csv", type=Path, required=True) predict.add_argument("--at", dest="at_ms", default=None) predict.add_argument("--symbol", default=None, help="symbol used for the fixed training asset profile") diff --git a/cbpred/experiments.py b/cbpred/experiments.py new file mode 100644 index 0000000..d43d8ff --- /dev/null +++ b/cbpred/experiments.py @@ -0,0 +1,799 @@ +"""Fair, progress-reporting feature ablation experiments for the 8H model.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import joblib +import numpy as np +import pandas as pd +from sklearn.ensemble import ExtraTreesRegressor, HistGradientBoostingRegressor + +from .pipeline import ( + EIGHT_H_MS, + HORIZON, + LOWER_QUANTILE, + TARGET_QUANTILE, + VOLATILITY_RANK_THRESHOLD, + VOLATILITY_RANK_WINDOW, + WINDOW, + _ASSET_FEATURE_NAMES, + _BASE_FEATURE_COLUMNS, + _rank_windows, + apply_calibration, + asset_profile, + compute_indicators, + evaluate_predictions, + fit_asset_scaler, + floor_to_8h, + load_manifest, + load_raw_frames, + months_before, + recent_split_boundaries, + scale_asset_profile, + split_holdout_symbols, +) + + +def _progress(message: str) -> None: + print(f"[cbpred-experiment] {message}", file=sys.stderr, flush=True) + + +@dataclass(frozen=True) +class FeatureVariant: + name: str + description: str + bar_columns: tuple[str, ...] + indicator_columns: tuple[str, ...] = () + global_columns: tuple[str, ...] = () + cross_columns: tuple[str, ...] = () + + @property + def uses_global(self) -> bool: + return bool(self.global_columns) + + @property + def uses_cross(self) -> bool: + return bool(self.cross_columns) + + +_VOLUME_COLUMNS = ("quote_volume_rank", "quote_volume_log_rel_median") +_ALL_BAR_COLUMNS = _BASE_FEATURE_COLUMNS +_COMPACT7_COLUMNS = tuple(name for name in _ALL_BAR_COLUMNS if name != "body_abs_range") +_COMPACT6_COLUMNS = tuple( + name for name in _COMPACT7_COLUMNS if name != "body_sign" +) +_GLOBAL_COLUMNS = ( + "global_median_return_1", + "global_median_trend_3", + "global_median_atr_pct", + "global_return_dispersion", + "global_up_ratio", + "global_median_quote_activity", +) +_CROSS_COLUMNS = ( + "cross_return_1_rank", + "cross_trend_3_rank", + "cross_atr_pct_rank", + "cross_quote_activity_rank", +) + + +def experiment_variants() -> list[FeatureVariant]: + """Return independent candidates; baseline is deliberately first.""" + base = tuple(_ALL_BAR_COLUMNS) + return [ + FeatureVariant("baseline", "current Stage 1 feature set", base), + FeatureVariant("compact7", "remove absolute body size", _COMPACT7_COLUMNS), + FeatureVariant( + "compact6", + "remove absolute body size and explicit body sign", + _COMPACT6_COLUMNS, + ), + FeatureVariant( + "rsi_sma", + "add normalized RSI and normalized SMA20/SMA50 distance", + base, + ("rsi_norm", "sma20_gap", "sma50_gap"), + ), + FeatureVariant( + "rsi_bbands", + "add normalized RSI and Bollinger position/width", + base, + ("rsi_norm", "bb_position", "bb_width"), + ), + FeatureVariant( + "rsi_trend", + "add normalized RSI, short trends, and rolling volatility", + base, + ("rsi_norm", "trend_3", "trend_9", "volatility_18"), + ), + FeatureVariant( + "full_indicators", + "add the complete existing indicator group", + base, + ( + "atr_pct", + "atr_rank_200", + "rsi_norm", + "bb_position", + "bb_width", + "ema9_gap", + "ema21_gap", + "trend_3", + "trend_9", + "volatility_18", + "drawdown_18", + ), + ), + FeatureVariant( + "global6", + "add six cross-symbol market-state summaries", + base, + global_columns=_GLOBAL_COLUMNS, + ), + FeatureVariant( + "cross4", + "add four current-symbol cross-sectional ranks", + base, + cross_columns=_CROSS_COLUMNS, + ), + FeatureVariant( + "global_cross", + "add global market summaries and current-symbol ranks", + base, + global_columns=_GLOBAL_COLUMNS, + cross_columns=_CROSS_COLUMNS, + ), + ] + + +def feature_names(variant: FeatureVariant) -> list[str]: + names: list[str] = [] + per_bar = (*variant.bar_columns, *_VOLUME_COLUMNS, *variant.indicator_columns) + for lag in range(WINDOW - 1, -1, -1): + names.extend(f"bar_{lag:02d}_{name}" for name in per_bar) + names.extend(_ASSET_FEATURE_NAMES) + names.extend(variant.global_columns) + names.extend(variant.cross_columns) + return names + + +def _cross_sectional_context( + indicators: dict[str, pd.DataFrame], + raw_profiles: dict[str, dict[str, float]], +) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]: + records: list[pd.DataFrame] = [] + for symbol, frame in indicators.items(): + profile = raw_profiles.get(symbol) + if profile is None: + continue + record = pd.DataFrame( + { + "symbol": symbol, + "timestamp": frame["timestamp"].to_numpy(dtype=np.int64), + "return_1": frame["log_return"].to_numpy(dtype=float), + "trend_3": frame["trend_3"].to_numpy(dtype=float), + "atr_pct": frame["atr_pct"].to_numpy(dtype=float), + "body_sign": frame["body_sign"].to_numpy(dtype=float), + "quote_activity": frame["quote_log"].to_numpy(dtype=float) + - float(profile["asset_quote_volume_top20_median_log"]), + } + ) + records.append(record) + if not records: + raise ValueError("no cross-sectional records available") + all_rows = pd.concat(records, ignore_index=True) + grouped = all_rows.groupby("timestamp", sort=False) + global_context = grouped.agg( + global_median_return_1=("return_1", "median"), + global_median_trend_3=("trend_3", "median"), + global_median_atr_pct=("atr_pct", "median"), + global_return_dispersion=("return_1", lambda values: values.std(ddof=0)), + global_up_ratio=("body_sign", lambda values: float(np.mean(values > 0))), + global_median_quote_activity=("quote_activity", "median"), + ) + rank_columns = { + "return_1": "cross_return_1_rank", + "trend_3": "cross_trend_3_rank", + "atr_pct": "cross_atr_pct_rank", + "quote_activity": "cross_quote_activity_rank", + } + for source, target in rank_columns.items(): + all_rows[target] = all_rows.groupby("timestamp")[source].rank( + method="average", pct=True + ) + cross_context = { + symbol: group.set_index("timestamp")[list(rank_columns.values())].sort_index() + for symbol, group in all_rows.groupby("symbol", sort=False) + } + return global_context.sort_index(), cross_context + + +def _variant_window_features( + frame: pd.DataFrame, + variant: FeatureVariant, + anchor_indices: np.ndarray, + asset: dict[str, float], + global_context: pd.DataFrame | None = None, + cross_context: pd.DataFrame | None = None, +) -> np.ndarray: + starts = anchor_indices - (WINDOW - 1) + quote = frame["quote_log"].to_numpy(dtype=float) + quote_windows = np.lib.stride_tricks.sliding_window_view(quote, WINDOW)[starts] + volume_rank = _rank_windows(quote_windows) + volume_median = np.median(quote_windows, axis=1, keepdims=True) + columns = [*variant.bar_columns, *variant.indicator_columns] + values = frame[columns].to_numpy(dtype=float) + windows = np.lib.stride_tricks.sliding_window_view(values, WINDOW, axis=0) + windows = windows.transpose(0, 2, 1)[starts] + base_count = len(variant.bar_columns) + parts = [windows[:, :, :base_count]] + parts.extend( + [ + volume_rank[:, :, None], + (quote_windows - volume_median)[:, :, None], + ] + ) + if variant.indicator_columns: + parts.append(windows[:, :, base_count:]) + feature_matrix = np.concatenate(parts, axis=2).reshape(len(anchor_indices), -1) + asset_values = np.array([asset[name] for name in _ASSET_FEATURE_NAMES], dtype=float) + feature_matrix = np.column_stack( + [feature_matrix, np.repeat(asset_values[None, :], len(anchor_indices), axis=0)] + ) + timestamps = frame["timestamp"].to_numpy(dtype=np.int64)[anchor_indices] + if variant.global_columns: + if global_context is None: + raise ValueError("global context is required by this variant") + values = global_context.reindex(timestamps)[list(variant.global_columns)].to_numpy(dtype=float) + feature_matrix = np.column_stack([feature_matrix, values]) + if variant.cross_columns: + if cross_context is None: + raise ValueError("cross-sectional context is required by this variant") + values = cross_context.reindex(timestamps)[list(variant.cross_columns)].to_numpy(dtype=float) + feature_matrix = np.column_stack([feature_matrix, values]) + return feature_matrix + + +def build_variant_dataset( + frames: dict[str, pd.DataFrame], + variant: FeatureVariant, + test_start_ms: int, + end_ms: int, + asset_scaler: dict[str, dict[str, float]], + train_symbols: list[str], + validation_symbols: list[str], + test_symbols: list[str], + validation_end_ms: int, + calibration_end_ms: int, + volatility_rank_threshold: float, +) -> tuple[pd.DataFrame, list[str], dict[str, dict[str, float]]]: + indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()} + raw_profiles = { + symbol: asset_profile(frame, test_start_ms) + for symbol, frame in indicators.items() + } + profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None} + scaled_profiles = { + symbol: scale_asset_profile(profile, asset_scaler) + for symbol, profile in profiles.items() + } + global_context = None + cross_context: dict[str, pd.DataFrame] = {} + if variant.uses_global or variant.uses_cross: + global_context, cross_context = _cross_sectional_context(indicators, profiles) + train_set = set(train_symbols) + validation_set = set(validation_symbols) + test_set = set(test_symbols) + step = EIGHT_H_MS + rows: list[pd.DataFrame] = [] + candidate_counts = {"train": 0, "validation": 0, "calibration": 0, "test": 0} + selected_counts = {name: 0 for name in candidate_counts} + names = feature_names(variant) + for symbol, frame in indicators.items(): + asset = scaled_profiles.get(symbol) + if asset is None or len(frame) < WINDOW + HORIZON: + continue + frame = frame.reset_index(drop=True) + timestamps = frame["timestamp"].to_numpy(dtype=np.int64) + anchor_indices = np.arange(WINDOW - 1, len(frame) - HORIZON, dtype=np.int64) + continuity = np.convolve( + (np.diff(timestamps) == step).astype(np.int8), + np.ones(WINDOW + HORIZON - 1, dtype=np.int8), + mode="valid", + ) == WINDOW + HORIZON - 1 + anchor_ms = timestamps[anchor_indices] + future_end_ms = timestamps[anchor_indices + HORIZON] + step + split_labels = np.full(len(anchor_indices), "", dtype=object) + if symbol in train_set: + mask = (anchor_ms < test_start_ms) & (future_end_ms <= test_start_ms) + split_labels[mask] = "train" + if symbol in validation_set: + mask = ( + (test_start_ms <= anchor_ms) + & (anchor_ms < validation_end_ms) + & (future_end_ms <= validation_end_ms) + ) + split_labels[mask] = "validation" + mask = ( + (validation_end_ms <= anchor_ms) + & (anchor_ms < calibration_end_ms) + & (future_end_ms <= calibration_end_ms) + ) + split_labels[mask] = "calibration" + if symbol in test_set: + mask = ( + (calibration_end_ms <= anchor_ms) + & (anchor_ms < end_ms) + & (future_end_ms <= end_ms) + ) + split_labels[mask] = "test" + candidate = continuity & (split_labels != "") + for split_name in candidate_counts: + candidate_counts[split_name] += int( + np.count_nonzero(candidate & (split_labels == split_name)) + ) + volatility_rank = frame["atr_rank_200"].to_numpy(dtype=float)[anchor_indices] + if volatility_rank_threshold == 0.0: + selected = candidate + else: + selected = candidate & np.isfinite(volatility_rank) & (volatility_rank >= volatility_rank_threshold) + for split_name in selected_counts: + selected_counts[split_name] += int( + np.count_nonzero(selected & (split_labels == split_name)) + ) + if not np.any(selected): + continue + selected_indices = anchor_indices[selected] + feature_matrix = _variant_window_features( + frame, + variant, + selected_indices, + asset, + global_context, + cross_context.get(symbol), + ) + low = frame["low"].to_numpy(dtype=float) + close = frame["close"].to_numpy(dtype=float)[selected_indices] + future_lows = np.column_stack( + [low[selected_indices + offset] for offset in range(1, HORIZON + 1)] + ) + future_reference = np.quantile(future_lows, 0.20, axis=1) + future_return = future_reference / np.maximum(close, 1e-12) - 1.0 + target_scale = max(float(profiles[symbol]["asset_atr_price_top20_median"]), 1e-9) + target = future_return / target_scale + sample_atr = frame["atr_pct"].to_numpy(dtype=float)[selected_indices] + finite = ( + np.isfinite(target) + & np.isfinite(sample_atr) + & np.isfinite(feature_matrix).all(axis=1) + ) + if not np.any(finite): + continue + block = pd.DataFrame(feature_matrix[finite], columns=names) + block["symbol"] = symbol + block["timestamp"] = anchor_ms[selected][finite] + block["target"] = target[finite] + block["target_scale"] = target_scale + block["split"] = split_labels[selected][finite] + block["sample_atr_pct"] = sample_atr[finite] + rows.append(block) + dataset = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame() + if dataset.empty: + raise ValueError(f"variant {variant.name} produced no rows") + dataset.attrs["volatility_filter"] = { + "window": VOLATILITY_RANK_WINDOW, + "threshold": volatility_rank_threshold, + "candidate_counts": candidate_counts, + "selected_counts": selected_counts, + } + return dataset, names, profiles + + +def _quantile_model(alpha: float, seed: int, n_estimators: int) -> HistGradientBoostingRegressor: + return HistGradientBoostingRegressor( + quantile=alpha, + max_iter=max(1, n_estimators), + learning_rate=0.04, + max_depth=2, + min_samples_leaf=8, + random_state=seed, + verbose=0, + ) + + +def _fit_variant( + dataset: pd.DataFrame, + names: list[str], + seed: int, + n_estimators: int, + n_jobs: int, +) -> dict[str, Any]: + train = dataset[dataset["split"] == "train"] + validation = dataset[dataset["split"] == "validation"] + if train.empty or validation.empty: + raise ValueError("variant train or validation rows are empty") + train_x = train[names] + train_y = train["target"].to_numpy(dtype=float) + counts = train["symbol"].value_counts() + weights = 1.0 / train["symbol"].map(counts).to_numpy(dtype=float) + weights /= weights.mean() + lower_model = _quantile_model(LOWER_QUANTILE, seed, n_estimators) + upper_model = _quantile_model(TARGET_QUANTILE, seed, n_estimators) + _progress(" fitting q20 quantile model") + lower_model.fit(train_x, train_y) + _progress(" fitting q80 quantile model") + upper_model.fit(train_x, train_y) + point_model = ExtraTreesRegressor( + criterion="squared_error", + n_estimators=n_estimators, + max_features=0.7, + min_samples_leaf=8, + random_state=seed, + n_jobs=n_jobs, + verbose=0, + ) + _progress(" fitting ExtraTrees point model") + point_model.fit(train_x, train_y, sample_weight=weights) + validation_x = validation[names] + validation_y = validation["target"].to_numpy(dtype=float) + point = point_model.predict(validation_x) + lower = lower_model.predict(validation_x) + upper = upper_model.predict(validation_x) + scale = validation["target_scale"].to_numpy(dtype=float) + metrics = evaluate_predictions(validation_y, point, lower, upper, scale) + score = ( + metrics["pinball_q20"] + + metrics["pinball_q80"] + + 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"]) + + 0.05 * metrics["mae_point"] + ) + calibration = dataset[dataset["split"] == "calibration"] + if calibration.empty: + raise ValueError("variant calibration rows are empty") + calibration_x = calibration[names] + calibration_y = calibration["target"].to_numpy(dtype=float) + cal_lower = lower_model.predict(calibration_x) + cal_upper = upper_model.predict(calibration_x) + offsets = { + "q20_offset": float(np.quantile(calibration_y - cal_lower, LOWER_QUANTILE)), + "q80_offset": float(np.quantile(calibration_y - cal_upper, TARGET_QUANTILE)), + } + return { + "point_model": point_model, + "lower_model": lower_model, + "upper_model": upper_model, + "feature_names": names, + "validation_metrics": metrics, + "selection_score": float(score), + "calibration": offsets, + } + + +def _evaluate_test(dataset: pd.DataFrame, fitted: dict[str, Any]) -> dict[str, Any]: + test = dataset[dataset["split"] == "test"] + features = test[fitted["feature_names"]] + point = fitted["point_model"].predict(features) + lower = fitted["lower_model"].predict(features) + upper = fitted["upper_model"].predict(features) + lower, upper = apply_calibration(lower, upper, fitted["calibration"]) + y = test["target"].to_numpy(dtype=float) + scale = test["target_scale"].to_numpy(dtype=float) + metrics = evaluate_predictions(y, point, lower, upper, scale) + metrics.update( + { + "calibrated_q80_coverage": float(np.mean(y <= upper)), + "calibrated_q20_hit_rate": float(np.mean(y >= lower)), + "test_rows": int(len(test)), + } + ) + return metrics + + +def _baseline_score(report: dict[str, Any]) -> float: + metrics = report["selected"]["validation"] + return float( + metrics["pinball_q20"] + + metrics["pinball_q80"] + + 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"]) + + 0.05 * metrics["mae_point"] + ) + + +def _artifact( + fitted: dict[str, Any], + variant: FeatureVariant, + profiles: dict[str, dict[str, float]], + scaler: dict[str, dict[str, float]], + manifest: dict[str, Any], + end_ms: int, + test_start_ms: int, +) -> dict[str, Any]: + return { + "version": 4, + "timeframe": "8h", + "window": WINDOW, + "horizon": HORIZON, + "target_kind": "signed_future_low_q20_return_atr_normalized", + "target": "quantile(future_6_low, 0.20) / anchor_close - 1", + "future_low_quantile": 0.20, + "target_scale": "asset_atr_price_top20_median", + "lower_quantile": LOWER_QUANTILE, + "upper_quantile": TARGET_QUANTILE, + "volatility_rank_window": VOLATILITY_RANK_WINDOW, + "volatility_rank_threshold": VOLATILITY_RANK_THRESHOLD, + "feature_variant": { + "name": variant.name, + "description": variant.description, + "bar_columns": list(variant.bar_columns), + "indicator_columns": list(variant.indicator_columns), + "global_columns": list(variant.global_columns), + "cross_columns": list(variant.cross_columns), + }, + "stage": variant.name, + "model_name": "extra_trees_squared_error", + "model_kind": "point_plus_gbdt_quantiles", + "feature_names": fitted["feature_names"], + "asset_scaler": scaler, + "asset_profiles": profiles, + "symbol_files": { + item["symbol"]: item["path"] + for item in manifest.get("symbols", []) + if item["symbol"] in profiles + }, + "calibration": fitted["calibration"], + "point_model": fitted["point_model"], + "lower_model": fitted["lower_model"], + "upper_model": fitted["upper_model"], + "end_ms": end_ms, + "test_start_ms": test_start_ms, + "symbols": sorted(profiles), + } + + +def run_experiments( + data_dir: Path, + output_dir: Path, + baseline_artifact: Path = Path("artifacts/v1/model.joblib"), + baseline_report: Path = Path("artifacts/v1/report.json"), + test_months: int = 10, + seed: int = 42, + n_estimators: int = 250, + n_jobs: int = -1, + volatility_rank_threshold: float = VOLATILITY_RANK_THRESHOLD, + variant_names: list[str] | None = None, +) -> dict[str, Any]: + started = time.monotonic() + output_dir.mkdir(parents=True, exist_ok=True) + _progress("loading manifest and raw 8H frames") + manifest = load_manifest(data_dir) + frames = load_raw_frames(data_dir, manifest) + end_ms = floor_to_8h(int(manifest["end_ms"])) + test_start_ms = months_before(end_ms, test_months) + indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()} + raw_profiles = { + symbol: asset_profile(frame, test_start_ms) + for symbol, frame in indicators.items() + } + profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None} + train_symbols, validation_symbols, test_symbols = split_holdout_symbols(profiles, seed) + scaler = fit_asset_scaler(profiles, train_symbols) + validation_end_ms, calibration_end_ms = recent_split_boundaries(test_start_ms, end_ms) + _progress( + f"dataset={len(frames)} symbols; train/validation/test symbols=" + f"{len(train_symbols)}/{len(validation_symbols)}/{len(test_symbols)}" + ) + stored_baseline = json.loads(baseline_report.read_text()) + stored_baseline_score = _baseline_score(stored_baseline) + _progress(f"stored baseline validation selection score={stored_baseline_score:.8f}") + results: list[dict[str, Any]] = [] + fitted_by_name: dict[str, tuple[FeatureVariant, dict[str, Any], pd.DataFrame, dict[str, dict[str, float]]]] = {} + comparison_baseline_score: float | None = None + variants = experiment_variants() + if variant_names: + wanted = set(variant_names) | {"baseline"} + unknown = wanted - {variant.name for variant in variants} + if unknown: + raise ValueError(f"unknown experiment variants: {sorted(unknown)}") + variants = [variant for variant in variants if variant.name in wanted] + for index, variant in enumerate(variants, start=1): + variant_started = time.monotonic() + _progress(f"variant {index}/{len(variants)} {variant.name}: {variant.description}") + dataset, names, variant_profiles = build_variant_dataset( + frames, + variant, + test_start_ms, + end_ms, + scaler, + train_symbols, + validation_symbols, + test_symbols, + validation_end_ms, + calibration_end_ms, + volatility_rank_threshold, + ) + filter_stats = dataset.attrs.get("volatility_filter", {}) + _progress( + f" features={len(names)}, rows=" + f"{int((dataset['split'] == 'train').sum())}/" + f"{int((dataset['split'] == 'validation').sum())}/" + f"{int((dataset['split'] == 'calibration').sum())}/" + f"{int((dataset['split'] == 'test').sum())}" + ) + fitted = _fit_variant(dataset, names, seed, n_estimators, n_jobs) + if variant.name == "baseline": + comparison_baseline_score = fitted["selection_score"] + _progress(f" rerun baseline validation selection score={comparison_baseline_score:.8f}") + if comparison_baseline_score is None: + raise RuntimeError("baseline variant must be evaluated before comparisons") + improved = ( + variant.name != "baseline" + and fitted["selection_score"] < comparison_baseline_score + ) + record = { + "name": variant.name, + "description": variant.description, + "feature_count": len(names), + "train_rows": int((dataset["split"] == "train").sum()), + "validation_rows": int((dataset["split"] == "validation").sum()), + "calibration_rows": int((dataset["split"] == "calibration").sum()), + "test_rows": int((dataset["split"] == "test").sum()), + "validation_selection_score": fitted["selection_score"], + "improves_baseline_validation": improved, + "validation": fitted["validation_metrics"], + "volatility_filter": filter_stats, + "elapsed_seconds": round(time.monotonic() - variant_started, 2), + } + results.append(record) + fitted_by_name[variant.name] = (variant, fitted, dataset, variant_profiles) + _progress( + f" result score={fitted['selection_score']:.8f}; " + f"baseline_delta={fitted['selection_score'] - comparison_baseline_score:+.8f}; " + f"elapsed={record['elapsed_seconds']:.1f}s" + ) + if comparison_baseline_score is None: + raise RuntimeError("baseline result is missing") + best_record = min(results, key=lambda item: item["validation_selection_score"]) + best_variant, best_fitted, best_dataset, best_profiles = fitted_by_name[best_record["name"]] + _progress(f"selected by validation: {best_variant.name}") + _, baseline_fitted, baseline_dataset, _ = fitted_by_name["baseline"] + baseline_test = _evaluate_test(baseline_dataset, baseline_fitted) + baseline = { + "artifact": str(baseline_artifact.resolve()), + "report": str(baseline_report.resolve()), + "feature_count": 182, + "stored_validation_selection_score": stored_baseline_score, + "validation_selection_score": comparison_baseline_score, + "validation": next(item["validation"] for item in results if item["name"] == "baseline"), + "test": baseline_test, + "stored_test": stored_baseline["test"], + } + best_test = _evaluate_test(best_dataset, best_fitted) + best_dir = output_dir / "best" + best_dir.mkdir(parents=True, exist_ok=True) + joblib.dump( + _artifact( + best_fitted, + best_variant, + best_profiles, + scaler, + manifest, + end_ms, + test_start_ms, + ), + best_dir / "model.joblib", + ) + report = { + "protocol": { + "data_dir": str(data_dir.resolve()), + "end_ms": end_ms, + "test_start_ms": test_start_ms, + "validation_end_ms": validation_end_ms, + "calibration_end_ms": calibration_end_ms, + "test_months": test_months, + "seed": seed, + "n_estimators": n_estimators, + "n_jobs": n_jobs, + "volatility_rank_threshold": volatility_rank_threshold, + "train_symbols": train_symbols, + "validation_symbols": validation_symbols, + "test_symbols": test_symbols, + }, + "baseline": baseline, + "variants": results, + "selected": { + **best_record, + "test": best_test, + "artifact": str((best_dir / "model.joblib").resolve()), + }, + "elapsed_seconds": round(time.monotonic() - started, 2), + } + (output_dir / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False)) + _write_report_markdown(output_dir / "report.md", report) + _progress(f"experiment report written to {output_dir}; total={report['elapsed_seconds']:.1f}s") + return report + + +def _write_report_markdown(path: Path, report: dict[str, Any]) -> None: + baseline = report["baseline"] + selected = report["selected"] + lines = [ + "# Feature experiment report", + "", + f"- baseline: `{baseline['feature_count']}` features, validation score `{baseline['validation_selection_score']:.8f}`", + f"- selected: `{selected['name']}`, `{selected['feature_count']}` features, validation score `{selected['validation_selection_score']:.8f}`", + f"- selected test MAE/RMSE: `{selected['test']['mae_point']:.6f}` / `{selected['test']['rmse_point']:.6f}`", + f"- baseline test MAE/RMSE: `{baseline['test']['mae_point']:.6f}` / `{baseline['test']['rmse_point']:.6f}`", + "", + "## Variant comparison", + "", + "| variant | features | validation score | delta vs baseline | validation MAE | q80 coverage |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for item in report["variants"]: + metrics = item["validation"] + delta = item["validation_selection_score"] - baseline["validation_selection_score"] + lines.append( + f"| `{item['name']}` | {item['feature_count']} | {item['validation_selection_score']:.8f} | {delta:+.8f} | " + f"{metrics['mae_point']:.6f} | {metrics['q80_coverage']:.4f} |" + ) + lines.extend( + [ + "", + "Selection uses only the validation selection score. The final test block is reported for the baseline and the validation-selected candidate; it is not used to choose features.", + ] + ) + path.write_text("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare compact, indicator, global, and cross-sectional features") + parser.add_argument("--data-dir", type=Path, default=Path("data")) + parser.add_argument("--output-dir", type=Path, default=Path("artifacts/experiments")) + parser.add_argument("--baseline-artifact", type=Path, default=Path("artifacts/v1/model.joblib")) + parser.add_argument("--baseline-report", type=Path, default=Path("artifacts/v1/report.json")) + parser.add_argument("--test-months", type=int, default=10) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--n-estimators", type=int, default=250) + parser.add_argument("--n-jobs", type=int, default=-1) + parser.add_argument("--volatility-rank-threshold", type=float, default=VOLATILITY_RANK_THRESHOLD) + parser.add_argument( + "--variants", + default=None, + help="comma-separated variants to run; baseline is always included", + ) + args = parser.parse_args() + report = run_experiments( + args.data_dir, + args.output_dir, + args.baseline_artifact, + args.baseline_report, + args.test_months, + args.seed, + args.n_estimators, + args.n_jobs, + args.volatility_rank_threshold, + [item.strip() for item in args.variants.split(",") if item.strip()] + if args.variants + else None, + ) + print(json.dumps({ + "selected": report["selected"]["name"], + "baseline_score": report["baseline"]["validation_selection_score"], + "selected_score": report["selected"]["validation_selection_score"], + "selected_test_mae": report["selected"]["test"]["mae_point"], + "selected_test_rmse": report["selected"]["test"]["rmse_point"], + "report": str((args.output_dir / "report.json").resolve()), + "artifact": report["selected"]["artifact"], + }, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/cbpred/pipeline.py b/cbpred/pipeline.py index 38af897..d5a7d0c 100644 --- a/cbpred/pipeline.py +++ b/cbpred/pipeline.py @@ -448,6 +448,9 @@ def compute_indicators(frame: pd.DataFrame) -> pd.DataFrame: df["rsi_norm"] = rsi.where(loss > 0, np.where(gain > 0, 1.0, 0.5)) sma20 = close.rolling(20, min_periods=20).mean() std20 = close.rolling(20, min_periods=20).std(ddof=0) + sma50 = close.rolling(50, min_periods=50).mean() + df["sma20_gap"] = (close / sma20 - 1).clip(-2, 2) + df["sma50_gap"] = (close / sma50 - 1).clip(-2, 2) df["bb_position"] = ((close - sma20) / (2 * std20.replace(0, np.nan))).clip(-5, 5) df["bb_width"] = (4 * std20 / sma20.replace(0, np.nan)).clip(0, 5) ema9 = close.ewm(span=9, adjust=False, min_periods=9).mean() @@ -1465,13 +1468,22 @@ def predict_from_csv( symbol: str | None = None, ) -> dict[str, Any]: artifact = joblib.load(artifact_path) - if ( - artifact.get("version") != 3 - or artifact.get("target_kind") != TARGET_KIND - or artifact.get("target_scale") != "asset_atr_price_top20_median" - ): + is_current_artifact = ( + artifact.get("version") == 3 + and artifact.get("target_kind") == TARGET_KIND + and artifact.get("target_scale") == "asset_atr_price_top20_median" + ) + variant_config = artifact.get("feature_variant") or {} + is_compatible_variant = ( + artifact.get("version") == 4 + and artifact.get("target_kind") == TARGET_KIND + and artifact.get("target_scale") == "asset_atr_price_top20_median" + and not variant_config.get("global_columns") + and not variant_config.get("cross_columns") + ) + if not is_current_artifact and not is_compatible_variant: raise ValueError( - "artifact uses a legacy target; retrain with the signed robust future-low target" + "artifact uses an unsupported target or requires full-universe feature context" ) frame = pd.read_csv(csv_path) for column in CSV_COLUMNS: @@ -1500,11 +1512,16 @@ def predict_from_csv( threshold = float(artifact.get("volatility_rank_threshold", VOLATILITY_RANK_THRESHOLD)) eligible = np.isfinite(rank_value) and rank_value >= threshold target_scale = max(float(profile["asset_atr_price_top20_median"]), 1e-9) + feature_stage = ( + str(variant_config.get("name")) + if is_compatible_variant + else artifact["stage"] + ) response = { "timestamp": int(frame.iloc[anchor_index]["timestamp"]), "close": float(frame.iloc[anchor_index]["close"]), "model": artifact["model_name"], - "feature_stage": artifact["stage"], + "feature_stage": feature_stage, "symbol": symbol, "volatility_rank_window": VOLATILITY_RANK_WINDOW, "volatility_rank_200": rank_value if np.isfinite(rank_value) else None, @@ -1539,13 +1556,33 @@ def predict_from_csv( } ) return response - features = build_prediction_features( - frame, - artifact["stage"], - anchor_index, - artifact["asset_scaler"], - profile, - ) + if is_compatible_variant: + from .experiments import FeatureVariant, _variant_window_features + + variant = FeatureVariant( + name=str(variant_config["name"]), + description=str(variant_config.get("description", "")), + bar_columns=tuple(variant_config.get("bar_columns", ())), + indicator_columns=tuple(variant_config.get("indicator_columns", ())), + global_columns=tuple(variant_config.get("global_columns", ())), + cross_columns=tuple(variant_config.get("cross_columns", ())), + ) + scaled_asset = scale_asset_profile(profile, artifact["asset_scaler"]) + matrix = _variant_window_features( + indicators, + variant, + np.array([anchor_index], dtype=np.int64), + scaled_asset, + ) + features = pd.DataFrame(matrix, columns=artifact["feature_names"]) + else: + features = build_prediction_features( + frame, + artifact["stage"], + anchor_index, + artifact["asset_scaler"], + profile, + ) point, lower, upper = predict_quantiles( artifact["point_model"], artifact["lower_model"], diff --git a/docs/fea_0823.md b/docs/v0_0823.md similarity index 100% rename from docs/fea_0823.md rename to docs/v0_0823.md diff --git a/docs/v1_0823.md b/docs/v1_0823.md new file mode 100644 index 0000000..65772b3 --- /dev/null +++ b/docs/v1_0823.md @@ -0,0 +1,438 @@ +我考虑下进行下面的尝试： + +1. 每个K线提取8个信息，其中冗余的不少，尝试压缩下；只要下降不是很多就可以保留。 +2. RSI和SMA等指标的特征考虑下进行加入对比 +3. 加入K线对应的全局概要特征，代表市场整体行情特征，这里要注意提取方式，尽量不要特别多，应该是几个数字指标代表 +4. 加入当前品种当前K线在全局的横向特征，比如涨跌幅排名，成交量排名，或者价格等其他信息，注意进行归一化确保不同品种可比 + +以当前的最新模型作为基线对比模型。 +继续使用当前的模型ExtraTrees，然后逐个尝试上面的改动，进行模型训练，然后和基线对比，如果有变好的保留，没有变好的忽略。 + +# 分析优化尝试结果 + +已完成特征实验，并保留表现最好的 compact6 模型。 + +正式 20 棵树对比结果： + + 模型 特征数 测试 MAE 测试 RMSE +━━━━━━━━━━ ━━━━━━━━ ━━━━━━━━━━ ━━━━━━━━━━━ + 基线 182 0.025140 0.033789 +────────── ──────── ────────── ─────────── + compact6 146 0.024918 0.033563 + +compact6 相比基线： + +- MAE 下降约 0.88% +- RMSE 下降约 0.67% +- q80 覆盖率：77.50% +- q20 下界命中率：91.60% + +最终保留的特征包括： + +- 每根K线：有符号实体比例、上下影线比例、收盘位置、振幅比例、对数收益 +- 最近18根K线的法币成交量排名及相对成交量 +- 品种独有特征：训练期 ATR/价格前20%中位数、成交量前20%中位数 + +未保留： + +- RSI/SMA、布林带、趋势指标组合：没有稳定改善 +- 全局市场概要特征：表现变差 +- 横向排名特征：筛选阶段略有改善，但正式复核弱于 compact6 + +模型文件： + +artifacts/v1/model.joblib + +artifacts/v1/report.md + +# 未保留部分详细分析 + +先说明结论：这些实验验证的是“当前这套构造方式”，不能证明 RSI、全局行情或横向排名本身无效。尤其 RSI/ +SMA 只做了筛选实验，没有在最终 20 棵树配置下全部复核，因此结论需要保留一定谨慎。 + +## 一、指标类特征 + +指标全部在 cbpred/pipeline.py:419 的 compute_indicators() 中计算。 + +基线每个样本包含： + +- 最近18根K线 +- 每根K线8个基础特征 +- 每根K线2个成交量特征 +- 2个品种独有特征 + +所以基线是： + +18 × (8 + 2) + 2 = 182个特征 + +指标实验不是只加入当前K线的指标，而是把指标复制到最近18根K线中的每一根。 + +### 1. RSI + SMA + +对应 rsi_sma，每根K线加入3个特征： + +rsi_norm +sma20_gap +sma50_gap + +总特征数： + +182 + 18 × 3 = 236 + +具体定义： + +true_range = max( + high - low, + abs(high - previous_close), + abs(low - previous_close) +) + +ATR14 = EWMA(true_range, alpha=1/14) + +RSI使用上涨和下跌幅度的EWMA： + +gain = EWMA(max(close_t - close_(t-1), 0)) +loss = EWMA(max(close_(t-1) - close_t, 0)) + +RSI = 100 - 100 / (1 + gain / loss) +rsi_norm = RSI / 100 + +代码中对极端情况做了处理： + +- 没有下跌时，rsi_norm = 1 +- 没有上涨也没有下跌时，rsi_norm = 0.5 + +SMA特征不是直接放均线价格，而是价格相对均线的距离： + +sma20_gap = close / SMA20 - 1 +sma50_gap = close / SMA50 - 1 + +然后裁剪到 [-2, 2]。 + +因此它们是无量纲特征，不会直接受到BTC价格是几万美元、某些山寨币价格只有几美分的影响。 + +### 2. RSI + 布林带 + +对应 rsi_bbands，每根K线加入： + +rsi_norm +bb_position +bb_width + +总特征数也是236。 + +布林带使用20根K线： + +SMA20 = 20根收盘价平均值 +std20 = 20根收盘价标准差 + +布林位置： + +bb_position = (close - SMA20) / (2 × std20) + +裁剪到 [-5, 5]。 + +布林带宽度： + +bb_width = 4 × std20 / SMA20 + +裁剪到 [0, 5]。 + +所以： + +- bb_position 表示当前价格在布林带中的相对位置 +- bb_width 表示当前波动带宽度 +- 二者都不是绝对价格，具备跨品种可比性 + +### 3. RSI + 趋势和波动 + +对应 rsi_trend，每根K线加入： + +rsi_norm +trend_3 +trend_9 +volatility_18 + +总特征数： + +182 + 18 × 4 = 254 + +定义为： + +trend_3 = close / close_(t-3) - 1 +trend_9 = close / close_(t-9) - 1 +volatility_18 = 18根log_return的标准差 + +trend_3 和 trend_9 都裁剪到 [-2, 2]，volatility_18 裁剪到 [0, 2]。 + +### 4. 完整指标组 + +对应 full_indicators，每根K线加入11个指标： + +atr_pct +atr_rank_200 +rsi_norm +bb_position +bb_width +ema9_gap +ema21_gap +trend_3 +trend_9 +volatility_18 +drawdown_18 + +总特征数： + +182 + 18 × 11 = 380 + +其中： + +atr_pct = ATR14 / close +atr_rank_200 = 当前ATR在过去200根ATR中的百分位排名 + +ema9_gap = close / EMA9 - 1 +ema21_gap = close / EMA21 - 1 + +drawdown_18 = close / 最近18根最高收盘价 - 1 + +atr_rank_200 是 [0, 1] 的百分位排名，其他价格指标基本都是比例或收益率。 + +### 指标构造中可能存在的问题 + +这里最值得注意的是：虽然这些指标是无量纲的，但不一定进行了“波动率归一化”。 + +例如： + +sma20_gap = 0.02 + +对高波动品种可能只是普通波动，但对低波动品种可能已经是非常大的偏离。当前没有进一步计算： + +sma20_gap / 当前品种局部ATR +trend_9 / 当前品种局部ATR + +另外： + +- RSI、SMA偏离、EMA偏离、趋势收益本质上有较多信息重叠 +- 所有指标被复制到18根K线，特征数量快速增加 +- atr_pct 是品种自身的比例，但没有除以该品种的长期ATR基准 +- bb_position 在极低波动时容易变得不稳定，只是通过裁剪限制了极端值 +- 没有加入指标的变化速度，例如 RSI斜率、布林带宽度变化、ATR变化方向 + +筛选报告中结果如下，分数越低越好： + + 方案 相对基线变化 +━━━━━━━━━━━━━━━━ ━━━━━━━━━━━━━━ + RSI + SMA -0.00000288 +──────────────── ────────────── + RSI + 布林带 +0.00011975 +──────────────── ────────────── + RSI + 趋势波动 +0.00025359 +──────────────── ────────────── + 完整指标组 +0.00032006 + +因此，只有 RSI+SMA 是非常微小的改善，其他指标组合都变差。这里的筛选实验使用64棵树，不能把这个结果解释 +成 RSI/SMA 已经被充分验证。 + +## 二、全局市场概要特征 + +全局特征在 cbpred/experiments.py:168 到 cbpred/experiments.py:216 中计算。 + +首先，对每个品种、每个时间戳生成： + +return_1 = 当前log收益 +trend_3 = 3根K线收益 +atr_pct = ATR14 / close +body_sign = sign(close - open) +quote_activity = + log1p(成交额) + - 该品种历史成交额前20%中位数 + +然后按照相同时间戳，把所有品种横向聚合成6个全局数字： + +global_median_return_1 +global_median_trend_3 +global_median_atr_pct +global_return_dispersion +global_up_ratio +global_median_quote_activity + +具体含义： + +### 1. 全局中位数收益 + +global_median_return_1 = +所有品种当前1根K线log收益的中位数 + +代表市场当前整体涨跌方向。 + +### 2. 全局中位数短趋势 + +global_median_trend_3 = +所有品种当前3根K线收益的中位数 + +代表市场短线趋势。 + +### 3. 全局中位数ATR比例 + +global_median_atr_pct = +所有品种 ATR14 / close 的中位数 + +代表市场整体波动水平。 + +### 4. 全局收益离散度 + +global_return_dispersion = +所有品种当前1根K线收益的总体标准差 + +代表市场分化程度。数值越大，说明不同品种当前表现越不一致。 + +### 5. 全局上涨比例 + +global_up_ratio = +当前阳线品种数 / 有效品种总数 + +这里依据的是 close > open，不是收益率大于0。 + +### 6. 全局成交活跃度 + +单个品种的成交活跃度先计算为： + +log1p(当前成交额) +- 该品种训练期成交额前20%中位数的log值 + +然后取所有品种的中位数。 + +这个特征代表当前市场成交量相对于各品种自身高成交量基准的整体水平。 + +### 重要问题 + +全局6个数字只加入样本当前锚点，不会重复加入最近18根K线。 + +也就是说，样本结构是： + +最近18根K线特征 ++ 2个品种特征 ++ 当前时刻6个全局特征 + +而不是： + +最近18个时刻 × 6个全局特征 + +因此它只能告诉模型“现在市场是什么状态”，不能告诉模型“过去18根K线市场状态如何变化”。 + +此外： + +- 使用的是所有可用品种的简单中位数 +- 没有按成交量或市值加权 +- 没有单独加入BTC、ETH等主要品种的市场基准 +- 没有加入全市场上涨比例的变化趋势 +- 没有加入全局波动率变化趋势 +- 全局特征本身没有再做统一的z-score，只依赖其比例、标准差或相对成交量形式 + +全局特征实验结果： + +global6 +0.00052393 +global_cross +0.00033642 + +这说明当前这6个市场概要对目标没有提供有效增益，但不能说明全局特征思路本身错误。当前最大的问题可能 +是“只取当前时刻一个横截面”，时间维度太弱。 + +## 三、当前品种横向排名特征 + +横向排名同样在 cbpred/experiments.py:168 中计算。 + +对每个时间戳，把所有品种进行横向排名，加入4个特征： + +cross_return_1_rank +cross_trend_3_rank +cross_atr_pct_rank +cross_quote_activity_rank + +排名使用： + +rank(method="average", pct=True) + +因此结果大致位于 (0, 1]： + +- 越接近1：当前在全市场中越高 +- 越接近0：当前在全市场中越低 +- 相同值使用平均排名 + +具体为： + +### 1. 当前收益排名 + +### 3. 当前波动率排名 + +cross_atr_pct_rank + +当前品种ATR/价格在所有品种中的百分位排名。 + +### 4. 当前成交活跃度排名 + +cross_quote_activity_rank + +当前品种相对自身历史高成交量基准的成交活跃度，在所有品种中的百分位排名。 + +这个方案只增加4个特征： + +182 + 4 = 186 + +同样，它们只加入当前锚点，不会给最近18根K线都增加横向排名。 + +### 横向特征存在的限制 + +1. 没有加入价格相关的横向特征 + +当前没有： + +相对自身长期价格位置排名 +相对历史高点距离排名 +相对自身均线偏离排名 + +只加入了收益、趋势、波动率和成交活跃度排名。 + +2. 横截面成员数量可能变化 + +某个时间戳上如果部分品种缺K线，排名实际是在较小的品种集合中计算的。这样不同时间的百分位含义可能不完全 +一致。 + +3. 可能受低流动性品种噪声影响 + +所有品种简单参与排名，没有按照流动性加权。大量低流动性品种可能影响整体排名分布。 + +4. 依赖全品种数据 + +预测单个CSV时，无法还原完整横向排名。当前预测接口明确拒绝带有全局或横向依赖的模型，见 cbpred/ +pipeline.py:1476。 + +这也是最终没有保留 cross4 的工程原因之一：即使它在部分筛选实验中略有改善，也无法直接用单个品种数据稳 +定预测。 + +横向排名筛选阶段比基线略好，但正式20棵树复核中： + +compact6: 0.15065802 +cross4: 0.15065995 + +所以最终保留了 compact6。 + +## 综合判断 + +当前未保留特征中，最可能存在构造不足的地方按优先级是： + + 优先级 可能问题 判断 +━━━━━━━━ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ━━━━━━━━━━━━ + 1 全局和横向特征只取当前时刻，没有形成18根K线的历史序列 高置信度 +──────── ─────────────────────────────────────────────────────── ──────────── + 2 SMA、趋势、ATR没有进一步除以品种局部波动率 中高置信度 +──────── ─────────────────────────────────────────────────────── ──────────── + 3 全局特征没有区分主要品种、低流动性品种和市场权重 中等置信度 +──────── ─────────────────────────────────────────────────────── ──────────── + 4 横向排名的品种集合随时间变化 中等置信度 +──────── ─────────────────────────────────────────────────────── ──────────── + 5 指标之间高度重复，增加了大量相似特征 高置信度 + +因此，本次结果更准确的表述是： + +> 当前实现的指标、全局和横向特征没有带来稳定增益；其中全局和横向特征的时间组织方式明显偏简单，RSI/SMA +> 等指标也没有做局部波动率归一化。不能据此断言这些特征方向本身无效。 diff --git a/tests/test_artifact_paths.py b/tests/test_artifact_paths.py new file mode 100644 index 0000000..9c81383 --- /dev/null +++ b/tests/test_artifact_paths.py @@ -0,0 +1,21 @@ +import unittest +from pathlib import Path + +from cbpred.__main__ import build_parser + + +class ArtifactPathTests(unittest.TestCase): + def test_cli_defaults_use_versioned_artifact_directory(self): + parser = build_parser() + + train = parser.parse_args(["train"]) + run = parser.parse_args(["run"]) + predict = parser.parse_args(["predict", "--csv", "data.csv"]) + + self.assertEqual(train.artifact_dir, Path("artifacts/v1")) + self.assertEqual(run.artifact_dir, Path("artifacts/v1")) + self.assertEqual(predict.artifact, Path("artifacts/v1/model.joblib")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_experiments.py b/tests/test_experiments.py new file mode 100644 index 0000000..02bf1d7 --- /dev/null +++ b/tests/test_experiments.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest + +import numpy as np + +from cbpred.experiments import ( + _CROSS_COLUMNS, + _GLOBAL_COLUMNS, + _cross_sectional_context, + experiment_variants, + feature_names, +) +from cbpred.pipeline import aggregate_8h, compute_indicators + +from tests.test_pipeline import make_hour_rows + + +class ExperimentFeatureTests(unittest.TestCase): + def test_variant_feature_counts(self): + counts = {variant.name: len(feature_names(variant)) for variant in experiment_variants()} + self.assertEqual(counts["baseline"], 182) + self.assertEqual(counts["compact7"], 164) + self.assertEqual(counts["compact6"], 146) + self.assertEqual(counts["rsi_sma"], 236) + self.assertEqual(counts["global6"], 188) + self.assertEqual(counts["cross4"], 186) + self.assertEqual(counts["global_cross"], 192) + + def test_cross_sectional_context_is_ranked_by_timestamp(self): + frames = { + f"COIN{index}": aggregate_8h(make_hour_rows(0, 8 * 90, index * 0.1)) + for index in range(4) + } + indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()} + profiles = { + symbol: { + "asset_atr_price_top20_median": 0.05, + "asset_quote_volume_top20_median_log": 12.0 + index, + } + for index, symbol in enumerate(frames) + } + global_context, cross_context = _cross_sectional_context(indicators, profiles) + self.assertEqual(list(global_context.columns), list(_GLOBAL_COLUMNS)) + self.assertEqual(set(next(iter(cross_context.values())).columns), set(_CROSS_COLUMNS)) + self.assertEqual(len(cross_context), 4) + for frame in cross_context.values(): + values = frame.iloc[20].to_numpy(dtype=float) + self.assertTrue(np.isfinite(values).all()) + self.assertTrue(np.all((0.0 < values) & (values <= 1.0))) + + +if __name__ == "__main__": + unittest.main()