From dacfb9262bb91a48342be077e265dcf00da85655 Mon Sep 17 00:00:00 2001 From: anyongjin Date: Sun, 23 Aug 2026 10:45:28 +0800 Subject: [PATCH] init --- .gitignore | 12 + README.md | 78 ++ cbpred.egg-info/PKG-INFO | 11 + cbpred.egg-info/SOURCES.txt | 12 + cbpred.egg-info/dependency_links.txt | 1 + cbpred.egg-info/entry_points.txt | 2 + cbpred.egg-info/requires.txt | 6 + cbpred.egg-info/top_level.txt | 1 + cbpred/__init__.py | 4 + cbpred/__main__.py | 138 +++ cbpred/pipeline.py | 1637 ++++++++++++++++++++++++++ cmd/cbpred-export/main.go | 109 ++ docs/abstract.md | 58 + go.mod | 112 ++ go.sum | 379 ++++++ pyproject.toml | 24 + tests/__init__.py | 1 + tests/test_pipeline.py | 167 +++ 18 files changed, 2752 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 cbpred.egg-info/PKG-INFO create mode 100644 cbpred.egg-info/SOURCES.txt create mode 100644 cbpred.egg-info/dependency_links.txt create mode 100644 cbpred.egg-info/entry_points.txt create mode 100644 cbpred.egg-info/requires.txt create mode 100644 cbpred.egg-info/top_level.txt create mode 100644 cbpred/__init__.py create mode 100644 cbpred/__main__.py create mode 100644 cbpred/pipeline.py create mode 100644 cmd/cbpred-export/main.go create mode 100644 docs/abstract.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/test_pipeline.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..99a0935 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +data/raw/ +data/manifest.json +artifacts/ +artifacts_*/ +reports/ +predictions/ +data_*/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..4e9bef8 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# cbpred + +这是一个统一模型的 8H U 本位 USDT 合约回调预测管线。它只使用 banbot 的 K 线 OHLCV 数据，默认调用 banbot 下载 1H 存储数据，再按完整的 8 根 1H K 线聚合为 8H；不使用资金费率、盘口、持仓量或外部行情。 + +目标定义为有符号的未来最低价收益，并按品种训练期波动尺度归一化： + +```text +raw_return = quantile(low[t+1:t+6], 0.20) / anchor_close - 1 +target = raw_return / asset_atr_price_top20_median +``` + +这里的 20% 分位数是未来窗口的稳健低点参考，不会被单根异常影线完全决定；它不是严格的 `lowestLow`。负值表示该低点参考低于当前收盘价，正值表示它仍高于当前收盘价；因此上涨阶段不会被强行压成 0。`asset_atr_price_top20_median` 是该品种训练期 ATR/price 降序前 20% 的中位数，预测输出会乘回该尺度。训练锚点和未来 6 根必须连续，跨越测试边界的训练样本会被丢弃，避免把测试期标签泄漏进训练集。 + +## 运行 + +先安装依赖： + +```bash +python3 -m venv .venv +.venv/bin/pip install -e . +``` + +使用 banbot 的配置连接数据库，筛选 `market=linear`、符号为 `BASE/USDT:USDT`、上市至少 730 天且未退市的品种；默认连接配置来自 `/home/ctyun/ban/banbot/biz/config.yml`： + +```bash +.venv/bin/python -m cbpred prepare \ + --banbot-config /home/ctyun/ban/banbot/biz/config.yml \ + --out-dir data +``` + +如果数据已经由 banbot 下载完成，可以跳过下载，只执行筛选和导出： + +```bash +.venv/bin/python -m cbpred prepare --skip-download --out-dir data +``` + +训练会执行三轮特征迭代。点模型候选包括随机森林平方误差和 ExtraTrees，另用 HistGradientBoosting 分位数模型输出 q20/q80；验证集和测试集均为最近 10 个月中的按品种随机拆分，测试集只在最终选模后使用： + +```bash +.venv/bin/python -m cbpred train \ + --data-dir data \ + --artifact-dir artifacts_signed +``` + +默认保留所有连续样本，把当前 ATR 在过去 200 根 8H K 线中的排名作为特征；可以通过 `--volatility-rank-threshold 0.80` 或 `0.90` 做高波动样本对照实验。训练阶段会把数据集行数、过滤前后数量、阶段和候选模型进度输出到 stderr。 + +一条命令完成准备和训练： + +```bash +.venv/bin/python -m cbpred run --out-dir data --artifact-dir artifacts_signed +``` + +输出在 `artifacts_signed/report.md`、`artifacts_signed/report.json` 和 `artifacts_signed/model.joblib`。报告会列出三轮特征的验证表现、最终模型和测试集 MAE/RMSE、q80 覆盖率、q20 下界未跌破率。 + +对某个已导出的 8H CSV 预测任意锚点： + +```bash +.venv/bin/python -m cbpred predict \ + --artifact artifacts_signed/model.joblib \ + --csv data/raw/123_BTC_USDT_USDT.csv \ + --at 2026010100000000 +``` + +`future_low_return_q20` 是稳健未来低点收益的较低分位，对应 `future_low_price_q20` 这个保守的价格下界；`future_low_return_q80` 是较浅、较乐观的分位。负收益才表示回撤，正收益表示稳健未来低点仍高于当前收盘价。兼容字段 `drawdown_*` 等于对应收益取负，实际覆盖率应以测试集校准报告为准，不能把 80% 当成保证概率。 + +如果预测时 `eligible_for_model` 为 `false`，说明过去 200 根不足以计算 ATR 排名，或显式设置的 ATR 排名门槛未达到，返回的模型数值为 `null`，应跳过该时刻。 + +## 特征迭代 + +第一轮使用 18 根 K 线的方向、实体/影线占比、收盘在 K 线中的位置、相对波幅、对数收益，以及基于法币成交额的 18 根窗口成交量排名和相对中位数；再加入两个固定的品种画像值：训练期 `ATR/price` 降序前 20% 的中位数、训练期法币成交额对数前 20% 的中位数。所有绝对价格都被转换为比例或滚动相对量，品种画像再用训练品种的均值/标准差标准化。 + +第二轮追加归一化 ATR、RSI、布林带位置/宽度、EMA 距离、趋势收益、滚动波动率和滚动回撤；第三轮追加 18 根窗口的趋势、波幅、方向比例、成交量趋势、收盘位置和指标交互摘要。三轮使用同一批训练/验证/测试边界，选择验证 q80 pinball loss 加覆盖率偏差后的最佳组合，最后只在测试集报告一次。 + +运行标准库回归测试： + +```bash +.venv/bin/python -m unittest discover -v +``` diff --git a/cbpred.egg-info/PKG-INFO b/cbpred.egg-info/PKG-INFO new file mode 100644 index 0000000..6b7e954 --- /dev/null +++ b/cbpred.egg-info/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 2.4 +Name: cbpred +Version: 0.1.0 +Summary: Unified 8H crypto futures drawdown prediction pipeline +Requires-Python: >=3.10 +Requires-Dist: joblib>=1.3 +Requires-Dist: numpy>=1.24 +Requires-Dist: pandas>=2.0 +Requires-Dist: psycopg[binary]>=3.1 +Requires-Dist: PyYAML>=6.0 +Requires-Dist: scikit-learn>=1.3 diff --git a/cbpred.egg-info/SOURCES.txt b/cbpred.egg-info/SOURCES.txt new file mode 100644 index 0000000..871a3a3 --- /dev/null +++ b/cbpred.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +README.md +pyproject.toml +cbpred/__init__.py +cbpred/__main__.py +cbpred/pipeline.py +cbpred.egg-info/PKG-INFO +cbpred.egg-info/SOURCES.txt +cbpred.egg-info/dependency_links.txt +cbpred.egg-info/entry_points.txt +cbpred.egg-info/requires.txt +cbpred.egg-info/top_level.txt +tests/test_pipeline.py \ No newline at end of file diff --git a/cbpred.egg-info/dependency_links.txt b/cbpred.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/cbpred.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/cbpred.egg-info/entry_points.txt b/cbpred.egg-info/entry_points.txt new file mode 100644 index 0000000..34e86fd --- /dev/null +++ b/cbpred.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +cbpred = cbpred.__main__:main diff --git a/cbpred.egg-info/requires.txt b/cbpred.egg-info/requires.txt new file mode 100644 index 0000000..6b62bb2 --- /dev/null +++ b/cbpred.egg-info/requires.txt @@ -0,0 +1,6 @@ +joblib>=1.3 +numpy>=1.24 +pandas>=2.0 +psycopg[binary]>=3.1 +PyYAML>=6.0 +scikit-learn>=1.3 diff --git a/cbpred.egg-info/top_level.txt b/cbpred.egg-info/top_level.txt new file mode 100644 index 0000000..b22744f --- /dev/null +++ b/cbpred.egg-info/top_level.txt @@ -0,0 +1 @@ +cbpred diff --git a/cbpred/__init__.py b/cbpred/__init__.py new file mode 100644 index 0000000..3c8f630 --- /dev/null +++ b/cbpred/__init__.py @@ -0,0 +1,4 @@ +"""Unified 8H drawdown prediction pipeline.""" + +__version__ = "0.1.0" + diff --git a/cbpred/__main__.py b/cbpred/__main__.py new file mode 100644 index 0000000..0799313 --- /dev/null +++ b/cbpred/__main__.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .pipeline import ( + VOLATILITY_RANK_THRESHOLD, + parse_time_ms, + predict_from_csv, + prepare_data, + run_training, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Train a unified 8H futures drawdown model") + sub = parser.add_subparsers(dest="command", required=True) + + prepare = sub.add_parser("prepare", help="filter symbols, call banbot, export complete 8H CSVs") + prepare.add_argument("--out-dir", type=Path, default=Path("data")) + prepare.add_argument("--banbot-dir", type=Path, default=Path("/home/ctyun/ban/banbot")) + prepare.add_argument("--banbot-config", type=Path, default=Path("/home/ctyun/ban/banbot/biz/config.yml")) + prepare.add_argument("--dsn", default=None) + prepare.add_argument("--db-type", choices=["auto", "questdb", "timescale"], default="auto") + prepare.add_argument("--exchange", default="binance") + prepare.add_argument("--end", dest="end_ms", default=None, help="UTC date/ISO timestamp or milliseconds") + prepare.add_argument("--history-months", type=int, default=36) + prepare.add_argument("--holdout-months", type=int, default=10) + prepare.add_argument("--min-age-days", type=int, default=730) + prepare.add_argument("--chunk-size", type=int, default=100) + prepare.add_argument("--banbot-cmd", default="go run .") + prepare.add_argument("--banbot-export-cmd", default="go run ./cmd/cbpred-export") + prepare.add_argument("--skip-download", action="store_true") + + train = sub.add_parser("train", help="build three feature stages, tune on validation symbols, test once") + train.add_argument("--data-dir", type=Path, default=Path("data")) + train.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed")) + train.add_argument("--end", dest="end_ms", default=None) + train.add_argument("--test-months", type=int, default=10) + train.add_argument("--seed", type=int, default=42) + train.add_argument("--n-estimators", type=int, default=250) + train.add_argument("--n-jobs", type=int, default=-1) + train.add_argument("--volatility-rank-threshold", type=float, default=VOLATILITY_RANK_THRESHOLD) + + run = sub.add_parser("run", help="prepare data and train the model") + for action in prepare._actions[1:]: + if action.dest not in {"help", "command"}: + run._add_action(action) + run.add_argument("--artifact-dir", type=Path, default=Path("artifacts_signed")) + run.add_argument("--test-months", type=int, default=10) + run.add_argument("--seed", type=int, default=42) + run.add_argument("--n-estimators", type=int, default=250) + run.add_argument("--n-jobs", type=int, default=-1) + run.add_argument("--volatility-rank-threshold", type=float, default=VOLATILITY_RANK_THRESHOLD) + + predict = sub.add_parser( + "predict", help="predict signed robust future-low q20/q80 from an exported 8H CSV" + ) + predict.add_argument("--artifact", type=Path, default=Path("artifacts_signed/model.joblib")) + predict.add_argument("--csv", type=Path, required=True) + predict.add_argument("--at", dest="at_ms", default=None) + predict.add_argument("--symbol", default=None, help="symbol used for the fixed training asset profile") + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.command == "prepare": + result = prepare_data( + out_dir=args.out_dir, + banbot_dir=args.banbot_dir, + banbot_config=args.banbot_config, + dsn=args.dsn, + db_type=args.db_type, + exchange=args.exchange, + end_ms=parse_time_ms(args.end_ms) if args.end_ms else None, + history_months=args.history_months, + holdout_months=args.holdout_months, + min_age_days=args.min_age_days, + chunk_size=args.chunk_size, + skip_download=args.skip_download, + banbot_cmd=args.banbot_cmd, + banbot_export_cmd=args.banbot_export_cmd, + ) + print(json.dumps({"selected": len(result["symbols"]), "excluded": len(result["excluded"])}, indent=2)) + elif args.command == "train": + result = run_training( + data_dir=args.data_dir, + artifact_dir=args.artifact_dir, + end_ms=parse_time_ms(args.end_ms) if args.end_ms else None, + test_months=args.test_months, + seed=args.seed, + n_estimators=args.n_estimators, + n_jobs=args.n_jobs, + volatility_rank_threshold=args.volatility_rank_threshold, + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + elif args.command == "run": + prepare_data( + out_dir=args.out_dir, + banbot_dir=args.banbot_dir, + banbot_config=args.banbot_config, + dsn=args.dsn, + db_type=args.db_type, + exchange=args.exchange, + end_ms=parse_time_ms(args.end_ms) if args.end_ms else None, + history_months=args.history_months, + holdout_months=args.holdout_months, + min_age_days=args.min_age_days, + chunk_size=args.chunk_size, + skip_download=args.skip_download, + banbot_cmd=args.banbot_cmd, + banbot_export_cmd=args.banbot_export_cmd, + ) + result = run_training( + data_dir=args.out_dir, + artifact_dir=args.artifact_dir, + end_ms=parse_time_ms(args.end_ms) if args.end_ms else None, + test_months=args.test_months, + seed=args.seed, + n_estimators=args.n_estimators, + n_jobs=args.n_jobs, + volatility_rank_threshold=args.volatility_rank_threshold, + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + else: + result = predict_from_csv( + artifact_path=args.artifact, + csv_path=args.csv, + at_ms=parse_time_ms(args.at_ms) if args.at_ms else None, + symbol=args.symbol, + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/cbpred/pipeline.py b/cbpred/pipeline.py new file mode 100644 index 0000000..38af897 --- /dev/null +++ b/cbpred/pipeline.py @@ -0,0 +1,1637 @@ +from __future__ import annotations + +import calendar +import json +import math +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +import joblib +import numpy as np +import pandas as pd +import yaml +from sklearn.ensemble import ( + ExtraTreesRegressor, + HistGradientBoostingRegressor, + RandomForestRegressor, +) +from sklearn.metrics import mean_absolute_error, mean_squared_error + + +EIGHT_H_MS = 8 * 60 * 60 * 1000 +HOUR_MS = 60 * 60 * 1000 +WINDOW = 18 +HORIZON = 6 +VOLATILITY_RANK_WINDOW = 200 +VOLATILITY_RANK_THRESHOLD = 0.0 +LOWER_QUANTILE = 1 - 0.80 +TARGET_QUANTILE = 0.80 +FUTURE_LOW_QUANTILE = 0.20 +TARGET_KIND = "signed_future_low_q20_return_atr_normalized" +CSV_COLUMNS = [ + "timestamp", + "open", + "high", + "low", + "close", + "volume", + "quote", + "buy_volume", + "trade_num", +] + + +def _progress(message: str) -> None: + print(f"[cbpred] {message}", file=sys.stderr, flush=True) + + +@dataclass(frozen=True) +class Symbol: + sid: int + exchange: str + market: str + symbol: str + list_ms: int + delist_ms: int + combined: bool = False + + +def parse_time_ms(value: str | int | None) -> int: + if value is None: + return floor_to_8h(int(datetime.now(timezone.utc).timestamp() * 1000)) + if isinstance(value, int): + return value + text = str(value).strip() + if text.isdigit(): + number = int(text) + if number > 10**11: + return number + if len(text) == 8: + dt = datetime.strptime(text, "%Y%m%d").replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + normalized = text.replace("Z", "+00:00") + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + + +def floor_to_8h(value_ms: int) -> int: + return value_ms - value_ms % EIGHT_H_MS + + +def months_before(value_ms: int, months: int) -> int: + dt = datetime.fromtimestamp(value_ms / 1000, tz=timezone.utc) + month = dt.month - months + year = dt.year + (month - 1) // 12 + month = (month - 1) % 12 + 1 + day = min(dt.day, calendar.monthrange(year, month)[1]) + result = dt.replace(year=year, month=month, day=day) + return int(result.timestamp() * 1000) + + +def load_database_url(config_path: Path | None, dsn: str | None) -> str: + if dsn: + return dsn + if not config_path: + raise ValueError("--dsn or --banbot-config is required") + data = yaml.safe_load(config_path.read_text()) or {} + url = ((data.get("database") or {}).get("url") or "").strip() + if not url: + raise ValueError(f"database.url is missing in {config_path}") + return url + + +def connect_database(dsn: str): + try: + import psycopg + except ImportError as exc: + raise RuntimeError("install dependencies with: python -m pip install -e .") from exc + return psycopg.connect(dsn, autocommit=True) + + +def detect_db_type(conn, requested: str = "auto") -> str: + if requested in {"questdb", "timescale"}: + return requested + with conn.cursor() as cur: + try: + cur.execute("SELECT 1 FROM exsymbol_q LIMIT 1") + return "questdb" + except Exception: + conn.rollback() + return "timescale" + + +def fetch_symbols( + conn, + db_type: str, + exchange: str, + eligible_at_ms: int, + min_age_days: int, +) -> tuple[list[Symbol], list[dict[str, Any]]]: + if db_type == "questdb": + sql = """ + SELECT sid, exchange, market, symbol, list_ms, delist_ms, combined + FROM exsymbol_q + LATEST BY sid + WHERE exchange = %s AND coalesce(is_deleted, false) = false + ORDER BY sid + """ + else: + sql = """ + SELECT id, exchange, market, symbol, list_ms, delist_ms, combined + FROM exsymbol + WHERE exchange = %s + ORDER BY id + """ + cutoff = eligible_at_ms - min_age_days * 24 * 60 * 60 * 1000 + selected: list[Symbol] = [] + excluded: list[dict[str, Any]] = [] + with conn.cursor() as cur: + cur.execute(sql, (exchange,)) + for row in cur.fetchall(): + sid, exg, market, symbol, list_ms, delist_ms, combined = row + item = { + "sid": int(sid), + "exchange": str(exg), + "market": str(market), + "symbol": str(symbol), + "list_ms": int(list_ms or 0), + "delist_ms": int(delist_ms or 0), + "combined": bool(combined), + } + reason = None + if item["market"].lower() != "linear": + reason = "market_not_linear" + elif not is_usdt_linear_symbol(item["symbol"]): + reason = "not_usdt_settled" + elif item["combined"]: + reason = "combined_symbol" + elif item["list_ms"] <= 0: + reason = "missing_list_ms" + elif item["list_ms"] > cutoff: + reason = "listed_less_than_min_age" + elif item["delist_ms"] and item["delist_ms"] <= eligible_at_ms: + reason = "delisted_before_training_snapshot" + if reason: + item["reason"] = reason + excluded.append(item) + continue + selected.append(Symbol(**item)) + return selected, excluded + + +def is_usdt_linear_symbol(symbol: str) -> bool: + parts = symbol.upper().split("/") + return len(parts) == 2 and parts[1] == "USDT:USDT" + + +def run_banbot_download( + symbols: list[Symbol], + banbot_dir: Path, + config_path: Path, + history_start_ms: int, + end_ms: int, + chunk_size: int = 100, + banbot_cmd: str = "go run .", +) -> None: + if not symbols: + raise ValueError("no eligible symbols to download") + base = shlex.split(banbot_cmd) + for offset in range(0, len(symbols), chunk_size): + chunk = symbols[offset : offset + chunk_size] + pairs = ",".join(item.symbol for item in chunk) + command = base + [ + "kline", + "down", + "--config", + str(config_path), + "--datadir", + str(config_path.parent), + "--no-default", + "--config-data", + "pairlists: []\npairmgr:\n force_filters: false\n", + "--pairs", + pairs, + "--timeframes", + "1h", + "--timestart", + str(history_start_ms), + "--timeend", + str(end_ms), + ] + subprocess.run(command, cwd=banbot_dir, check=True) + + +def run_banbot_export( + symbols: list[Symbol], + banbot_dir: Path, + banbot_config: Path, + out_dir: Path, + history_start_ms: int, + end_ms: int, + banbot_export_cmd: str = "go run ./cmd/cbpred-export", +) -> None: + if not symbols: + raise ValueError("no eligible symbols to export") + out_dir.mkdir(parents=True, exist_ok=True) + pairs = ",".join(item.symbol for item in symbols) + command = shlex.split(banbot_export_cmd) + [ + "--config", + str(banbot_config), + "--data-dir", + str(banbot_config.parent), + "--pairs", + pairs, + "--start-ms", + str(history_start_ms), + "--end-ms", + str(end_ms), + "--out-dir", + str(out_dir), + ] + project_dir = Path(__file__).resolve().parents[1] + subprocess.run(command, cwd=project_dir, check=True) + + +def aggregate_8h(rows: Iterable[tuple[Any, ...]]) -> pd.DataFrame: + columns = CSV_COLUMNS + values = list(rows) + if not values: + return pd.DataFrame(columns=columns) + frame = pd.DataFrame(values, columns=columns) + for column in columns: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + frame = frame.dropna(subset=["timestamp", "open", "high", "low", "close"]) + frame = frame.sort_values("timestamp").drop_duplicates("timestamp", keep="last") + frame["timestamp"] = frame["timestamp"].astype(np.int64) + frame["quote"] = frame["quote"].fillna(0.0) + frame["buy_volume"] = frame["buy_volume"].fillna(0.0) + frame["trade_num"] = frame["trade_num"].fillna(0).astype(np.int64) + frame["quote"] = np.where(frame["quote"] > 0, frame["quote"], frame["volume"] * frame["close"]) + frame["bucket"] = (frame["timestamp"] // EIGHT_H_MS) * EIGHT_H_MS + output: list[dict[str, Any]] = [] + for bucket, group in frame.groupby("bucket", sort=True): + group = group.sort_values("timestamp") + expected = bucket + np.arange(8, dtype=np.int64) * HOUR_MS + if len(group) != 8 or not np.array_equal(group["timestamp"].to_numpy(), expected): + continue + output.append( + { + "timestamp": int(bucket), + "open": float(group.iloc[0]["open"]), + "high": float(group["high"].max()), + "low": float(group["low"].min()), + "close": float(group.iloc[-1]["close"]), + "volume": float(group["volume"].sum()), + "quote": float(group["quote"].sum()), + "buy_volume": float(group["buy_volume"].sum()), + "trade_num": int(group["trade_num"].sum()), + } + ) + return pd.DataFrame(output, columns=columns) + + +def symbol_filename(symbol: Symbol) -> str: + slug = symbol.symbol.replace("/", "_").replace(":", "_") + return f"{symbol.sid}_{slug}.csv" + + +def prepare_data( + out_dir: Path, + banbot_dir: Path, + banbot_config: Path, + dsn: str | None = None, + db_type: str = "auto", + exchange: str = "binance", + end_ms: int | None = None, + history_months: int = 36, + holdout_months: int = 10, + min_age_days: int = 730, + chunk_size: int = 100, + skip_download: bool = False, + banbot_cmd: str = "go run .", + banbot_export_cmd: str = "go run ./cmd/cbpred-export", +) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + raw_dir = out_dir / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + end_ms = floor_to_8h(end_ms or parse_time_ms(None)) + history_start_ms = months_before(end_ms, history_months) + training_snapshot_ms = months_before(end_ms, holdout_months) + database_url = load_database_url(banbot_config, dsn) + conn = connect_database(database_url) + try: + actual_db_type = detect_db_type(conn, db_type) + symbols, excluded = fetch_symbols( + conn, + actual_db_type, + exchange, + training_snapshot_ms, + min_age_days, + ) + finally: + conn.close() + if not skip_download: + run_banbot_download( + symbols, + banbot_dir, + banbot_config, + history_start_ms, + end_ms, + chunk_size, + banbot_cmd, + ) + run_banbot_export( + symbols, + banbot_dir, + banbot_config, + raw_dir, + history_start_ms, + end_ms, + banbot_export_cmd, + ) + manifest_symbols: list[dict[str, Any]] = [] + for symbol in symbols: + path = raw_dir / symbol_filename(symbol) + if not path.exists(): + excluded.append({**symbol.__dict__, "reason": "no_complete_8h_bars"}) + continue + frame = pd.read_csv(path) + frame["timestamp"] = pd.to_numeric(frame["timestamp"], errors="coerce") + frame = frame.dropna(subset=["timestamp"]).sort_values("timestamp") + if symbol.delist_ms: + frame = frame[frame["timestamp"] < symbol.delist_ms] + frame.to_csv(path, index=False, float_format="%.12g") + if frame.empty: + excluded.append({**symbol.__dict__, "reason": "no_complete_8h_bars"}) + path.unlink(missing_ok=True) + continue + manifest_symbols.append({**symbol.__dict__, "path": str(path.relative_to(out_dir)), "bars": len(frame)}) + manifest = { + "version": 2, + "timeframe": "8h", + "storage_timeframe": "banbot_orm", + "step_ms": EIGHT_H_MS, + "end_ms": end_ms, + "history_start_ms": history_start_ms, + "training_snapshot_ms": training_snapshot_ms, + "db_type": actual_db_type, + "exchange": exchange, + "symbols": manifest_symbols, + "excluded": excluded, + } + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False)) + return manifest + + +def load_manifest(data_dir: Path) -> dict[str, Any]: + path = data_dir / "manifest.json" + if not path.exists(): + raise FileNotFoundError(f"missing {path}; run prepare first") + return json.loads(path.read_text()) + + +def load_raw_frames(data_dir: Path, manifest: dict[str, Any]) -> dict[str, pd.DataFrame]: + frames: dict[str, pd.DataFrame] = {} + for item in manifest.get("symbols", []): + path = data_dir / item["path"] + frame = pd.read_csv(path) + for column in CSV_COLUMNS: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + frame = frame.dropna(subset=["timestamp", "open", "high", "low", "close"]) + frame = frame.sort_values("timestamp").drop_duplicates("timestamp", keep="last").reset_index(drop=True) + frame["timestamp"] = frame["timestamp"].astype(np.int64) + delist_ms = int(item.get("delist_ms", 0) or 0) + if delist_ms: + frame = frame[frame["timestamp"] < delist_ms].reset_index(drop=True) + frame["quote"] = np.where(frame["quote"] > 0, frame["quote"], frame["volume"] * frame["close"]) + frames[item["symbol"]] = frame + return frames + + +def compute_indicators(frame: pd.DataFrame) -> pd.DataFrame: + df = frame.copy() + close = df["close"].astype(float) + high = df["high"].astype(float) + low = df["low"].astype(float) + open_ = df["open"].astype(float) + previous = close.shift(1) + true_range = pd.concat( + [high - low, (high - previous).abs(), (low - previous).abs()], axis=1 + ).max(axis=1) + price_range = (high - low).clip(lower=1e-12) + log_return = np.log(close / previous.replace(0, np.nan)) + df["body_sign"] = np.sign(close - open_) + df["body_range"] = (close - open_) / price_range + df["body_abs_range"] = (close - open_).abs() / price_range + df["upper_wick"] = (high - pd.concat([open_, close], axis=1).max(axis=1)) / price_range + df["lower_wick"] = (pd.concat([open_, close], axis=1).min(axis=1) - low) / price_range + df["close_location"] = 2 * (close - low) / price_range - 1 + df["range_pct"] = price_range / close.replace(0, np.nan) + df["log_return"] = log_return.clip(-2, 2) + df["atr_pct"] = true_range.ewm(alpha=1 / 14, adjust=False, min_periods=14).mean() / close + df["atr_rank_200"] = df["atr_pct"].rolling( + VOLATILITY_RANK_WINDOW, min_periods=VOLATILITY_RANK_WINDOW + ).rank(method="average", pct=True) + delta = close.diff() + gain = delta.clip(lower=0).ewm(alpha=1 / 14, adjust=False, min_periods=14).mean() + loss = (-delta.clip(upper=0)).ewm(alpha=1 / 14, adjust=False, min_periods=14).mean() + rs = gain / loss.replace(0, np.nan) + rsi = (100 - 100 / (1 + rs)) / 100 + df["rsi_norm"] = rsi.where(loss > 0, np.where(gain > 0, 1.0, 0.5)) + sma20 = close.rolling(20, min_periods=20).mean() + std20 = close.rolling(20, min_periods=20).std(ddof=0) + df["bb_position"] = ((close - sma20) / (2 * std20.replace(0, np.nan))).clip(-5, 5) + df["bb_width"] = (4 * std20 / sma20.replace(0, np.nan)).clip(0, 5) + ema9 = close.ewm(span=9, adjust=False, min_periods=9).mean() + ema21 = close.ewm(span=21, adjust=False, min_periods=21).mean() + df["ema9_gap"] = (close / ema9 - 1).clip(-2, 2) + df["ema21_gap"] = (close / ema21 - 1).clip(-2, 2) + df["trend_3"] = (close / close.shift(3) - 1).clip(-2, 2) + df["trend_9"] = (close / close.shift(9) - 1).clip(-2, 2) + df["volatility_18"] = log_return.rolling(18, min_periods=18).std(ddof=0).clip(0, 2) + df["drawdown_18"] = (close / close.rolling(18, min_periods=18).max() - 1).clip(-2, 0) + df["quote_log"] = np.log1p(df["quote"].clip(lower=0)) + return df + + +def asset_profile(indicators: pd.DataFrame, train_end_ms: int) -> dict[str, float] | None: + history = indicators[indicators["timestamp"] < train_end_ms] + atr = history["atr_pct"].dropna().to_numpy(dtype=float) + volume = history["quote_log"].replace([np.inf, -np.inf], np.nan).dropna().to_numpy(dtype=float) + if len(atr) < 20 or len(volume) < 20: + return None + atr_top = np.sort(atr)[-max(1, math.ceil(len(atr) * 0.20)) :] + volume_top = np.sort(volume)[-max(1, math.ceil(len(volume) * 0.20)) :] + return { + "asset_atr_price_top20_median": float(np.median(atr_top)), + "asset_quote_volume_top20_median_log": float(np.median(volume_top)), + } + + +def fit_asset_scaler( + profiles: dict[str, dict[str, float]], + symbols: Iterable[str] | None = None, +) -> dict[str, dict[str, float]]: + names = ["asset_atr_price_top20_median", "asset_quote_volume_top20_median_log"] + selected = set(symbols) if symbols is not None else set(profiles) + result: dict[str, dict[str, float]] = {} + for name in names: + values = np.array([profiles[symbol][name] for symbol in selected if symbol in profiles], dtype=float) + if len(values) == 0: + raise ValueError("no asset profiles available for scaler") + result[name] = {"mean": float(values.mean()), "std": float(max(values.std(ddof=0), 1e-9))} + return result + + +def scale_asset_profile(profile: dict[str, float], scaler: dict[str, dict[str, float]]) -> dict[str, float]: + return { + name: (profile[name] - values["mean"]) / values["std"] + for name, values in scaler.items() + } + + +_BASE_FEATURE_COLUMNS = ( + "body_sign", + "body_range", + "body_abs_range", + "upper_wick", + "lower_wick", + "close_location", + "range_pct", + "log_return", +) +_INDICATOR_FEATURE_COLUMNS = ( + "atr_pct", + "atr_rank_200", + "rsi_norm", + "bb_position", + "bb_width", + "ema9_gap", + "ema21_gap", + "trend_3", + "trend_9", + "volatility_18", + "drawdown_18", +) +_ASSET_FEATURE_NAMES = ( + "asset_atr_price_top20_median", + "asset_quote_volume_top20_median_log", +) +_SUMMARY_FEATURE_NAMES = ( + "summary_return_18", + "summary_return_3", + "summary_return_6", + "summary_range_mean", + "summary_range_std", + "summary_range_max_rel_mean", + "summary_up_ratio", + "summary_down_ratio", + "summary_close_position", + "summary_volume_trend", + "summary_return_mean", + "summary_return_std", + "summary_rsi_norm", + "summary_bb_position", + "summary_bb_width", +) + + +def _window_feature_names(stage: int) -> list[str]: + names: list[str] = [] + for lag in range(WINDOW - 1, -1, -1): + names.extend(f"bar_{lag:02d}_{name}" for name in _BASE_FEATURE_COLUMNS) + names.extend( + [ + f"bar_{lag:02d}_quote_volume_rank", + f"bar_{lag:02d}_quote_volume_log_rel_median", + ] + ) + if stage >= 2: + names.extend(f"bar_{lag:02d}_{name}" for name in _INDICATOR_FEATURE_COLUMNS) + names.extend(_ASSET_FEATURE_NAMES) + if stage >= 3: + names.extend(_SUMMARY_FEATURE_NAMES) + return names + + +def _rank_windows(values: np.ndarray) -> np.ndarray: + """Return pandas-compatible average percentile ranks for each row.""" + width = values.shape[1] + order = np.argsort(values, axis=1, kind="mergesort") + sorted_values = np.take_along_axis(values, order, axis=1) + same_previous = np.zeros(sorted_values.shape, dtype=bool) + same_previous[:, 1:] = sorted_values[:, 1:] == sorted_values[:, :-1] + group_start = ~same_previous + positions = np.arange(1, width + 1, dtype=float)[None, :] + start_ranks = np.where(group_start, positions, 0.0) + start_ranks = np.maximum.accumulate(start_ranks, axis=1) + + same_next = np.zeros(sorted_values.shape, dtype=bool) + same_next[:, :-1] = sorted_values[:, :-1] == sorted_values[:, 1:] + group_end = ~same_next + end_ranks = np.where(group_end, positions, float(width + 1)) + end_ranks = np.minimum.accumulate(end_ranks[:, ::-1], axis=1)[:, ::-1] + sorted_ranks = (start_ranks + end_ranks) / (2.0 * width) + ranks = np.empty_like(sorted_ranks) + np.put_along_axis(ranks, order, sorted_ranks, axis=1) + return ranks + + +def _vectorized_window_features( + frame: pd.DataFrame, + stage: int, + anchor_indices: np.ndarray, + asset: dict[str, float], +) -> np.ndarray: + starts = anchor_indices - (WINDOW - 1) + quote = frame["quote_log"].to_numpy(dtype=float) + quote_windows = np.lib.stride_tricks.sliding_window_view(quote, WINDOW)[starts] + volume_rank = _rank_windows(quote_windows) + volume_median = np.median(quote_windows, axis=1, keepdims=True) + + columns = list(_BASE_FEATURE_COLUMNS) + if stage >= 2: + columns.extend(_INDICATOR_FEATURE_COLUMNS) + values = frame[columns].to_numpy(dtype=float) + windows = np.lib.stride_tricks.sliding_window_view(values, WINDOW, axis=0) + windows = windows.transpose(0, 2, 1)[starts] + parts = [windows[:, :, : len(_BASE_FEATURE_COLUMNS)]] + parts.extend( + [ + volume_rank[:, :, None], + (quote_windows - volume_median)[:, :, None], + ] + ) + if stage >= 2: + parts.append(windows[:, :, len(_BASE_FEATURE_COLUMNS) :]) + feature_matrix = np.concatenate(parts, axis=2).reshape(len(anchor_indices), -1) + + asset_values = np.array([asset[name] for name in _ASSET_FEATURE_NAMES], dtype=float) + feature_matrix = np.column_stack( + [feature_matrix, np.repeat(asset_values[None, :], len(anchor_indices), axis=0)] + ) + if stage < 3: + return feature_matrix + + close = frame["close"].to_numpy(dtype=float) + high = frame["high"].to_numpy(dtype=float) + low = frame["low"].to_numpy(dtype=float) + log_return = frame["log_return"].to_numpy(dtype=float) + body_sign = frame["body_sign"].to_numpy(dtype=float) + range_pct = frame["range_pct"].to_numpy(dtype=float) + close_windows = np.lib.stride_tricks.sliding_window_view(close, WINDOW)[starts] + high_windows = np.lib.stride_tricks.sliding_window_view(high, WINDOW)[starts] + low_windows = np.lib.stride_tricks.sliding_window_view(low, WINDOW)[starts] + return_windows = np.lib.stride_tricks.sliding_window_view(log_return, WINDOW)[starts] + sign_windows = np.lib.stride_tricks.sliding_window_view(body_sign, WINDOW)[starts] + range_windows = np.lib.stride_tricks.sliding_window_view(range_pct, WINDOW)[starts] + x = np.arange(WINDOW, dtype=float) + x_centered = x - x.mean() + x_sum_squares = float(np.sum(x_centered * x_centered)) + range_mean = range_windows.mean(axis=1) + summaries = np.column_stack( + [ + np.log(close_windows[:, -1] / close_windows[:, 0]), + np.log(close_windows[:, -1] / close_windows[:, -4]), + np.log(close_windows[:, -1] / close_windows[:, -7]), + range_mean, + range_windows.std(axis=1), + range_windows[:, -1] / np.maximum(range_mean, 1e-9), + np.mean(sign_windows > 0, axis=1), + np.mean(sign_windows < 0, axis=1), + (close_windows[:, -1] - low_windows.min(axis=1)) + / np.maximum(high_windows.max(axis=1) - low_windows.min(axis=1), 1e-9), + ((quote_windows - quote_windows.mean(axis=1, keepdims=True)) * x_centered).sum(axis=1) + / x_sum_squares, + np.nanmean(return_windows, axis=1), + np.nanstd(return_windows, axis=1), + frame["rsi_norm"].to_numpy(dtype=float)[anchor_indices], + frame["bb_position"].to_numpy(dtype=float)[anchor_indices], + frame["bb_width"].to_numpy(dtype=float)[anchor_indices], + ] + ) + return np.column_stack([feature_matrix, summaries]) + + +def _window_features(window: pd.DataFrame, stage: int, asset: dict[str, float]) -> dict[str, float]: + result: dict[str, float] = {} + volume_rank = window["quote_log"].rank(method="average", pct=True).to_numpy(dtype=float) + volume_median = float(window["quote_log"].median()) + base_values = window[list(_BASE_FEATURE_COLUMNS)].to_numpy(dtype=float) + indicator_values = ( + window[list(_INDICATOR_FEATURE_COLUMNS)].to_numpy(dtype=float) if stage >= 2 else None + ) + quote_values = window["quote_log"].to_numpy(dtype=float) + for position in range(len(window)): + lag = WINDOW - 1 - position + for column_index, name in enumerate(_BASE_FEATURE_COLUMNS): + result[f"bar_{lag:02d}_{name}"] = float(base_values[position, column_index]) + result[f"bar_{lag:02d}_quote_volume_rank"] = float(volume_rank[position]) + result[f"bar_{lag:02d}_quote_volume_log_rel_median"] = float(quote_values[position] - volume_median) + if indicator_values is not None: + for column_index, name in enumerate(_INDICATOR_FEATURE_COLUMNS): + result[f"bar_{lag:02d}_{name}"] = float(indicator_values[position, column_index]) + result.update(asset) + if stage >= 3: + close = window["close"].to_numpy(dtype=float) + returns = window["log_return"].to_numpy(dtype=float) + ranges = window["range_pct"].to_numpy(dtype=float) + signs = window["body_sign"].to_numpy(dtype=float) + result.update( + { + "summary_return_18": float(np.log(close[-1] / close[0])), + "summary_return_3": float(np.log(close[-1] / close[-4])), + "summary_return_6": float(np.log(close[-1] / close[-7])), + "summary_range_mean": float(np.mean(ranges)), + "summary_range_std": float(np.std(ranges)), + "summary_range_max_rel_mean": float(ranges[-1] / max(np.mean(ranges), 1e-9)), + "summary_up_ratio": float(np.mean(signs > 0)), + "summary_down_ratio": float(np.mean(signs < 0)), + "summary_close_position": float( + (close[-1] - window["low"].min()) + / max(window["high"].max() - window["low"].min(), 1e-9) + ), + "summary_volume_trend": float( + np.polyfit(np.arange(WINDOW), window["quote_log"].to_numpy(dtype=float), 1)[0] + ), + "summary_return_mean": float(np.nanmean(returns)), + "summary_return_std": float(np.nanstd(returns)), + "summary_rsi_norm": float(window["rsi_norm"].iloc[-1]), + "summary_bb_position": float(window["bb_position"].iloc[-1]), + "summary_bb_width": float(window["bb_width"].iloc[-1]), + } + ) + return result + + +def build_feature_dataset( + frames: dict[str, pd.DataFrame], + stage: int, + test_start_ms: int, + end_ms: int, + asset_scaler: dict[str, dict[str, float]], + train_symbols: Iterable[str] | None = None, + validation_symbols: Iterable[str] | None = None, + test_symbols: Iterable[str] | None = None, + validation_end_ms: int | None = None, + calibration_end_ms: int | None = None, + volatility_rank_threshold: float = VOLATILITY_RANK_THRESHOLD, +) -> tuple[pd.DataFrame, list[str], dict[str, dict[str, float]]]: + if not 0.0 <= volatility_rank_threshold <= 1.0: + raise ValueError("volatility_rank_threshold must be between 0 and 1") + indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()} + raw_profiles = { + symbol: asset_profile(frame, test_start_ms) + for symbol, frame in indicators.items() + } + profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None} + if not profiles: + raise ValueError("no symbol has enough pre-test history for asset profiles") + scaled_profiles = { + symbol: scale_asset_profile(profile, asset_scaler) + for symbol, profile in profiles.items() + } + rows: list[pd.DataFrame] = [] + candidate_counts = {"train": 0, "validation": 0, "calibration": 0, "test": 0, "recent": 0} + selected_counts = {name: 0 for name in candidate_counts} + step = EIGHT_H_MS + explicit_splits = train_symbols is not None and validation_symbols is not None and test_symbols is not None + train_set = set(train_symbols or []) + validation_set = set(validation_symbols or []) + test_set = set(test_symbols or []) + if explicit_splits and (validation_end_ms is None or calibration_end_ms is None): + raise ValueError("validation_end_ms and calibration_end_ms are required for explicit splits") + for symbol, frame in indicators.items(): + asset = scaled_profiles.get(symbol) + if asset is None: + continue + frame = frame.reset_index(drop=True) + timestamps = frame["timestamp"].to_numpy(dtype=np.int64) + if len(frame) < WINDOW + HORIZON: + continue + anchor_indices = np.arange(WINDOW - 1, len(frame) - HORIZON, dtype=np.int64) + continuity = np.convolve( + (np.diff(timestamps) == step).astype(np.int8), + np.ones(WINDOW + HORIZON - 1, dtype=np.int8), + mode="valid", + ) == WINDOW + HORIZON - 1 + anchor_ms = timestamps[anchor_indices] + future_end_ms = timestamps[anchor_indices + HORIZON] + step + split_labels = np.full(len(anchor_indices), "", dtype=object) + if explicit_splits: + if symbol in train_set: + mask = (anchor_ms < test_start_ms) & (future_end_ms <= test_start_ms) + split_labels[mask] = "train" + if symbol in validation_set: + mask = ( + (test_start_ms <= anchor_ms) + & (anchor_ms < int(validation_end_ms)) + & (future_end_ms <= int(validation_end_ms)) + ) + split_labels[mask] = "validation" + mask = ( + (int(validation_end_ms) <= anchor_ms) + & (anchor_ms < int(calibration_end_ms)) + & (future_end_ms <= int(calibration_end_ms)) + ) + split_labels[mask] = "calibration" + if symbol in test_set: + mask = ( + (int(calibration_end_ms) <= anchor_ms) + & (anchor_ms < end_ms) + & (future_end_ms <= end_ms) + ) + split_labels[mask] = "test" + else: + mask = (anchor_ms < test_start_ms) & (future_end_ms <= test_start_ms) + split_labels[mask] = "train" + mask = ( + (test_start_ms <= anchor_ms) + & (anchor_ms < end_ms) + & (future_end_ms <= end_ms) + ) + split_labels[mask] = "recent" + candidate = continuity & (split_labels != "") + for split_name in candidate_counts: + candidate_counts[split_name] += int(np.count_nonzero(candidate & (split_labels == split_name))) + volatility_rank = frame["atr_rank_200"].to_numpy(dtype=float)[anchor_indices] + if volatility_rank_threshold == 0.0: + selected = candidate + else: + selected = candidate & np.isfinite(volatility_rank) & (volatility_rank >= volatility_rank_threshold) + for split_name in selected_counts: + selected_counts[split_name] += int(np.count_nonzero(selected & (split_labels == split_name))) + if not np.any(selected): + continue + selected_indices = anchor_indices[selected] + feature_matrix = _vectorized_window_features(frame, stage, selected_indices, asset) + low = frame["low"].to_numpy(dtype=float) + close = frame["close"].to_numpy(dtype=float)[selected_indices] + future_lows = np.column_stack( + [low[selected_indices + offset] for offset in range(1, HORIZON + 1)] + ) + future_low_reference = np.quantile(future_lows, FUTURE_LOW_QUANTILE, axis=1) + future_low_return = future_low_reference / np.maximum(close, 1e-12) - 1.0 + target_scale = max( + float(profiles[symbol]["asset_atr_price_top20_median"]), + 1e-9, + ) + target = future_low_return / target_scale + sample_atr_pct = frame["atr_pct"].to_numpy(dtype=float)[selected_indices] + finite = ( + np.isfinite(target) + & np.isfinite(target_scale) + & np.isfinite(sample_atr_pct) + & np.isfinite(feature_matrix).all(axis=1) + ) + if not np.any(finite): + continue + feature_names = _window_feature_names(stage) + block = pd.DataFrame(feature_matrix[finite], columns=feature_names) + block["symbol"] = symbol + block["timestamp"] = anchor_ms[selected][finite] + block["target"] = target[finite] + block["target_scale"] = target_scale + block["split"] = split_labels[selected][finite] + block["sample_atr_pct"] = sample_atr_pct[finite] + rows.append(block) + dataset = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame() + if dataset.empty: + raise ValueError("feature dataset is empty") + dataset.attrs["volatility_filter"] = { + "window": VOLATILITY_RANK_WINDOW, + "threshold": volatility_rank_threshold, + "candidate_counts": candidate_counts, + "selected_counts": selected_counts, + } + feature_names = _window_feature_names(stage) + return dataset, feature_names, profiles + + +def split_holdout_symbols(symbols: Iterable[str], seed: int = 42, holdout_ratio: float = 0.40) -> tuple[list[str], list[str], list[str]]: + available = sorted(set(symbols)) + if len(available) < 3: + raise ValueError("at least 3 symbols are required for train/validation/test by symbol") + rng = np.random.default_rng(seed) + shuffled = list(np.array(available)[rng.permutation(len(available))]) + holdout_count = max(2, int(math.ceil(len(shuffled) * holdout_ratio))) + holdout_count = min(holdout_count, len(shuffled) - 1) + validation_count = max(1, holdout_count // 2) + validation = sorted(shuffled[:validation_count]) + test = sorted(shuffled[validation_count:holdout_count]) + train_symbols = sorted(shuffled[holdout_count:]) + return train_symbols, validation, test + + +def recent_split_boundaries(start_ms: int, end_ms: int) -> tuple[int, int]: + """Use the first 30% for model selection, next 20% for calibration, last 50% for final test.""" + duration = end_ms - start_ms + validation_end = floor_to_8h(start_ms + int(duration * 0.30)) + calibration_end = floor_to_8h(start_ms + int(duration * 0.50)) + validation_end = max(start_ms + EIGHT_H_MS, min(validation_end, end_ms - 2 * EIGHT_H_MS)) + calibration_end = max(validation_end + EIGHT_H_MS, min(calibration_end, end_ms - EIGHT_H_MS)) + return validation_end, calibration_end + + +def _candidate_specs(random_state: int, n_estimators: int, n_jobs: int) -> list[dict[str, Any]]: + tree_common = dict( + n_estimators=n_estimators, + max_features=0.7, + min_samples_leaf=8, + random_state=random_state, + n_jobs=n_jobs, + verbose=1, + ) + return [ + { + "name": "random_forest_squared_error", + "kind": "point_plus_gbdt_quantiles", + "point_factory": lambda: RandomForestRegressor(criterion="squared_error", **tree_common), + }, + { + "name": "extra_trees_squared_error", + "kind": "point_plus_gbdt_quantiles", + "point_factory": lambda: ExtraTreesRegressor(criterion="squared_error", **tree_common), + }, + ] + + +def _quantile_model( + alpha: float, random_state: int, n_estimators: int +) -> HistGradientBoostingRegressor: + return HistGradientBoostingRegressor( + quantile=alpha, + max_iter=max(1, n_estimators), + learning_rate=0.04, + max_depth=2, + min_samples_leaf=8, + random_state=random_state, + verbose=1, + ) + + +def predict_quantiles( + point_model, + lower_model, + upper_model, + features: pd.DataFrame, + model_kind: str, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + if model_kind != "point_plus_gbdt_quantiles": + raise ValueError(f"unsupported model kind: {model_kind}") + point = np.asarray(point_model.predict(features), dtype=float) + lower = np.asarray(lower_model.predict(features), dtype=float) + upper = np.asarray(upper_model.predict(features), dtype=float) + return point, lower, upper + + +def pinball_loss(y_true: np.ndarray, prediction: np.ndarray, quantile: float) -> float: + error = y_true - prediction + return float(np.mean(np.maximum(quantile * error, (quantile - 1) * error))) + + +def evaluate_predictions( + y_true: np.ndarray, + point: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + target_scale: np.ndarray | None = None, +) -> dict[str, float]: + scale = np.ones_like(y_true) if target_scale is None else np.asarray(target_scale, dtype=float) + actual_y = y_true * scale + actual_point = point * scale + actual_lower = lower * scale + actual_upper = upper * scale + q20_coverage = float(np.mean(y_true <= lower)) + q80_coverage = float(np.mean(y_true <= upper)) + return { + "n": int(len(y_true)), + "mae_point": float(mean_absolute_error(actual_y, actual_point)), + "rmse_point": float(math.sqrt(mean_squared_error(actual_y, actual_point))), + "mae_q20": float(mean_absolute_error(actual_y, actual_lower)), + "rmse_q20": float(math.sqrt(mean_squared_error(actual_y, actual_lower))), + "mae_q80": float(mean_absolute_error(actual_y, actual_upper)), + "rmse_q80": float(math.sqrt(mean_squared_error(actual_y, actual_upper))), + "pinball_q20": pinball_loss(actual_y, actual_lower, LOWER_QUANTILE), + "pinball_q80": pinball_loss(actual_y, actual_upper, TARGET_QUANTILE), + "q20_coverage": q20_coverage, + "q80_coverage": q80_coverage, + "q20_hit_rate": float(np.mean(y_true >= lower)), + "q20_calibration_gap": float(abs(q20_coverage - LOWER_QUANTILE)), + "q80_calibration_gap": float(abs(q80_coverage - TARGET_QUANTILE)), + "quantile_cross_rate": float(np.mean(lower > upper)), + } + + +def _fit_stage( + dataset: pd.DataFrame, + feature_names: list[str], + validation_symbols: list[str], + stage: int, + random_state: int, + n_estimators: int, + n_jobs: int, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + train = dataset[dataset["split"] == "train"] + validation = dataset[(dataset["split"] == "validation") & dataset["symbol"].isin(validation_symbols)] + if train.empty or validation.empty: + raise ValueError("train or validation samples are empty") + train_x = train[feature_names] + train_y = train["target"].to_numpy(dtype=float) + symbol_counts = train["symbol"].value_counts() + train_weight = ( + 1.0 / train["symbol"].map(symbol_counts).to_numpy(dtype=float) + ) + train_weight /= train_weight.mean() + validation_x = validation[feature_names] + validation_y = validation["target"].to_numpy(dtype=float) + validation_scale = validation["target_scale"].to_numpy(dtype=float) + results: list[dict[str, Any]] = [] + best: dict[str, Any] | None = None + # Quantile models depend only on the feature stage, not on the point-model + # candidate. Fit them once per stage so candidate comparison stays cheap. + _progress(f"stage {stage}: fitting q20 quantile model") + lower_model = _quantile_model(LOWER_QUANTILE, random_state, n_estimators) + upper_model = _quantile_model(TARGET_QUANTILE, random_state, n_estimators) + lower_model.fit(train_x, train_y) + _progress(f"stage {stage}: q20 quantile model ready") + _progress(f"stage {stage}: fitting q80 quantile model") + upper_model.fit(train_x, train_y) + _progress(f"stage {stage}: quantile models ready") + for spec in _candidate_specs(random_state, n_estimators, n_jobs): + _progress(f"stage {stage}: fitting {spec['name']} point model") + point_model = spec["point_factory"]() + point_model.fit(train_x, train_y, sample_weight=train_weight) + point, lower, upper = predict_quantiles( + point_model, lower_model, upper_model, validation_x, spec["kind"] + ) + metrics = evaluate_predictions(validation_y, point, lower, upper, validation_scale) + selection_score = ( + metrics["pinball_q20"] + + metrics["pinball_q80"] + + 0.20 * (metrics["q20_calibration_gap"] + metrics["q80_calibration_gap"]) + + 0.05 * metrics["mae_point"] + ) + record = {"model": spec["name"], "selection_score": float(selection_score), **metrics} + results.append(record) + _progress( + f"stage {stage}: {spec['name']} validation score={selection_score:.6f}" + ) + if best is None or selection_score < best["selection_score"]: + best = { + "spec": spec, + "point_model": point_model, + "lower_model": lower_model, + "upper_model": upper_model, + "validation_metrics": metrics, + "selection_score": float(selection_score), + } + if best is None: + raise RuntimeError("no model candidate was trained") + best["feature_names"] = feature_names + return best, results + + +def _calibration_offsets(best: dict[str, Any], calibration: pd.DataFrame) -> dict[str, float]: + features = calibration[best["feature_names"]] + y = calibration["target"].to_numpy(dtype=float) + _, lower, upper = predict_quantiles( + best["point_model"], + best["lower_model"], + best["upper_model"], + features, + best["spec"]["kind"], + ) + return { + "q20_offset": float(np.quantile(y - lower, LOWER_QUANTILE)), + "q80_offset": float(np.quantile(y - upper, TARGET_QUANTILE)), + } + + +def apply_calibration( + lower: np.ndarray, + upper: np.ndarray, + offsets: dict[str, float], +) -> tuple[np.ndarray, np.ndarray]: + lower = lower + float(offsets.get("q20_offset", 0.0)) + upper = upper + float(offsets.get("q80_offset", 0.0)) + return np.minimum(lower, upper), np.maximum(lower, upper) + + +def _group_metrics( + dataset: pd.DataFrame, + y_true: np.ndarray, + point: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + column: str, +) -> dict[str, dict[str, float]]: + result: dict[str, dict[str, float]] = {} + for group, group_frame in dataset.reset_index(drop=True).groupby(column, sort=True): + indices = group_frame.index.to_numpy(dtype=np.int64) + result[str(group)] = evaluate_predictions( + y_true[indices], point[indices], lower[indices], upper[indices] + ) + return result + + +def _stratified_metrics( + dataset: pd.DataFrame, + y_true: np.ndarray, + point: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, +) -> dict[str, dict[str, dict[str, float]]]: + frame = dataset.reset_index(drop=True).copy() + frame["month"] = pd.to_datetime(frame["timestamp"], unit="ms", utc=True).dt.strftime("%Y-%m") + result: dict[str, dict[str, dict[str, float]]] = { + "by_symbol": _group_metrics(frame, y_true, point, lower, upper, "symbol"), + "by_month": _group_metrics(frame, y_true, point, lower, upper, "month"), + } + if frame["sample_atr_pct"].nunique() >= 3: + frame["volatility_bucket"] = pd.qcut( + frame["sample_atr_pct"], q=3, labels=["low", "medium", "high"], duplicates="drop" + ).astype(str) + result["by_volatility"] = _group_metrics( + frame, y_true, point, lower, upper, "volatility_bucket" + ) + else: + result["by_volatility"] = _group_metrics( + frame.assign(volatility_bucket="all"), y_true, point, lower, upper, "volatility_bucket" + ) + return result + + +def _block_bootstrap_ci( + dataset: pd.DataFrame, + y_true: np.ndarray, + point: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + seed: int, + repetitions: int = 100, + block_size: int = 18, +) -> dict[str, dict[str, float]]: + frame = dataset.reset_index(drop=True) + groups = [group.index.to_numpy(dtype=np.int64) for _, group in frame.groupby("symbol", sort=False)] + if not groups: + return {} + rng = np.random.default_rng(seed) + values: dict[str, list[float]] = {"pinball_q20": [], "pinball_q80": [], "q20_hit_rate": [], "q80_coverage": []} + for _ in range(repetitions): + sampled: list[int] = [] + for indices in groups: + if len(indices) == 0: + continue + need = len(indices) + local: list[int] = [] + while len(local) < need: + start = int(rng.integers(0, len(indices))) + local.extend(indices[start : start + block_size].tolist()) + sampled.extend(local[:need]) + if not sampled: + continue + metrics = evaluate_predictions( + y_true[np.asarray(sampled)], + point[np.asarray(sampled)], + lower[np.asarray(sampled)], + upper[np.asarray(sampled)], + ) + for name in values: + values[name].append(metrics[name]) + return { + name: { + "lower_95": float(np.quantile(items, 0.025)), + "upper_95": float(np.quantile(items, 0.975)), + } + for name, items in values.items() + if items + } + + +def _json_safe(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, (np.integer, np.floating)): + return value.item() + return value + + +def run_training( + data_dir: Path, + artifact_dir: Path, + end_ms: int | None = None, + test_months: int = 10, + seed: int = 42, + n_estimators: int = 250, + n_jobs: int = -1, + volatility_rank_threshold: float = VOLATILITY_RANK_THRESHOLD, +) -> dict[str, Any]: + if not 0.0 <= volatility_rank_threshold <= 1.0: + raise ValueError("volatility_rank_threshold must be between 0 and 1") + _progress("training: loading manifest and raw 8H frames") + manifest = load_manifest(data_dir) + frames = load_raw_frames(data_dir, manifest) + if len(frames) < 3: + raise ValueError("at least 3 symbols with raw data are required") + end_ms = floor_to_8h(end_ms or int(manifest["end_ms"])) + test_start_ms = months_before(end_ms, test_months) + profile_indicators = {symbol: compute_indicators(frame) for symbol, frame in frames.items()} + raw_profiles = { + symbol: asset_profile(indicators, test_start_ms) + for symbol, indicators in profile_indicators.items() + } + profiles = {symbol: profile for symbol, profile in raw_profiles.items() if profile is not None} + if len(profiles) < 3: + raise ValueError("at least 3 symbols with enough pre-test history are required") + train_symbols, validation_symbols, test_symbols = split_holdout_symbols(profiles, seed) + asset_scaler = fit_asset_scaler(profiles, train_symbols) + validation_end_ms, calibration_end_ms = recent_split_boundaries(test_start_ms, end_ms) + _progress( + f"training: {len(frames)} symbols, {len(profiles)} profiles; " + f"train/validation/test symbols={len(train_symbols)}/{len(validation_symbols)}/{len(test_symbols)}" + ) + _progress( + f"training: ATR rank filter window={VOLATILITY_RANK_WINDOW}, " + f"threshold={volatility_rank_threshold:.2f}" + ) + stage_results: list[dict[str, Any]] = [] + best_overall: dict[str, Any] | None = None + best_dataset: pd.DataFrame | None = None + best_stage = 0 + for stage in (1, 2, 3): + _progress(f"stage {stage}/3: building feature dataset") + dataset, feature_names, _ = build_feature_dataset( + frames, + stage, + test_start_ms, + end_ms, + asset_scaler, + train_symbols, + validation_symbols, + test_symbols, + validation_end_ms, + calibration_end_ms, + volatility_rank_threshold, + ) + filter_stats = dataset.attrs.get("volatility_filter", {}) + _progress( + f"stage {stage}/3: features={len(feature_names)}, rows=" + f"train/validation/calibration/test={int((dataset['split'] == 'train').sum())}/" + f"{int((dataset['split'] == 'validation').sum())}/" + f"{int((dataset['split'] == 'calibration').sum())}/" + f"{int((dataset['split'] == 'test').sum())}; " + f"filter candidates={sum(filter_stats.get('candidate_counts', {}).values())}, " + f"selected={sum(filter_stats.get('selected_counts', {}).values())}" + ) + best, candidate_results = _fit_stage( + dataset, + feature_names, + validation_symbols, + stage, + seed, + n_estimators, + n_jobs, + ) + stage_record = { + "stage": stage, + "feature_count": len(feature_names), + "train_rows": int((dataset["split"] == "train").sum()), + "validation_rows": int((dataset["split"] == "validation").sum()), + "calibration_rows": int((dataset["split"] == "calibration").sum()), + "test_rows": int((dataset["split"] == "test").sum()), + "volatility_filter": filter_stats, + "best_model": best["spec"]["name"], + "best_validation": best["validation_metrics"], + "candidates": candidate_results, + } + stage_results.append(stage_record) + _progress( + f"stage {stage}/3: best={best['spec']['name']}, " + f"validation selection score={best['selection_score']:.6f}" + ) + if best_overall is None or best["selection_score"] < best_overall["selection_score"]: + best_overall = best + best_dataset = dataset + best_stage = stage + if best_overall is None or best_dataset is None: + raise RuntimeError("no stage was trained") + _progress(f"training: selected stage {best_stage}; fitting final calibration/test report") + best_features = best_overall["feature_names"] + validation = best_dataset[ + best_dataset["split"] == "validation" + ] + calibration = best_dataset[best_dataset["split"] == "calibration"] + test = best_dataset[ + best_dataset["split"] == "test" + ] + if calibration.empty or test.empty: + raise ValueError("calibration and final test samples are required") + offsets = _calibration_offsets(best_overall, calibration) + test_x = test[best_features] + test_y = test["target"].to_numpy(dtype=float) + point, lower, upper = predict_quantiles( + best_overall["point_model"], + best_overall["lower_model"], + best_overall["upper_model"], + test_x, + best_overall["spec"]["kind"], + ) + calibrated_lower, calibrated_upper = apply_calibration(lower, upper, offsets) + test_scale = test["target_scale"].to_numpy(dtype=float) + actual_test_y = test_y * test_scale + actual_point = point * test_scale + actual_calibrated_lower = calibrated_lower * test_scale + actual_calibrated_upper = calibrated_upper * test_scale + test_metrics = { + **evaluate_predictions( + test_y, + point, + calibrated_lower, + calibrated_upper, + target_scale=test_scale, + ), + "raw_q80_coverage": float(np.mean(test_y <= upper)), + "raw_q20_hit_rate": float(np.mean(test_y >= lower)), + "calibrated_q80_coverage": float(np.mean(test_y <= calibrated_upper)), + "calibrated_q20_hit_rate": float(np.mean(test_y >= calibrated_lower)), + "raw_quantile_cross_rate": float(np.mean(lower > upper)), + } + test_metrics["stratified"] = _stratified_metrics( + test, + actual_test_y, + actual_point, + actual_calibrated_lower, + actual_calibrated_upper, + ) + test_metrics["bootstrap_95_ci"] = _block_bootstrap_ci( + test, + actual_test_y, + actual_point, + actual_calibrated_lower, + actual_calibrated_upper, + seed, + ) + symbol_prediction = pd.DataFrame( + { + "symbol": test["symbol"].to_numpy(), + "point": actual_point, + "q20": actual_calibrated_lower, + "q80": actual_calibrated_upper, + } + ).groupby("symbol", sort=True).median() + test_metrics["prediction_dispersion_by_symbol"] = { + name: { + "min": float(symbol_prediction[name].min()), + "median": float(symbol_prediction[name].median()), + "max": float(symbol_prediction[name].max()), + "std": float(symbol_prediction[name].std(ddof=0)), + } + for name in ("point", "q20", "q80") + } + _progress( + f"training: final test rows={len(test)}, calibrated q80 coverage=" + f"{test_metrics['calibrated_q80_coverage']:.4f}" + ) + all_target = best_dataset["target"].to_numpy(dtype=float) + all_target_scale = best_dataset["target_scale"].to_numpy(dtype=float) + actual_target = all_target * all_target_scale + target_summary = { + "n": int(len(actual_target)), + "positive_fraction": float(np.mean(actual_target > 0)), + "negative_fraction": float(np.mean(actual_target < 0)), + "zero_fraction": float(np.mean(actual_target == 0)), + "quantiles": { + str(quantile): float(np.quantile(actual_target, quantile)) + for quantile in (0.01, 0.10, 0.50, 0.90, 0.99) + }, + } + symbol_files = { + item["symbol"]: item["path"] + for item in manifest.get("symbols", []) + if item["symbol"] in profiles + } + artifact_dir.mkdir(parents=True, exist_ok=True) + artifact = { + "version": 3, + "timeframe": "8h", + "window": WINDOW, + "horizon": HORIZON, + "target_kind": TARGET_KIND, + "target": "quantile(future_6_low, 0.20) / anchor_close - 1", + "future_low_quantile": FUTURE_LOW_QUANTILE, + "target_scale": "asset_atr_price_top20_median", + "lower_quantile": LOWER_QUANTILE, + "upper_quantile": TARGET_QUANTILE, + "volatility_rank_window": VOLATILITY_RANK_WINDOW, + "volatility_rank_threshold": volatility_rank_threshold, + "stage": best_stage, + "model_name": best_overall["spec"]["name"], + "model_kind": best_overall["spec"]["kind"], + "feature_names": best_features, + "asset_scaler": asset_scaler, + "asset_profiles": profiles, + "symbol_files": symbol_files, + "calibration": offsets, + "point_model": best_overall["point_model"], + "lower_model": best_overall["lower_model"], + "upper_model": best_overall["upper_model"], + "end_ms": end_ms, + "test_start_ms": test_start_ms, + "symbols": sorted(profiles), + } + joblib.dump(artifact, artifact_dir / "model.joblib") + report = { + "protocol": { + "train_anchor_before": test_start_ms, + "recent_window_start": test_start_ms, + "recent_window_end": end_ms, + "validation_end": validation_end_ms, + "calibration_end": calibration_end_ms, + "final_test_start": calibration_end_ms, + "test_months": test_months, + "validation_symbols": validation_symbols, + "test_symbols": test_symbols, + "training_symbols_with_history": train_symbols, + "seed": seed, + "future_bars": HORIZON, + "feature_window_bars": WINDOW, + "target": { + "kind": TARGET_KIND, + "future_low_quantile": FUTURE_LOW_QUANTILE, + "scale": "asset_atr_price_top20_median", + "summary": target_summary, + }, + "volatility_filter": { + "atr_rank_window": VOLATILITY_RANK_WINDOW, + "min_rank": volatility_rank_threshold, + }, + }, + "selected": { + "stage": best_stage, + "model": best_overall["spec"]["name"], + "model_kind": best_overall["spec"]["kind"], + "feature_count": len(best_features), + "validation": best_overall["validation_metrics"], + "calibration_rows": len(calibration), + "calibration": offsets, + }, + "stages": stage_results, + "test": test_metrics, + "feature_names": best_features, + "artifact": str((artifact_dir / "model.joblib").resolve()), + } + report = _json_safe(report) + (artifact_dir / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False)) + (artifact_dir / "report.md").write_text(render_report(report)) + _progress(f"training: artifacts written to {artifact_dir}") + return report + + +def build_prediction_features( + frame: pd.DataFrame, + stage: int, + anchor_index: int, + asset_scaler: dict[str, dict[str, float]], + profile: dict[str, float] | None = None, +) -> pd.DataFrame: + indicators = compute_indicators(frame).reset_index(drop=True) + if profile is None: + profile = asset_profile(indicators, int(indicators.iloc[anchor_index]["timestamp"]) + 1) + if profile is None: + raise ValueError("prediction history is too short to calculate asset profile") + asset = scale_asset_profile(profile, asset_scaler) + if anchor_index < WINDOW - 1: + raise ValueError(f"at least {WINDOW} closed 8H bars are required before anchor") + window = indicators.iloc[anchor_index - WINDOW + 1 : anchor_index + 1] + values = _window_features(window, stage, asset) + return pd.DataFrame([values]) + + +def predict_from_csv( + artifact_path: Path, + csv_path: Path, + at_ms: int | None = None, + symbol: str | None = None, +) -> dict[str, Any]: + artifact = joblib.load(artifact_path) + if ( + artifact.get("version") != 3 + or artifact.get("target_kind") != TARGET_KIND + or artifact.get("target_scale") != "asset_atr_price_top20_median" + ): + raise ValueError( + "artifact uses a legacy target; retrain with the signed robust future-low target" + ) + frame = pd.read_csv(csv_path) + for column in CSV_COLUMNS: + frame[column] = pd.to_numeric(frame[column], errors="coerce") + frame = frame.sort_values("timestamp").drop_duplicates("timestamp", keep="last").reset_index(drop=True) + frame["quote"] = np.where(frame["quote"] > 0, frame["quote"], frame["volume"] * frame["close"]) + if at_ms is None: + anchor_index = len(frame) - 1 + else: + if int(at_ms) < int(artifact.get("test_start_ms", 0)): + raise ValueError("--at is earlier than the model training snapshot") + matches = np.flatnonzero(frame["timestamp"].to_numpy(dtype=np.int64) == int(at_ms)) + if len(matches) == 0: + raise ValueError("--at must match an 8H bar timestamp in the CSV") + anchor_index = int(matches[-1]) + if symbol is None: + for candidate, path in artifact.get("symbol_files", {}).items(): + if Path(path).name == csv_path.name: + symbol = candidate + break + profile = artifact.get("asset_profiles", {}).get(symbol) if symbol else None + if profile is None: + raise ValueError("symbol must match a symbol with a stored training asset profile") + indicators = compute_indicators(frame) + rank_value = float(indicators.iloc[anchor_index]["atr_rank_200"]) + threshold = float(artifact.get("volatility_rank_threshold", VOLATILITY_RANK_THRESHOLD)) + eligible = np.isfinite(rank_value) and rank_value >= threshold + target_scale = max(float(profile["asset_atr_price_top20_median"]), 1e-9) + response = { + "timestamp": int(frame.iloc[anchor_index]["timestamp"]), + "close": float(frame.iloc[anchor_index]["close"]), + "model": artifact["model_name"], + "feature_stage": artifact["stage"], + "symbol": symbol, + "volatility_rank_window": VOLATILITY_RANK_WINDOW, + "volatility_rank_200": rank_value if np.isfinite(rank_value) else None, + "volatility_rank_threshold": threshold, + "future_low_quantile": float(artifact.get("future_low_quantile", FUTURE_LOW_QUANTILE)), + "target_scale": target_scale, + "eligible_for_model": bool(eligible), + } + if not eligible: + response["skip_reason"] = ( + "atr_rank_200_unavailable" + if not np.isfinite(rank_value) + else "atr_rank_200_below_threshold" + ) + response.update( + { + "drawdown_point": None, + "drawdown_hit80_q20": None, + "drawdown_upper80_q80": None, + "price_level_hit80": None, + "price_level_upper80": None, + "future_low_return_point": None, + "future_low_return_q20": None, + "future_low_return_q80": None, + "future_low_reference_return_point": None, + "future_low_reference_return_q20": None, + "future_low_reference_return_q80": None, + "future_low_price_q20": None, + "future_low_price_q80": None, + "future_low_reference_price_q20": None, + "future_low_reference_price_q80": None, + } + ) + return response + features = build_prediction_features( + frame, + artifact["stage"], + anchor_index, + artifact["asset_scaler"], + profile, + ) + point, lower, upper = predict_quantiles( + artifact["point_model"], + artifact["lower_model"], + artifact["upper_model"], + features, + artifact["model_kind"], + ) + lower, upper = apply_calibration(lower, upper, artifact.get("calibration", {})) + signed_point = float(point[0]) * target_scale + signed_q20 = float(lower[0]) * target_scale + signed_q80 = float(upper[0]) * target_scale + close = response["close"] + response.update( + { + "future_low_return_point": signed_point, + "future_low_return_q20": signed_q20, + "future_low_return_q80": signed_q80, + "future_low_reference_return_point": signed_point, + "future_low_reference_return_q20": signed_q20, + "future_low_reference_return_q80": signed_q80, + "future_low_price_q20": close * (1 + signed_q20), + "future_low_price_q80": close * (1 + signed_q80), + "future_low_reference_price_q20": close * (1 + signed_q20), + "future_low_reference_price_q80": close * (1 + signed_q80), + "drawdown_point": -signed_point, + "drawdown_hit80_q20": -signed_q80, + "drawdown_upper80_q80": -signed_q20, + "price_level_hit80": close * (1 + signed_q80), + "price_level_upper80": close * (1 + signed_q20), + } + ) + return response + + +def render_report(report: dict[str, Any]) -> str: + selected = report["selected"] + test = report["test"] + volatility_filter = report["protocol"]["volatility_filter"] + lines = [ + "# cbpred report", + "", + f"- selected stage: {selected['stage']}", + f"- selected model: `{selected['model']}`", + f"- feature count: {selected['feature_count']}", + "- target: signed 20th-percentile future-low return, normalized by each asset's training ATR/price scale", + f"- target positive/negative fraction: {report['protocol']['target']['summary']['positive_fraction']:.4f}/" + f"{report['protocol']['target']['summary']['negative_fraction']:.4f}", + f"- volatility filter: ATR rank over {volatility_filter['atr_rank_window']} bars >= " + f"{volatility_filter['min_rank']:.2f}", + f"- validation q80 coverage: {selected['validation']['q80_coverage']:.4f}", + f"- test calibrated q20 pinball: {test['pinball_q20']:.6f}", + f"- test calibrated q80 pinball: {test['pinball_q80']:.6f}", + f"- test calibrated q80 coverage: {test['calibrated_q80_coverage']:.4f}", + f"- test calibrated q20 lower-bound survival rate: {test['calibrated_q20_hit_rate']:.4f}", + f"- test point MAE: {test['mae_point']:.6f}", + f"- test point RMSE: {test['rmse_point']:.6f}", + f"- test q80 pinball loss: {test['pinball_q80']:.6f}", + f"- test per-symbol median point return range: " + f"{test['prediction_dispersion_by_symbol']['point']['min']:.6f} - " + f"{test['prediction_dispersion_by_symbol']['point']['max']:.6f}", + "", + "The target is quantile(future_6_low, 0.20) / anchor_close - 1. Negative values are drawdowns; " + "positive values mean the robust low reference stayed above the anchor close. " + "q20 is the lower future-low return quantile and therefore the conservative lower price level; " + "q80 is the shallower, more optimistic level. Coverage is calibrated on the held-out test protocol.", + "", + "## Three feature iterations", + "", + ] + for item in report["stages"]: + best = item["best_validation"] + lines.append( + f"- stage {item['stage']}: {item['feature_count']} features, " + f"best `{item['best_model']}`, validation q20/q80 pinball=" + f"{best['pinball_q20']:.6f}/{best['pinball_q80']:.6f}, " + f"q80 coverage={best['q80_coverage']:.4f}" + ) + ci = test.get("bootstrap_95_ci", {}) + if ci: + lines.extend(["", "## Final test block-bootstrap 95% intervals", ""]) + for name, interval in ci.items(): + label = ( + "q20 lower-bound survival rate" + if name == "q20_hit_rate" + else name + ) + lines.append(f"- {label}: {interval['lower_95']:.6f} - {interval['upper_95']:.6f}") + lines.extend(["", "## Feature names", "", "```text", *report["feature_names"], "```", ""]) + return "\n".join(lines) diff --git a/cmd/cbpred-export/main.go b/cmd/cbpred-export/main.go new file mode 100644 index 0000000..91a5a57 --- /dev/null +++ b/cmd/cbpred-export/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/banbox/banbot/biz" + "github.com/banbox/banbot/config" + "github.com/banbox/banbot/core" + "github.com/banbox/banbot/orm" +) + +var columns = []string{ + "timestamp", "open", "high", "low", "close", "volume", "quote", "buy_volume", "trade_num", +} + +func main() { + configPath := flag.String("config", "", "banbot config path") + dataDir := flag.String("data-dir", "", "banbot data directory") + pairs := flag.String("pairs", "", "comma-separated symbols") + startMS := flag.Int64("start-ms", 0, "inclusive start timestamp in milliseconds") + endMS := flag.Int64("end-ms", 0, "exclusive end timestamp in milliseconds") + outDir := flag.String("out-dir", "", "CSV output directory") + flag.Parse() + if *configPath == "" || *dataDir == "" || *pairs == "" || *outDir == "" || *startMS <= 0 || *endMS <= *startMS { + flag.Usage() + os.Exit(2) + } + + args := &config.CmdArgs{ + Configs: config.ArrString{*configPath}, + DataDir: *dataDir, + NoDefault: true, + RawPairs: *pairs, + TimeStart: strconv.FormatInt(*startMS, 10), + TimeEnd: strconv.FormatInt(*endMS, 10), + LogLevel: "error", + } + if err := biz.SetupComs(args); err != nil { + fail(err) + } + defer core.RunExitCalls() + if err := os.MkdirAll(*outDir, 0o755); err != nil { + fail(err) + } + + for _, rawPair := range strings.Split(*pairs, ",") { + pair := strings.TrimSpace(rawPair) + if pair == "" { + continue + } + exs, err := orm.GetExSymbolCur(pair) + if err != nil { + fail(err) + } + _, rows, err := orm.GetSeries(exs, "8h", *startMS, *endMS, 0, false) + if err != nil { + fail(err) + } + name := fmt.Sprintf("%d_%s.csv", exs.ID, strings.NewReplacer("/", "_", ":", "_").Replace(pair)) + if err := writeCSV(filepath.Join(*outDir, name), rows, exs); err != nil { + fail(err) + } + } +} + +func writeCSV(path string, rows []*orm.DataSeries, exs *orm.ExSymbol) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + writer := csv.NewWriter(file) + if err := writer.Write(columns); err != nil { + return err + } + for _, row := range rows { + bar, err := row.OHLCV(exs) + if err != nil { + return err + } + values := []string{ + strconv.FormatInt(bar.Time, 10), + strconv.FormatFloat(bar.Open, 'f', -1, 64), + strconv.FormatFloat(bar.High, 'f', -1, 64), + strconv.FormatFloat(bar.Low, 'f', -1, 64), + strconv.FormatFloat(bar.Close, 'f', -1, 64), + strconv.FormatFloat(bar.Volume, 'f', -1, 64), + strconv.FormatFloat(bar.Quote, 'f', -1, 64), + strconv.FormatFloat(bar.BuyVolume, 'f', -1, 64), + strconv.FormatInt(bar.TradeNum, 10), + } + if err := writer.Write(values); err != nil { + return err + } + } + writer.Flush() + return writer.Error() +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/docs/abstract.md b/docs/abstract.md new file mode 100644 index 0000000..5eecf99 --- /dev/null +++ b/docs/abstract.md @@ -0,0 +1,58 @@ + + +我现在有很多品种的K线数据，需要你帮我训练一个机器学习模型，基于8H级别的K线 + +提取特征，输入最近18个K线（提取bar特征，比如阴线阳线，实体占比，实体的中心位置）， + +预测未来6个K线的最大回调幅度（相比预测时最新收盘价），有80%概率命中的位置最佳，或者最大回调幅度的大概80%位置。 + +应该用统一的模型预测所有品种，每个品种的还要提取一些特异信息，比如： + +1. 最近18个K线的成交量排名（统一基于法币） + +2. 训练数据中所有ATR/价格，降序，前20%的中位数 + +保留最新10个月的数据作为测试数据，使用此前的数据作为训练数据 + +核心目标是只使用K线数据，提取特征，结合品种独有特点，训练统一的模型，预测给定品种在任意时刻后续的回调幅度。 + +特征的选择上面只是一些示例，你应该根据K线特点，考虑还有哪些特征可以构造，哪些比较重要 + +比如或许可以结合一些指标，比如布林带之类的，但是注意使用指标提取特征时务必注意要进行归一化，确保不同品种的特征可比。 + +训练数据的某些特征是可以事先提取的，比如上面说到品种的所有训练数据的ATR/价格中位数，这样的特征应该是属于品种独有的特征，每个特征只提取一两个值表示即可，不应该采样过于复杂的数据避免过拟合。 + +然后在训练模型时，优先使用随机森林算法，也可以对比其他模型算法； + +测试数据按品种随机划分为两部分，一部分作为验证集，一部分作为测试集。 + +我们的K线数据是通过 /home/ctyun/ban/banbot 项目下载维护到postgresql中的 + +应该只关注U本位合约中的USDT的品种，且上市至少2年； + +先通过banbot获取所有符合的品种，下载它的8H的K线确保数据库中完整。 + +然后基于品种导出数据，方便python进行处理分析。 + +然后考虑如何构造特征，如何组织和处理数据作为训练友好的数据格式 + +然后创建多种机器学习模型，进行模型训练，使用验证集进行验证调优 + +最后统一使用测试集测试，给出最佳的模型 + +然后使用最佳模型，再次不断反思特征如何优化，考虑banbot支持的指标，重新处理数据重新训练 + +这个特征重新处理并训练的过程要重复3次； + +最后给我输出最佳特征提取方法，以及模型的测试结果。 + +模型预测的未来6个K线的最大回调幅度，应该使用未来6个K线的lowestLow，这里可以尝试绝对误差和平方误差，或者其他你认为有必要常识的误差计算方法。 + + + +首先是，模型训练时必须输出进度，不能一直无限等待； + +另外，其实大部分品种，它只是在少部分波动很大的时候才会有大幅回撤，这时候模型的输出才有意义，但波动大的只占K线的或许20%或者10%以内；训练数据中应该剔除波动太小的时候，这时候没有意义，不需要关注；我想到的一个方法是，可以用ATR在过去200个K线中的排名来度量波动大小，排名不够前的直接忽略，你实际找十来个品种的数据检测观察下，看看取前10%还是20%或者其他比例；然后对所有的训练数据处理后再继续。 + +实际上不同品种的回撤幅度差异很大，而且和他们所处的阶段有关系；在刚开始上升的时候，多半未来6个k线的最低点依然高于当前的收盘价，但这时候也不应该直接输出0，而是应该根据过往该品种的特征，输出一个符合当下的大概可能的回撤，这个应该作为真实值，这样计算的损失和误差就不会特别大，导致过低估计回撤了；或许你也可以考虑其他方法。总之不应该完全用未来6个k线的最低值作为回撤，而是应该用更有意义的值 + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..bdcf7e3 --- /dev/null +++ b/go.mod @@ -0,0 +1,112 @@ +module cbpred-export + +go 1.24.0 + +require github.com/banbox/banbot v0.0.0 + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/banbox/banexg v0.2.60 // indirect + github.com/banbox/banta v0.3.7 // indirect + github.com/banbox/bntp v0.1.0 // indirect + github.com/banbox/cron/v3 v3.1.2 // indirect + github.com/beevik/ntp v1.4.3 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cloudwego/eino v0.6.0 // indirect + github.com/cloudwego/eino-ext/components/model/openai v0.1.1 // indirect + github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250918130948-16e3a249e721 // indirect + github.com/dgraph-io/ristretto v0.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.9.0 // indirect + github.com/eino-contrib/jsonschema v1.0.2 // indirect + github.com/evanphx/json-patch v0.5.2 // indirect + github.com/felixge/fgprof v0.9.5 // indirect + github.com/flopp/go-findfont v0.1.0 // indirect + github.com/fogleman/gg v1.3.0 // indirect + github.com/getkin/kin-openapi v0.118.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/go-telegram/bot v1.17.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/yaml v0.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/meguminnnnnnnnn/go-openai v0.0.0-20250821095446-07791bea23a0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/perimeterx/marshmallow v1.1.4 // indirect + github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sasha-s/go-deadlock v0.3.6 // indirect + github.com/schollz/progressbar/v3 v3.18.0 // indirect + github.com/shirou/gopsutil/v4 v4.25.9 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/tiendc/go-deepcopy v1.7.1 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/excelize/v2 v2.10.0 // indirect + github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect + golang.org/x/image v0.32.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + gonum.org/v1/gonum v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/grpc v1.76.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.39.1 // indirect +) + +replace github.com/banbox/banbot => /home/ctyun/ban/banbot diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..719b220 --- /dev/null +++ b/go.sum @@ -0,0 +1,379 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/banbox/banexg v0.2.60 h1:0IQaSATKwf6SNLnehGmjdAidnYJsRqhJqruUfh9U8LM= +github.com/banbox/banexg v0.2.60/go.mod h1:T9JWRGblxAibQjwKK6VABG+r87y1LRnyYe7sz5aBnR4= +github.com/banbox/banta v0.3.7 h1:R1xVVtds0Temlc1fATX08uG7rR0pbukr3L6FikyKscE= +github.com/banbox/banta v0.3.7/go.mod h1:43XoXngLPip/pX1OvohnNUScScqb1/jtKxVqgltwZUU= +github.com/banbox/bntp v0.1.0 h1:kZROpbb3OME8XM8QeM+2SeQWQrHRDFN05okr8OITKAU= +github.com/banbox/bntp v0.1.0/go.mod h1:w0WP7GROKQo07wHFBm/FVgH4xlZuOYCoro0ky4iTC3M= +github.com/banbox/cron/v3 v3.1.2 h1:gBNnz/OBxIMU7iBCtObZKhb85rzbaXr8AZLZb40CgeM= +github.com/banbox/cron/v3 v3.1.2/go.mod h1:SZzzT03qsZ1y8/Pt2VqKb9Av/HfnuSargOQRmql7n18= +github.com/beevik/ntp v1.4.3 h1:PlbTvE5NNy4QHmA4Mg57n7mcFTmr1W1j3gcK7L1lqho= +github.com/beevik/ntp v1.4.3/go.mod h1:Unr8Zg+2dRn7d8bHFuehIMSvvUYssHMxW3Q5Nx4RW5Q= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8= +github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.6.0 h1:pobGKMOfcQHVNhD9UT/HrvO0eYG6FC2ML/NKY2Eb9+Q= +github.com/cloudwego/eino v0.6.0/go.mod h1:JNapfU+QUrFFpboNDrNOFvmz0m9wjBFHHCr77RH6a50= +github.com/cloudwego/eino-ext/components/model/openai v0.1.1 h1:VRdUDcnfi/T8F0jcuovhdADU9Io/oMqiKpY2ZJTBc1o= +github.com/cloudwego/eino-ext/components/model/openai v0.1.1/go.mod h1:VwAXEY1ik2K9KFPZvymnkfBQQKgLHbpg90yg+7hrTt8= +github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250918130948-16e3a249e721 h1:5Hd8GxNEmu+ppTGCRBU6kLKfCQNXPMwi31xA83PzEqo= +github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250918130948-16e3a249e721/go.mod h1:fHn/6OqPPY1iLLx9wzz+MEVT5Dl9gwuZte1oLEnCoYw= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A4k= +github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/eino-contrib/jsonschema v1.0.2 h1:HaxruBMUdnXa7Lg/lX8g0Hk71ZIfdTZXmBQz0e3esr8= +github.com/eino-contrib/jsonschema v1.0.2/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= +github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= +github.com/flopp/go-findfont v0.1.0 h1:lPn0BymDUtJo+ZkV01VS3661HL6F4qFlkhcJN55u6mU= +github.com/flopp/go-findfont v0.1.0/go.mod h1:wKKxRDjD024Rh7VMwoU90i6ikQRCr+JTHB5n4Ejkqvw= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM= +github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-telegram/bot v1.17.0 h1:Hs0kGxSj97QFqOQP0zxduY/4tSx8QDzvNI9uVRS+zmY= +github.com/go-telegram/bot v1.17.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= +github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= +github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/meguminnnnnnnnn/go-openai v0.0.0-20250821095446-07791bea23a0 h1:nIohpHs1ViKR0SVgW/cbBstHjmnqFZDM9RqgX9m9Xu8= +github.com/meguminnnnnnnnn/go-openai v0.0.0-20250821095446-07791bea23a0/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= +github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe h1:vHpqOnPlnkba8iSxU4j/CvDSS9J4+F4473esQsYLGoE= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sasha-s/go-deadlock v0.3.6 h1:TR7sfOnZ7x00tWPfD397Peodt57KzMDo+9Ae9rMiUmw= +github.com/sasha-s/go-deadlock v0.3.6/go.mod h1:CUqNyyvMxTyjFqDT7MRg9mb4Dv/btmGTqSR+rky/UXo= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/shirou/gopsutil/v4 v4.25.9 h1:JImNpf6gCVhKgZhtaAHJ0serfFGtlfIlSC08eaKdTrU= +github.com/shirou/gopsutil/v4 v4.25.9/go.mod h1:gxIxoC+7nQRwUl/xNhutXlD8lq+jxTgpIkEf3rADHL8= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tiendc/go-deepcopy v1.7.1 h1:LnubftI6nYaaMOcaz0LphzwraqN8jiWTwm416sitff4= +github.com/tiendc/go-deepcopy v1.7.1/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.10.0 h1:8aKsP7JD39iKLc6dH5Tw3dgV3sPRh8uRVXu/fMstfW4= +github.com/xuri/excelize/v2 v2.10.0/go.mod h1:SC5TzhQkaOsTWpANfm+7bJCldzcnU/jrhqkTi/iBHBU= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ= +golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.39.1 h1:H+/wGFzuSCIEVCvXYVHX5RQglwhMOvtHSv+VtidL2r4= +modernc.org/sqlite v1.39.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..179230c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "cbpred" +version = "0.1.0" +description = "Unified 8H crypto futures drawdown prediction pipeline" +requires-python = ">=3.10" +dependencies = [ + "joblib>=1.3", + "numpy>=1.24", + "pandas>=2.0", + "psycopg[binary]>=3.1", + "PyYAML>=6.0", + "scikit-learn>=1.3", +] + +[project.scripts] +cbpred = "cbpred.__main__:main" + +[tool.setuptools.packages.find] +include = ["cbpred*"] + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..2f38c58 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd + +from cbpred.pipeline import ( + CSV_COLUMNS, + EIGHT_H_MS, + aggregate_8h, + build_feature_dataset, + compute_indicators, + fit_asset_scaler, + _vectorized_window_features, + _window_feature_names, + _window_features, + run_training, +) + + +def make_hour_rows(start_ms: int, hours: int, phase: float = 0.0): + rows = [] + price = 100.0 + phase + for index in range(hours): + timestamp = start_ms + index * 60 * 60 * 1000 + drift = 0.0015 * np.sin(index / 17 + phase) + open_ = price + close = price * (1 + drift) + high = max(open_, close) * 1.003 + low = min(open_, close) * 0.997 + rows.append((timestamp, open_, high, low, close, 1000 + index, 100000 + index * 10, 500, 10)) + price = close + return rows + + +class PipelineTests(unittest.TestCase): + def test_aggregate_8h_drops_incomplete_buckets(self): + rows = make_hour_rows(0, 17) + frame = aggregate_8h(rows) + self.assertEqual(len(frame), 2) + self.assertEqual(frame.iloc[0]["timestamp"], 0) + self.assertAlmostEqual(frame.iloc[0]["quote"], sum(row[6] for row in rows[:8])) + + def test_future_target_changes_but_past_features_do_not(self): + start = 0 + raw = aggregate_8h(make_hour_rows(start, 8 * 70)) + indicators = compute_indicators(raw) + profiles = {"x": {"asset_atr_price_top20_median": 0.01, "asset_quote_volume_top20_median_log": 12.0}} + scaler = fit_asset_scaler(profiles) + one, names, _ = build_feature_dataset( + {"x": raw}, + 1, + int(raw.iloc[55]["timestamp"]), + int(raw.iloc[-1]["timestamp"] + EIGHT_H_MS), + scaler, + volatility_rank_threshold=0.0, + ) + changed = raw.copy() + changed.loc[62:, "low"] *= 0.5 + two, _, _ = build_feature_dataset( + {"x": changed}, + 1, + int(raw.iloc[55]["timestamp"]), + int(raw.iloc[-1]["timestamp"] + EIGHT_H_MS), + scaler, + volatility_rank_threshold=0.0, + ) + anchor = int(raw.iloc[60]["timestamp"]) + first = one[one["timestamp"] == anchor].iloc[0] + second = two[two["timestamp"] == anchor].iloc[0] + self.assertEqual(len(names), 18 * 10 + 2) + self.assertGreater(first["target_scale"], 0.0) + for name in names: + if name != "target": + self.assertEqual(first[name], second[name]) + self.assertNotEqual(first["target"], second["target"]) + + rising = raw.copy() + anchor_close = float(rising.iloc[60]["close"]) + for index in range(61, 67): + open_ = anchor_close * (1.01 + 0.002 * (index - 61)) + close = open_ * 1.002 + rising.loc[index, "open"] = open_ + rising.loc[index, "low"] = open_ * 0.999 + rising.loc[index, "high"] = close * 1.001 + rising.loc[index, "close"] = close + rising_dataset, _, _ = build_feature_dataset( + {"x": rising}, + 1, + int(raw.iloc[55]["timestamp"]), + int(raw.iloc[-1]["timestamp"] + EIGHT_H_MS), + scaler, + volatility_rank_threshold=0.0, + ) + rising_row = rising_dataset[rising_dataset["timestamp"] == anchor].iloc[0] + self.assertGreater(rising_row["target"], 0.0) + expected_target = ( + np.quantile(rising.loc[61:66, "low"].to_numpy(dtype=float), 0.20) + / anchor_close + - 1.0 + ) / rising_row["target_scale"] + self.assertAlmostEqual(rising_row["target"], expected_target) + + def test_vectorized_features_match_window_features(self): + raw = aggregate_8h(make_hour_rows(0, 8 * 80)) + indicators = compute_indicators(raw) + asset = { + "asset_atr_price_top20_median": 0.01, + "asset_quote_volume_top20_median_log": 12.0, + } + for stage in (1, 2, 3): + names = _window_feature_names(stage) + for anchor in (18, 30, 55): + window = indicators.iloc[anchor - 17 : anchor + 1] + expected = _window_features(window, stage, asset) + actual = _vectorized_window_features( + indicators, stage, np.array([anchor]), asset + )[0] + np.testing.assert_allclose( + actual, + np.array([expected[name] for name in names]), + rtol=1e-12, + atol=1e-12, + equal_nan=True, + ) + + def test_three_stage_training_writes_report_and_model(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + data_dir = root / "data" + raw_dir = data_dir / "raw" + raw_dir.mkdir(parents=True) + symbols = [] + end_ms = 0 + for idx in range(4): + symbol = f"COIN{idx}/USDT:USDT" + frame = aggregate_8h(make_hour_rows(0, 8 * 400, idx * 0.1)) + path = raw_dir / f"{idx}.csv" + frame.to_csv(path, index=False) + end_ms = max(end_ms, int(frame.iloc[-1]["timestamp"] + EIGHT_H_MS)) + symbols.append({"sid": idx, "symbol": symbol, "path": str(path.relative_to(data_dir)), "bars": len(frame)}) + (data_dir / "manifest.json").write_text( + json.dumps({"end_ms": end_ms, "symbols": symbols}) + ) + report = run_training( + data_dir, + root / "artifacts", + end_ms=end_ms, + test_months=1, + n_estimators=4, + n_jobs=1, + ) + self.assertIn(report["selected"]["stage"], {1, 2, 3}) + self.assertEqual(len(report["stages"]), 3) + self.assertTrue((root / "artifacts" / "model.joblib").exists()) + self.assertTrue((root / "artifacts" / "report.md").exists()) + self.assertGreaterEqual(report["test"]["calibrated_q80_coverage"], 0.0) + self.assertEqual(report["protocol"]["volatility_filter"]["atr_rank_window"], 200) + self.assertAlmostEqual(report["protocol"]["volatility_filter"]["min_rank"], 0.0) + + +if __name__ == "__main__": + unittest.main()