banbox/cbpred
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新脚本

Browse Source
This commit is contained in:
anyongjin
2026-09-17 22:02:51 +08:00
parent 9e1b5e251a
commit 79b02d1c7d
8 changed files with 213 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
cbpred/latest_predict.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse
import csv
import fcntl
import json
import socket
import subprocess
@@ -53,6 +54,47 @@ def default_output_dir(timeframe: str) -> Path:
return REPO_ROOT / f"predictions/latest_v1_{timeframe}"
def default_data_dir(timeframe: str) -> Path:
timeframe_step_ms(timeframe)
return REPO_ROOT / ("data" if timeframe == "8h" else "data_1d")
def default_artifact(timeframe: str) -> Path:
timeframe_step_ms(timeframe)
return REPO_ROOT / (
"artifacts/v1/model.joblib" if timeframe == "8h" else "artifacts_1d/best/model.joblib"
)
def _validate_artifact_timeframe(artifact: Path, timeframe: str) -> None:
artifact_timeframe = str(joblib.load(artifact).get("timeframe", "8h"))
if artifact_timeframe != timeframe:
raise ValueError(
f"artifact {artifact} is for {artifact_timeframe}, not requested timeframe {timeframe}"
)
def _validate_freshness(
manifest: dict[str, object], prediction_items: list[dict[str, object]], data_dir: Path, timeframe: str
) -> None:
end_ms = manifest.get("end_ms")
if end_ms is None:
return
expected = int(end_ms) - timeframe_step_ms(timeframe)
stale: list[str] = []
for item in prediction_items:
frame = pd.read_csv(data_dir / str(item["path"]), usecols=["timestamp"])
latest = int(pd.to_numeric(frame["timestamp"], errors="coerce").max())
if latest != expected:
stale.append(f"{item['symbol']}={_utc_label(latest)}")
if stale:
sample = ", ".join(stale[:5])
raise RuntimeError(
f"incomplete {timeframe} data: expected latest bar {_utc_label(expected)}; "
f"{len(stale)}/{len(prediction_items)} prediction symbols differ ({sample})"
)
def ensure_database(
banbot_config: Path,
dsn: str | None,
@@ -98,6 +140,11 @@ def build_predictions(data_dir: Path, artifact: Path, output_dir: Path) -> dict[
manifest = json.loads((data_dir / "manifest.json").read_text())
artifact_data = joblib.load(artifact)
timeframe = str(manifest.get("timeframe", artifact_data.get("timeframe", "8h")))
artifact_timeframe = str(artifact_data.get("timeframe", "8h"))
if artifact_timeframe != timeframe:
raise ValueError(
f"artifact {artifact} is for {artifact_timeframe}, not exported timeframe {timeframe}"
)
output_columns = output_columns_for_timeframe(timeframe)
volume_column = output_columns[4]
trained_symbols = set(artifact_data.get("asset_profiles", {}))
@@ -113,6 +160,7 @@ def build_predictions(data_dir: Path, artifact: Path, output_dir: Path) -> dict[
else:
skipped_symbols = []
prediction_items = manifest["symbols"]
_validate_freshness(manifest, prediction_items, data_dir, timeframe)
if skipped_symbols:
print(
f"[latest-predict] skipped {len(skipped_symbols)} symbols without stored training profiles",
@@ -180,8 +228,8 @@ def build_parser() -> argparse.ArgumentParser:
type=Path,
default=Path("/data/ban/banbot/biz/config.yml"),
)
parser.add_argument("--data-dir", type=Path, default=REPO_ROOT / "data")
parser.add_argument("--artifact", type=Path, default=REPO_ROOT / "artifacts/v1/model.joblib")
parser.add_argument("--data-dir", type=Path, default=None)
parser.add_argument("--artifact", type=Path, default=None)
parser.add_argument(
"--output-dir",
type=Path,
@@ -202,32 +250,36 @@ def build_parser() -> argparse.ArgumentParser:
def main() -> None:
args = build_parser().parse_args()
data_dir = args.data_dir.resolve()
artifact = args.artifact.resolve()
data_dir = (args.data_dir or default_data_dir(args.timeframe)).resolve()
artifact = (args.artifact or default_artifact(args.timeframe)).resolve()
output_dir = (
args.output_dir if args.output_dir is not None else default_output_dir(args.timeframe)
).resolve()
ensure_database(args.banbot_config, args.dsn)
manifest = prepare_data(
out_dir=data_dir,
banbot_dir=args.banbot_dir,
banbot_config=args.banbot_config,
dsn=args.dsn,
db_type=args.db_type,
exchange=args.exchange,
end_ms=parse_time_ms(args.end, args.timeframe) if args.end else None,
history_months=args.history_months,
holdout_months=args.holdout_months,
min_age_days=args.min_age_days,
chunk_size=args.chunk_size,
timeframe=args.timeframe,
)
print(
f"[latest-predict] exported {len(manifest['symbols'])} symbols; "
f"excluded {len(manifest['excluded'])}",
flush=True,
)
summary = build_predictions(data_dir, artifact, output_dir)
_validate_artifact_timeframe(artifact, args.timeframe)
lock_path = args.banbot_config.parent / ".cbpred-latest-predict.lock"
with lock_path.open("a+") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
ensure_database(args.banbot_config, args.dsn)
manifest = prepare_data(
out_dir=data_dir,
banbot_dir=args.banbot_dir,
banbot_config=args.banbot_config,
dsn=args.dsn,
db_type=args.db_type,
exchange=args.exchange,
end_ms=parse_time_ms(args.end, args.timeframe) if args.end else None,
history_months=args.history_months,
holdout_months=args.holdout_months,
min_age_days=args.min_age_days,
chunk_size=args.chunk_size,
timeframe=args.timeframe,
)
print(
f"[latest-predict] exported {len(manifest['symbols'])} symbols; "
f"excluded {len(manifest['excluded'])}",
flush=True,
)
summary = build_predictions(data_dir, artifact, output_dir)
print(json.dumps(summary, indent=2, ensure_ascii=False))

70
cbpred/pipeline.py
View File

@@ -4,6 +4,7 @@ import calendar
import json
import math
import os
import re
import shlex
import subprocess
import sys
@@ -50,6 +51,8 @@ CSV_COLUMNS = [
"buy_volume",
"trade_num",
]
INVALID_SYMBOL_RE = re.compile(r"\[InvalidSymbol\]\s+symbol\s+(\S+)\s+not found")
LINEAR_DOWNLOAD_CONFIG = "market_type: linear\npairlists: []\npairmgr:\n force_filters: false\n"
def _progress(message: str) -> None:
@@ -217,33 +220,53 @@ def run_banbot_download(
end_ms: int,
chunk_size: int = 100,
banbot_cmd: str = "go run .",
) -> None:
) -> tuple[list[Symbol], list[Symbol]]:
if not symbols:
raise ValueError("no eligible symbols to download")
base = shlex.split(banbot_cmd)
downloaded: list[Symbol] = []
unavailable: list[Symbol] = []
for offset in range(0, len(symbols), chunk_size):
chunk = symbols[offset : offset + chunk_size]
pairs = ",".join(item.symbol for item in chunk)
command = base + [
"kline",
"down",
"--config",
str(config_path),
"--datadir",
str(config_path.parent),
"--no-default",
"--config-data",
"pairlists: []\npairmgr:\n force_filters: false\n",
"--pairs",
pairs,
"--timeframes",
"1h",
"--timestart",
str(history_start_ms),
"--timeend",
str(end_ms),
]
subprocess.run(command, cwd=banbot_dir, check=True)
while chunk:
pairs = ",".join(item.symbol for item in chunk)
command = base + [
"kline",
"down",
"--config",
str(config_path),
"--datadir",
str(config_path.parent),
"--no-default",
"--config-data",
LINEAR_DOWNLOAD_CONFIG,
"--pairs",
pairs,
"--timeframes",
"1h",
"--timestart",
str(history_start_ms),
"--timeend",
str(end_ms),
]
try:
subprocess.run(command, cwd=banbot_dir, check=True, stderr=subprocess.PIPE, text=True)
except subprocess.CalledProcessError as exc:
invalid_symbols = set(INVALID_SYMBOL_RE.findall(exc.stderr or ""))
rejected = [item for item in chunk if item.symbol in invalid_symbols]
if not rejected:
if exc.stderr:
sys.stderr.write(exc.stderr)
raise
_progress(
"skipping unavailable symbols: " + ", ".join(item.symbol for item in rejected)
)
unavailable.extend(rejected)
chunk = [item for item in chunk if item not in rejected]
else:
downloaded.extend(chunk)
break
return downloaded, unavailable
def run_banbot_export(
@@ -369,7 +392,7 @@ def prepare_data(
finally:
conn.close()
if not skip_download:
run_banbot_download(
symbols, unavailable = run_banbot_download(
symbols,
banbot_dir,
banbot_config,
@@ -378,6 +401,7 @@ def prepare_data(
chunk_size,
banbot_cmd,
)
excluded.extend({**symbol.__dict__, "reason": "unavailable_on_exchange"} for symbol in unavailable)
run_banbot_export(
symbols,
banbot_dir,