update

cbpred/latest_predict.py

@@ -2,19 +2,34 @@ from __future__ import annotations
 
 import argparse
 import csv
-import fcntl
+import errno
 import json
+import os
 import socket
 import subprocess
 import tempfile
+import time
 from datetime import datetime, timezone
 from pathlib import Path
-from urllib.parse import urlparse
+
+if os.name == "nt":
+    import msvcrt
+else:
+    import fcntl
 
 import joblib
 import pandas as pd
+import yaml
+from psycopg.conninfo import conninfo_to_dict
 
-from .pipeline import load_database_url, parse_time_ms, predict_from_csv, prepare_data, timeframe_step_ms
+from .pipeline import (
+    effective_banbot_config,
+    load_database_url,
+    parse_time_ms,
+    predict_from_csv,
+    prepare_data,
+    timeframe_step_ms,
+)
 
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
@@ -32,6 +47,22 @@ OUTPUT_COLUMNS = (
 )
 
 
+def _acquire_update_lock(lock) -> None:
+    if os.name != "nt":
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        return
+    # Windows locks a byte range; every caller locks the first byte.
+    while True:
+        lock.seek(0)
+        try:
+            msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
+            return
+        except OSError as exc:
+            if exc.errno not in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
+                raise
+            time.sleep(0.1)
+
+
 def _utc_label(timestamp_ms: int) -> str:
     return datetime.fromtimestamp(timestamp_ms / 1000, timezone.utc).strftime(
         "%Y-%m-%d %H:%M:%S UTC"
@@ -39,8 +70,35 @@ def _utc_label(timestamp_ms: int) -> str:
 
 
 def _database_endpoint(dsn: str) -> tuple[str, int]:
-    parsed = urlparse(dsn)
-    return parsed.hostname or "127.0.0.1", parsed.port or 8812
+    parameters = conninfo_to_dict(dsn)
+    host = parameters.get("host") or parameters.get("hostaddr") or "127.0.0.1"
+    port = parameters.get("port") or "5432"
+    return host.split(",")[0], int(port.split(",")[0] or 5432)
+
+
+def default_banbot_dir() -> Path:
+    configured = os.environ.get("CBPRED_BANBOT_DIR")
+    if configured:
+        return Path(configured)
+    candidates = (
+        REPO_ROOT.parent.parent / "ban/banbot",
+        REPO_ROOT.parent / "banbot",
+        Path("/data/ban/banbot"),
+    )
+    return next((path for path in candidates if path.is_dir()), candidates[-1])
+
+
+def default_banbot_config(banbot_dir: Path) -> Path:
+    configured = os.environ.get("CBPRED_BANBOT_CONFIG")
+    if configured:
+        return Path(configured)
+    candidates = (
+        REPO_ROOT / "config/banbot.local.yml",
+        banbot_dir.parent / "data/config.timescale.yml",
+        banbot_dir / "biz/config.timescale.yml",
+        banbot_dir / "biz/config.yml",
+    )
+    return next((path for path in candidates if path.is_file()), candidates[-1])
 
 
 def output_columns_for_timeframe(timeframe: str) -> tuple[str, ...]:
@@ -98,28 +156,40 @@ def _validate_freshness(
 def ensure_database(
     banbot_config: Path,
     dsn: str | None,
+    db_type: str = "auto",
 ) -> None:
     database_url = load_database_url(banbot_config, dsn)
+    if db_type == "auto" and dsn is None:
+        config = yaml.safe_load(banbot_config.read_text(encoding="utf-8")) or {}
+        db_type = (config.get("database") or {}).get("db_type") or "auto"
+    # PostgreSQL connections are validated by prepare_data via libpq, which also
+    # supports Unix sockets, multiple hosts and connection-service definitions.
+    if db_type not in {"auto", "questdb"}:
+        return
+    parameters = conninfo_to_dict(database_url)
+    if any("," in parameters.get(key, "") for key in ("host", "hostaddr", "port")):
+        return
     host, port = _database_endpoint(database_url)
+    if db_type == "auto" and port != 8812:
+        return
     try:
         with socket.create_connection((host, port), timeout=1):
             return
     except OSError:
         pass
-    if port != 8812:
-        return
-
     print("[latest-predict] QuestDB is unavailable; letting banbot start it", flush=True)
     start_ms = int(datetime.now(timezone.utc).timestamp() * 1000) - 8 * 60 * 60 * 1000
     end_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
-    with tempfile.TemporaryDirectory(prefix="cbpred-db-bootstrap-") as output_dir:
+    with tempfile.TemporaryDirectory(prefix="cbpred-db-bootstrap-") as output_dir, effective_banbot_config(
+        banbot_config, database_url, "questdb"
+    ) as effective_config:
         subprocess.run(
             [
                 "go",
                 "run",
                 "./cmd/cbpred-export",
                 "--config",
-                str(banbot_config),
+                str(effective_config),
                 "--data-dir",
                 str(banbot_config.parent),
                 "--pairs",
@@ -222,11 +292,12 @@ def build_predictions(data_dir: Path, artifact: Path, output_dir: Path) -> dict[
 
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="Update banbot K-lines and predict with the current model")
-    parser.add_argument("--banbot-dir", type=Path, default=Path("/data/ban/banbot"))
+    parser.add_argument("--banbot-dir", type=Path, default=default_banbot_dir())
     parser.add_argument(
         "--banbot-config",
         type=Path,
-        default=Path("/data/ban/banbot/biz/config.yml"),
+        default=None,
+        help="banbot config (default: local config or existing TimescaleDB config)",
     )
     parser.add_argument("--data-dir", type=Path, default=None)
     parser.add_argument("--artifact", type=Path, default=None)
@@ -241,8 +312,11 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--min-age-days", type=int, default=730)
     parser.add_argument("--chunk-size", type=int, default=100)
     parser.add_argument("--end", default=None, help="UTC date/ISO timestamp or milliseconds")
-    parser.add_argument("--dsn", default=None)
-    parser.add_argument("--db-type", choices=["auto", "questdb", "timescale"], default="auto")
+    parser.add_argument("--dsn", default=os.environ.get("CBPRED_DSN"))
+    parser.add_argument(
+        "--db-type", choices=["auto", "questdb", "timescale"],
+        default=os.environ.get("CBPRED_DB_TYPE", "auto"),
+    )
     parser.add_argument("--exchange", default="binance")
     parser.add_argument("--timeframe", choices=["8h", "1d"], default="8h")
     return parser
@@ -250,6 +324,9 @@ def build_parser() -> argparse.ArgumentParser:
 
 def main() -> None:
     args = build_parser().parse_args()
+    args.banbot_config = (
+        args.banbot_config or default_banbot_config(args.banbot_dir)
+    ).resolve()
     data_dir = (args.data_dir or default_data_dir(args.timeframe)).resolve()
     artifact = (args.artifact or default_artifact(args.timeframe)).resolve()
     output_dir = (
@@ -258,8 +335,8 @@ def main() -> None:
     _validate_artifact_timeframe(artifact, args.timeframe)
     lock_path = args.banbot_config.parent / ".cbpred-latest-predict.lock"
     with lock_path.open("a+") as lock:
-        fcntl.flock(lock, fcntl.LOCK_EX)
-        ensure_database(args.banbot_config, args.dsn)
+        _acquire_update_lock(lock)
+        ensure_database(args.banbot_config, args.dsn, args.db_type)
         manifest = prepare_data(
             out_dir=data_dir,
             banbot_dir=args.banbot_dir,

cbpred/pipeline.py

@@ -8,10 +8,12 @@ import re
 import shlex
 import subprocess
 import sys
+import tempfile
+from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any, Iterable, Iterator
 
 import joblib
 import numpy as np
@@ -141,13 +143,61 @@ def detect_db_type(conn, requested: str = "auto") -> str:
         return requested
     with conn.cursor() as cur:
         try:
-            cur.execute("SELECT 1 FROM exsymbol_q LIMIT 1")
-            return "questdb"
-        except Exception:
+            cur.execute("SELECT to_regclass('exsymbol'), to_regclass('exsymbol_q')")
+            timescale_table, questdb_table = cur.fetchone()
+        except Exception as exc:
             conn.rollback()
+            # QuestDB speaks the PostgreSQL wire protocol but lacks to_regclass.
+            message = str(exc).lower()
+            if getattr(exc, "sqlstate", None) != "42883" and not (
+                "to_regclass" in message
+                and any(text in message for text in ("unknown function", "does not exist", "unsupported", "no signature"))
+            ):
+                raise
+        else:
+            if timescale_table is not None:
+                return "timescale"
+            if questdb_table is not None:
+                return "questdb"
+            return "timescale"
+        for table, backend in (("exsymbol", "timescale"), ("exsymbol_q", "questdb")):
+            try:
+                cur.execute(f"SELECT 1 FROM {table} LIMIT 1")
+                return backend
+            except Exception as exc:
+                conn.rollback()
+                message = str(exc).lower()
+                if getattr(exc, "sqlstate", None) != "42P01" and not (
+                    table in message and any(text in message for text in ("does not exist", "not found"))
+                ):
+                    raise
     return "timescale"
 
 
+@contextmanager
+def effective_banbot_config(config_path: Path, dsn: str, db_type: str) -> Iterator[Path]:
+    """Give Python and Go the same database without changing the original config."""
+    config_path = config_path.resolve()
+    config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    database = dict(config.get("database") or {})
+    database.update(url=dsn, db_type=db_type)
+    config["database"] = database
+    config["market_type"] = "linear"
+    temporary_path = None
+    try:
+        # Keep relative assets and --datadir anchored to the original directory.
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", suffix=".yml", prefix=".cbpred-db-",
+            dir=config_path.parent, delete=False,
+        ) as handle:
+            temporary_path = Path(handle.name)
+            yaml.safe_dump(config, handle, allow_unicode=True, sort_keys=False)
+        yield temporary_path
+    finally:
+        if temporary_path is not None:
+            temporary_path.unlink(missing_ok=True)
+
+
 def fetch_symbols(
     conn,
     db_type: str,
@@ -250,7 +300,10 @@ def run_banbot_download(
                 str(end_ms),
             ]
             try:
-                subprocess.run(command, cwd=banbot_dir, check=True, stderr=subprocess.PIPE, text=True)
+                subprocess.run(
+                    command, cwd=banbot_dir, check=True,
+                    stderr=subprocess.PIPE, text=True, encoding="utf-8",
+                )
             except subprocess.CalledProcessError as exc:
                 invalid_symbols = set(INVALID_SYMBOL_RE.findall(exc.stderr or ""))
                 rejected = [item for item in chunk if item.symbol in invalid_symbols]
@@ -391,27 +444,28 @@ def prepare_data(
         )
     finally:
         conn.close()
-    if not skip_download:
-        symbols, unavailable = run_banbot_download(
+    with effective_banbot_config(banbot_config, database_url, actual_db_type) as effective_config:
+        if not skip_download:
+            symbols, unavailable = run_banbot_download(
+                symbols,
+                banbot_dir,
+                effective_config,
+                history_start_ms,
+                end_ms,
+                chunk_size,
+                banbot_cmd,
+            )
+            excluded.extend({**symbol.__dict__, "reason": "unavailable_on_exchange"} for symbol in unavailable)
+        run_banbot_export(
             symbols,
             banbot_dir,
-            banbot_config,
+            effective_config,
+            raw_dir,
             history_start_ms,
             end_ms,
-            chunk_size,
-            banbot_cmd,
+            banbot_export_cmd,
+            timeframe,
         )
-        excluded.extend({**symbol.__dict__, "reason": "unavailable_on_exchange"} for symbol in unavailable)
-    run_banbot_export(
-        symbols,
-        banbot_dir,
-        banbot_config,
-        raw_dir,
-        history_start_ms,
-        end_ms,
-        banbot_export_cmd,
-        timeframe,
-    )
     manifest_symbols: list[dict[str, Any]] = []
     for symbol in symbols:
         path = raw_dir / symbol_filename(symbol)