923 lines
26 KiB
Go
923 lines
26 KiB
Go
package adv
|
||
|
||
import (
|
||
"errors"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"math"
|
||
"net/http"
|
||
"path/filepath"
|
||
"slices"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"gonum.org/v1/gonum/floats"
|
||
|
||
"go.uber.org/zap"
|
||
|
||
"github.com/banbox/banbot/biz"
|
||
"github.com/banbox/banbot/btime"
|
||
"github.com/banbox/banbot/config"
|
||
"github.com/banbox/banbot/core"
|
||
"github.com/banbox/banbot/entry"
|
||
"github.com/banbox/banbot/opt"
|
||
"github.com/banbox/banbot/orm"
|
||
"github.com/banbox/banbot/orm/ormo"
|
||
utils2 "github.com/banbox/banbot/utils"
|
||
"github.com/banbox/banexg/errs"
|
||
"github.com/banbox/banexg/log"
|
||
"github.com/banbox/banexg/utils"
|
||
"github.com/spf13/cobra"
|
||
)
|
||
|
||
func init() {
|
||
runConfig := func(args *config.CmdArgs, run func(*config.CmdArgs) *errs.Error) error {
|
||
core.SetRunMode(core.RunModeOther)
|
||
args.Init()
|
||
return run(args)
|
||
}
|
||
bindCommon := func(command *cobra.Command, args *config.CmdArgs) {
|
||
flags := command.Flags()
|
||
flags.StringArrayVar((*[]string)(&args.Configs), "config", nil, "config path; may be repeated")
|
||
flags.StringVar(&args.ConfigData, "config-data", "", "inline YAML config")
|
||
flags.StringVar(&args.Logfile, "logfile", "", "log file path")
|
||
flags.StringVar(&args.DataDir, "datadir", "", "path to the data directory")
|
||
flags.BoolVar(&args.NoDefault, "no-default", false, "ignore config.yml and config.local.yml")
|
||
flags.StringVar(&args.LogLevel, "level", "info", "logging level")
|
||
flags.IntVar(&args.MaxPoolSize, "max-pool-size", 0, "maximum database pool size")
|
||
flags.BoolVar(&core.NetDisable, "net-off", false, "disable network requests")
|
||
}
|
||
newConfigCommand := func(name, help string, run func(*config.CmdArgs) *errs.Error,
|
||
bind func(*cobra.Command, *config.CmdArgs)) *cobra.Command {
|
||
args := &config.CmdArgs{}
|
||
command := &cobra.Command{
|
||
Use: name,
|
||
Short: help,
|
||
Args: cobra.NoArgs,
|
||
RunE: func(_ *cobra.Command, _ []string) error {
|
||
return runConfig(args, run)
|
||
},
|
||
}
|
||
bindCommon(command, args)
|
||
if bind != nil {
|
||
bind(command, args)
|
||
}
|
||
return command
|
||
}
|
||
|
||
dayRets := newConfigCommand("dayRets", "calc return rate for pairs by months", CalcPairDayPerfs,
|
||
func(command *cobra.Command, args *config.CmdArgs) {
|
||
command.Flags().StringVar(&args.InPath, "in", "", "input file or directory")
|
||
})
|
||
entry.AddCommand("tool", dayRets)
|
||
|
||
entry.AddCommand("tool", newConfigCommand("updMkt", "update coin markets by month", UpdateCoinMarkets, nil))
|
||
entry.AddCommand("tool", newConfigCommand("buildFac", "build factor samples", BuildFacSamples, nil))
|
||
entry.AddCommand("tool", &cobra.Command{
|
||
Use: "testFac",
|
||
Short: "test factors",
|
||
DisableFlagParsing: true,
|
||
RunE: func(_ *cobra.Command, args []string) error {
|
||
return TestFactors(args)
|
||
},
|
||
})
|
||
}
|
||
|
||
var urlCoinMarket = "https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listings/historical?convertId=2781&date=%s&limit=%d&start=%d"
|
||
var coinSlugMap = map[string]string{
|
||
"BTCDOM": "",
|
||
"1000XEC": "ecash",
|
||
"LUNA2": "terra-luna-v2",
|
||
"DODOX": "dodo",
|
||
"BEAMX": "onbeam",
|
||
"1000RATS": "",
|
||
"RONIN": "ronin",
|
||
"1000SHIB": "shiba-inu",
|
||
"1000LUNC": "terra-luna",
|
||
"1000FLOKI": "floki-inu",
|
||
"1000PEPE": "pepe",
|
||
"1000BONK": "bonk1",
|
||
}
|
||
|
||
type PairReturns struct {
|
||
StartMS int64
|
||
EndMS int64
|
||
Pairs map[string]*ItemReturns
|
||
}
|
||
|
||
type ItemReturns struct {
|
||
StartMS int64
|
||
EndMS int64
|
||
Arr []float64
|
||
}
|
||
|
||
type CoinMarket struct {
|
||
Name string `json:"name"`
|
||
Symbol string `json:"symbol"`
|
||
ListMS int64 `json:"list_ms"`
|
||
Slug string `json:"slug"`
|
||
FirstMS int64 `json:"first_ms"`
|
||
LastMS int64 `json:"last_ms"`
|
||
MarketHis []*MarketSlice `json:"market_his"`
|
||
}
|
||
|
||
type MarketSlice struct {
|
||
TimeMS int64 `json:"time_ms"`
|
||
CirculatingSupply float64 `json:"circulating_supply"`
|
||
TotalSupply float64 `json:"total_supply"`
|
||
Price float64 `json:"price"`
|
||
MarketCap float64 `json:"market_cap"`
|
||
}
|
||
|
||
type CoinMarketHis struct {
|
||
Coins map[string]*CoinMarket
|
||
Times []int64
|
||
}
|
||
|
||
type CoinMarketResponse struct {
|
||
Data []struct {
|
||
ID int `json:"id"`
|
||
Name string `json:"name"`
|
||
Symbol string `json:"symbol"`
|
||
Slug string `json:"slug"`
|
||
DateAdded string `json:"dateAdded"`
|
||
CirculatingSupply float64 `json:"circulatingSupply"`
|
||
TotalSupply float64 `json:"totalSupply"`
|
||
Quotes []struct {
|
||
Name string `json:"name"`
|
||
Price float64 `json:"price"`
|
||
MarketCap float64 `json:"marketCap"`
|
||
} `json:"quotes"`
|
||
} `json:"data"`
|
||
Status struct {
|
||
ErrorCode string `json:"error_code"`
|
||
ErrorMessage string `json:"error_message"`
|
||
} `json:"status"`
|
||
}
|
||
|
||
/*
|
||
某个品种在某个时间的历史收益，市场、未来回报等信息
|
||
*/
|
||
type PairSample struct {
|
||
Offset int // 值int是在ItemReturns.Arr的位置索引
|
||
Pair string
|
||
Code string
|
||
Price float64
|
||
MarketCap float64
|
||
Vols []float64
|
||
}
|
||
|
||
type FacHis struct {
|
||
Times []int64
|
||
Pairs map[string]*ItemReturns
|
||
Data map[int64]map[string]*PairSample // time: pair: sample
|
||
}
|
||
|
||
type FacState struct {
|
||
Ret *ItemReturns
|
||
Sample *PairSample
|
||
Groups map[string]string
|
||
Label string
|
||
Tmp float64 // 因子计算临时使用
|
||
OutScore float64 // 输出分数
|
||
OutID int // 输出分数排序序号
|
||
}
|
||
|
||
type GpState struct {
|
||
count float64 // 该块中FacState的数量
|
||
sumScore float64 // 该块的平均分数
|
||
}
|
||
|
||
/*
|
||
CalcPairDayPerfs 从订单计算每个品种的每日收益率
|
||
*/
|
||
func CalcPairDayPerfs(args *config.CmdArgs) *errs.Error {
|
||
err := biz.SetupComs(args)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
orders, err := ormo.LoadOrdersGob(args.InPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
pairOrders := make(map[string][]*ormo.InOutOrder)
|
||
minTimeMS, maxTimeMS := btime.TimeMS(), int64(0)
|
||
totalCost := float64(0)
|
||
for _, o := range orders {
|
||
odList, _ := pairOrders[o.Symbol]
|
||
pairOrders[o.Symbol] = append(odList, o)
|
||
minTimeMS = min(minTimeMS, o.Enter.CreateAt)
|
||
maxTimeMS = max(maxTimeMS, o.Exit.CreateAt)
|
||
totalCost += o.EnterCost() / o.Leverage
|
||
}
|
||
// 假设初始金额能同时开50笔
|
||
initStake := totalCost / float64(len(orders)) * 50
|
||
// 解析开始时间和结束时间，对齐到月的1号
|
||
startYear, startMon, _ := btime.MSToTime(minTimeMS).Date()
|
||
endYear, endMon, endDay := btime.MSToTime(maxTimeMS).Date()
|
||
endAddMon := 0
|
||
if endDay > 1 {
|
||
endAddMon = 1
|
||
}
|
||
startMS := time.Date(startYear, startMon, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
|
||
endTime := time.Date(endYear, endMon, 1, 0, 0, 0, 0, time.UTC)
|
||
endMS := endTime.AddDate(0, endAddMon, 0).UnixMilli()
|
||
symbolMap := orm.GetExSymbolMap(core.ExgName, core.Market)
|
||
dayMSecs := int64(utils.TFToSecs("1d") * 1000)
|
||
retMap := make(map[string]*ItemReturns)
|
||
var addNum int
|
||
for pair, odList := range pairOrders {
|
||
sort.Slice(odList, func(i, j int) bool {
|
||
a, b := odList[i], odList[j]
|
||
return a.Enter.CreateAt < b.Enter.CreateAt
|
||
})
|
||
exs := symbolMap[pair]
|
||
_, bars, err := orm.GetOHLCV(exs, "1d", startMS, endMS, 0, false)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var closes []float64
|
||
if len(bars) > 0 {
|
||
bars, addNum = utils2.FillOHLCVLacks(bars, startMS, endMS, dayMSecs)
|
||
if addNum > 0 {
|
||
log.Info("fill ohlcv lack", zap.String("pair", pair), zap.Int("num", addNum))
|
||
}
|
||
closes = make([]float64, len(bars))
|
||
for i, b := range bars {
|
||
closes[i] = b.Close
|
||
}
|
||
}
|
||
// 计算每日回报率
|
||
returns, dayOff, dayEnd := ormo.CalcUnitReturns(odList, closes, startMS, endMS, dayMSecs)
|
||
retRates := make([]float64, len(returns))
|
||
for i, ret := range returns {
|
||
retRates[i] = ret / initStake
|
||
}
|
||
retMap[pair] = &ItemReturns{
|
||
StartMS: startMS + int64(dayOff)*dayMSecs,
|
||
EndMS: startMS + int64(dayEnd)*dayMSecs,
|
||
Arr: retRates[dayOff:dayEnd],
|
||
}
|
||
}
|
||
outPath := filepath.Join(filepath.Dir(args.InPath), "out.gob")
|
||
return utils2.EncodeGob(outPath, &PairReturns{
|
||
StartMS: startMS,
|
||
EndMS: endMS,
|
||
Pairs: retMap,
|
||
})
|
||
}
|
||
|
||
func loadHisMarkets() (*CoinMarketHis, string, *errs.Error) {
|
||
path := filepath.Join(config.GetDataDir(), "cryptoMarkets.gob")
|
||
var data = &CoinMarketHis{
|
||
Coins: make(map[string]*CoinMarket),
|
||
Times: make([]int64, 0),
|
||
}
|
||
err := utils2.DecodeGobFile(path, data)
|
||
if err != nil {
|
||
return data, path, err
|
||
}
|
||
return data, path, nil
|
||
}
|
||
|
||
/*
|
||
UpdateCoinMarkets 更新BanDataDir目录下的cryptoMarkets.gob全币种市场信息。每个月更新一次。
|
||
*/
|
||
func UpdateCoinMarkets(args *config.CmdArgs) *errs.Error {
|
||
err := biz.SetupComs(args)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
data, path, err := loadHisMarkets()
|
||
if err != nil {
|
||
log.Warn("load cryptoMarkets error", zap.Error(err))
|
||
}
|
||
|
||
// 确定开始时间
|
||
startTime := time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)
|
||
if len(data.Times) > 0 {
|
||
startTime = time.UnixMilli(data.Times[len(data.Times)-1]).UTC().AddDate(0, 1, 0)
|
||
}
|
||
lastTimeMS := startTime.AddDate(0, -1, 0).UnixMilli()
|
||
batchSize := 2000
|
||
isLast := false
|
||
|
||
// 循环获取每个月的数据
|
||
for {
|
||
if startTime.After(time.Now()) {
|
||
pyear, pmon, pday := time.Now().UTC().Date()
|
||
startTime = time.Date(pyear, pmon, pday, 0, 0, 0, 0, time.UTC).AddDate(0, 0, -1)
|
||
isLast = true
|
||
}
|
||
|
||
// 获取该月所有币种数据
|
||
monthData := make(map[string]*MarketSlice)
|
||
start := 1
|
||
fetchNum, fixLack := 0, 0
|
||
timeMS := startTime.UnixMilli()
|
||
for {
|
||
url := fmt.Sprintf(urlCoinMarket, startTime.Format("2006-01-02"), batchSize, start)
|
||
|
||
time.Sleep(time.Second * 1)
|
||
resp, err_ := http.Get(url)
|
||
log.Info("request", zap.String("url", url))
|
||
if err_ != nil {
|
||
return errs.New(errs.CodeNetFail, err_)
|
||
}
|
||
|
||
body, err_ := io.ReadAll(resp.Body)
|
||
resp.Body.Close()
|
||
if err_ != nil {
|
||
return errs.New(errs.CodeNetFail, err_)
|
||
}
|
||
|
||
var rsp CoinMarketResponse
|
||
if err_ = utils.Unmarshal(body, &rsp, utils.JsonNumAuto); err_ != nil {
|
||
return errs.New(errs.CodeUnmarshalFail, err_)
|
||
}
|
||
|
||
if rsp.Status.ErrorCode != "0" {
|
||
return errs.NewMsg(errs.CodeRunTime, "code: %v, msg: %v", rsp.Status.ErrorCode, rsp.Status.ErrorMessage)
|
||
}
|
||
|
||
if len(rsp.Data) == 0 {
|
||
break
|
||
}
|
||
|
||
for _, coin := range rsp.Data {
|
||
slice := &MarketSlice{
|
||
TimeMS: timeMS,
|
||
CirculatingSupply: coin.CirculatingSupply,
|
||
TotalSupply: coin.TotalSupply,
|
||
}
|
||
if len(coin.Quotes) > 0 {
|
||
slice.Price = coin.Quotes[0].Price
|
||
slice.MarketCap = coin.Quotes[0].MarketCap
|
||
}
|
||
monthData[coin.Slug] = slice
|
||
|
||
// 更新或创建币种信息
|
||
if cm, exists := data.Coins[coin.Slug]; exists {
|
||
if len(cm.MarketHis) == 0 || timeMS > cm.MarketHis[len(cm.MarketHis)-1].TimeMS {
|
||
// 检查是否有缺失的月份数据需要补充
|
||
if !isLast {
|
||
if cm.LastMS < lastTimeMS {
|
||
fixLack += 1
|
||
lastSlice := cm.MarketHis[len(cm.MarketHis)-1]
|
||
// 从上次更新的下个月开始，直到上个月为止，补充所有缺失的月份
|
||
curTime := time.UnixMilli(cm.LastMS).UTC().AddDate(0, 1, 0)
|
||
for curTime.UnixMilli() <= lastTimeMS {
|
||
newSlice := &MarketSlice{
|
||
TimeMS: curTime.UnixMilli(),
|
||
CirculatingSupply: lastSlice.CirculatingSupply,
|
||
TotalSupply: lastSlice.TotalSupply,
|
||
Price: lastSlice.Price,
|
||
MarketCap: lastSlice.MarketCap,
|
||
}
|
||
cm.MarketHis = append(cm.MarketHis, newSlice)
|
||
cm.LastMS = curTime.UnixMilli()
|
||
curTime = curTime.AddDate(0, 1, 0)
|
||
}
|
||
}
|
||
// 添加当前月份的数据
|
||
cm.MarketHis = append(cm.MarketHis, slice)
|
||
cm.LastMS = timeMS
|
||
}
|
||
if cm.Name != coin.Name {
|
||
cm.Name = coin.Name
|
||
}
|
||
if cm.Symbol != coin.Symbol {
|
||
cm.Symbol = coin.Symbol
|
||
}
|
||
fetchNum += 1
|
||
}
|
||
} else if !isLast {
|
||
listTime, _ := time.Parse("2006-01-02T15:04:05.000Z", coin.DateAdded)
|
||
data.Coins[coin.Slug] = &CoinMarket{
|
||
Name: coin.Name,
|
||
Symbol: coin.Symbol,
|
||
ListMS: listTime.UnixMilli(),
|
||
Slug: coin.Slug,
|
||
FirstMS: timeMS,
|
||
LastMS: timeMS,
|
||
MarketHis: []*MarketSlice{slice},
|
||
}
|
||
fetchNum += 1
|
||
}
|
||
}
|
||
|
||
// 这里可能返回数量不足batchSize个，但不代表结束了，继续请求返回为空才表示结束
|
||
start += batchSize
|
||
}
|
||
|
||
// 添加时间戳
|
||
data.Times = append(data.Times, timeMS)
|
||
|
||
// 保存数据
|
||
if err := utils2.EncodeGob(path, data); err != nil {
|
||
log.Error("save market data failed", zap.Error(err))
|
||
return err
|
||
} else {
|
||
log.Info("update coin markets ok", zap.String("date", startTime.Format("2006-01-02")),
|
||
zap.Int("fetch_num", fetchNum), zap.Int("fixLack", fixLack))
|
||
}
|
||
|
||
// 移动到下一个月
|
||
lastTimeMS = timeMS
|
||
startTime = startTime.AddDate(0, 1, 0)
|
||
if isLast {
|
||
break
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
/*
|
||
BuildFacSamples 构建因子定期采样历史
|
||
*/
|
||
func BuildFacSamples(args *config.CmdArgs) *errs.Error {
|
||
err := biz.SetupComs(args)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
data, _, err := loadHisMarkets()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
retData := &PairReturns{Pairs: make(map[string]*ItemReturns)}
|
||
retPath := "E:\\trade\\go\\bandata\\backtest\\383f7dc32a_all_pairs\\out.gob"
|
||
if !utils2.Exists(retPath) {
|
||
args.InPath = filepath.Join(filepath.Dir(retPath), "orders.gob")
|
||
err = CalcPairDayPerfs(args)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
}
|
||
err = utils2.DecodeGobFile(retPath, retData)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(retData.Pairs) == 0 {
|
||
return errs.NewMsg(errs.CodeRunTime, "no Pairs for PairReturns")
|
||
}
|
||
res := &FacHis{Pairs: retData.Pairs, Data: make(map[int64]map[string]*PairSample)}
|
||
startMS := retData.StartMS
|
||
|
||
// 构建Symbol到CoinMarket的映射
|
||
var b strings.Builder
|
||
symbolToCoin := make(map[string]*CoinMarket)
|
||
badCoins := make(map[string]bool)
|
||
skipCoins := make(map[string]bool)
|
||
dupCoins := make(map[string][]string)
|
||
for _, coin := range data.Coins {
|
||
if old, ok := symbolToCoin[coin.Symbol]; ok {
|
||
dups, _ := dupCoins[coin.Symbol]
|
||
if len(dups) == 0 {
|
||
dups = append(dups, old.Slug)
|
||
}
|
||
dupCoins[coin.Symbol] = append(dups, coin.Slug)
|
||
continue
|
||
}
|
||
symbolToCoin[coin.Symbol] = coin
|
||
b.WriteString(coin.Symbol)
|
||
b.WriteString(", ")
|
||
}
|
||
symbolMap := orm.GetExSymbolMap(core.ExgName, core.Market)
|
||
log.Info("all coins", zap.String("v", b.String()))
|
||
if len(dupCoins) > 0 {
|
||
log.Info("dupCoins", zap.String("v", utils2.MapToStr(dupCoins, true, 0)))
|
||
}
|
||
|
||
// 设置月份间隔（这里设置为3个月）
|
||
monNum := 2
|
||
|
||
// 将startMS转换为月初
|
||
startTime := time.UnixMilli(startMS).UTC()
|
||
startTime = time.Date(startTime.Year(), startTime.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||
startTime = startTime.AddDate(0, monNum, 0)
|
||
endTime := time.UnixMilli(retData.EndMS).UTC()
|
||
|
||
// 从startTime开始遍历时间点
|
||
for curTime := startTime; !curTime.After(endTime); curTime = curTime.AddDate(0, monNum, 0) {
|
||
timeMS := curTime.UnixMilli()
|
||
timeMap := make(map[string]*PairSample)
|
||
prevTime := curTime.AddDate(0, -monNum, 0)
|
||
nextTime := curTime.AddDate(0, monNum, 0)
|
||
|
||
// 对每个pair的returns进行检查
|
||
for pair, itemRet := range retData.Pairs {
|
||
// 检查时间范围是否符合要求
|
||
if itemRet.StartMS > prevTime.UnixMilli() || itemRet.EndMS < nextTime.UnixMilli() {
|
||
continue
|
||
}
|
||
|
||
// 计算当前时间点在Arr中的索引
|
||
offset := int((timeMS - itemRet.StartMS) / (24 * 3600 * 1000)) // 转换为天数
|
||
if offset < 0 || offset >= len(itemRet.Arr) {
|
||
continue
|
||
}
|
||
|
||
// 查找对应的CoinMarket
|
||
baseCode, _, _, _ := core.SplitSymbol(pair)
|
||
coin, ok := symbolToCoin[baseCode]
|
||
if !ok {
|
||
if slug, ok := coinSlugMap[baseCode]; ok {
|
||
if slug == "" {
|
||
skipCoins[baseCode] = true
|
||
continue
|
||
}
|
||
coin = data.Coins[slug]
|
||
} else {
|
||
badCoins[baseCode] = true
|
||
continue
|
||
}
|
||
}
|
||
|
||
// 查找当前时间点对应的MarketSlice
|
||
var mktSlice *MarketSlice
|
||
for _, slice := range coin.MarketHis {
|
||
if slice.TimeMS <= timeMS {
|
||
mktSlice = slice
|
||
} else {
|
||
break
|
||
}
|
||
}
|
||
if mktSlice == nil {
|
||
// 太新，跳过
|
||
continue
|
||
}
|
||
|
||
// 获取成交量数据
|
||
exs := symbolMap[pair]
|
||
if exs == nil {
|
||
return errs.NewMsg(errs.CodeRunTime, "ExSymbol not exist: %v", pair)
|
||
}
|
||
|
||
// 计算2周和1个月的成交量
|
||
barStartMS := prevTime.UnixMilli()
|
||
_, bars, err := orm.GetOHLCV(exs, "1d", barStartMS, timeMS, 0, false)
|
||
if err != nil {
|
||
log.Warn("get volume failed", zap.String("pair", pair), zap.Error(err))
|
||
continue
|
||
}
|
||
|
||
// 计算成交量
|
||
vols := make([]float64, len(bars))
|
||
for i, bar := range bars {
|
||
vols[i] = bar.Close * bar.Volume
|
||
}
|
||
|
||
// 构建PairSample
|
||
sample := &PairSample{
|
||
Offset: offset,
|
||
Pair: pair,
|
||
Code: baseCode,
|
||
Price: mktSlice.Price,
|
||
MarketCap: mktSlice.MarketCap,
|
||
Vols: vols,
|
||
}
|
||
|
||
timeMap[pair] = sample
|
||
}
|
||
|
||
if len(timeMap) > 0 {
|
||
res.Data[timeMS] = timeMap
|
||
res.Times = append(res.Times, timeMS)
|
||
}
|
||
}
|
||
|
||
if len(badCoins) > 0 {
|
||
log.Warn("unknown coins", zap.String("v", utils2.MapToStr(badCoins, false, 0)))
|
||
}
|
||
if len(skipCoins) > 0 {
|
||
log.Warn("skip coins", zap.String("v", utils2.MapToStr(skipCoins, false, 0)))
|
||
}
|
||
|
||
// 保存结果
|
||
name := fmt.Sprintf("facHis_%dM.gob", monNum)
|
||
outPath := filepath.Join(config.GetDataDir(), name)
|
||
return utils2.EncodeGob(outPath, res)
|
||
}
|
||
|
||
func TestFactors(args []string) error {
|
||
parser := flag.NewFlagSet("", flag.ExitOnError)
|
||
var monNum, outNum, offNum int
|
||
var facStr, outBy string
|
||
parser.IntVar(&monNum, "mon", 2, "month for period")
|
||
parser.IntVar(&outNum, "out", 3, "output group num")
|
||
parser.IntVar(&offNum, "off", 0, "skip group num in head")
|
||
parser.StringVar(&facStr, "fac", "", "factors to test")
|
||
parser.StringVar(&outBy, "outBy", "", "calc output score by")
|
||
err_ := parser.Parse(args)
|
||
if err_ != nil {
|
||
return err_
|
||
}
|
||
facArr := utils2.SplitSolid(facStr, ",", true)
|
||
if len(facArr) == 0 {
|
||
return errors.New("-fac is required")
|
||
}
|
||
dayNum := monNum * 30
|
||
fac := &FacHis{Pairs: make(map[string]*ItemReturns), Data: make(map[int64]map[string]*PairSample)}
|
||
facPath := filepath.Join(config.GetDataDir(), fmt.Sprintf("facHis_%dM.gob", monNum))
|
||
err := utils2.DecodeGobFile(facPath, fac)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
groupMeas := make(map[string][]*GpState)
|
||
for i, stampMS := range fac.Times {
|
||
if i <= offNum {
|
||
continue
|
||
}
|
||
pairSamples := fac.Data[stampMS]
|
||
items := make([]*FacState, 0)
|
||
for pair, smp := range pairSamples {
|
||
ret := fac.Pairs[pair]
|
||
items = append(items, &FacState{
|
||
Ret: ret,
|
||
Sample: smp,
|
||
Groups: make(map[string]string),
|
||
})
|
||
}
|
||
// 计算每个采样所属的因子分组
|
||
for _, factor := range facArr {
|
||
var pos []int
|
||
var tags []string
|
||
if factor == "marketCap" {
|
||
pos, tags = facMarketCap(items)
|
||
} else if factor == "vol1M" {
|
||
// 优于vol2w和marketCap
|
||
pos, tags = facVol1M(items)
|
||
} else if factor == "volStd" {
|
||
// 成交量波动较低的
|
||
pos, tags = facVolStd(items)
|
||
} else if factor == "mont1" {
|
||
// 3M下，跟mont2不相上下
|
||
pos, tags = facMont1(items, dayNum)
|
||
} else if factor == "mont2" {
|
||
pos, tags = facMont2(items, dayNum)
|
||
} else if factor == "mont1y" {
|
||
pos, tags = facMont1Y(items, dayNum)
|
||
} else if factor == "sharpe1y" {
|
||
pos, tags = facSharpe1Y(items, dayNum)
|
||
} else if factor == "score1y" {
|
||
pos, tags = facScore1Y(items, dayNum)
|
||
} else {
|
||
return errors.New("unknown factor: " + factor)
|
||
}
|
||
if len(pos)+1 != len(tags) {
|
||
return errors.New("invalid factor: " + factor)
|
||
}
|
||
start := 0
|
||
for i, end := range pos {
|
||
tag := tags[i]
|
||
for j := start; j < end; j++ {
|
||
items[j].Groups[factor] = tag
|
||
}
|
||
start = end
|
||
}
|
||
tag := tags[len(tags)-1]
|
||
for j := start; j < len(items); j++ {
|
||
items[j].Groups[factor] = tag
|
||
}
|
||
}
|
||
// 组合因子分组
|
||
groups := make(map[string][]*FacState)
|
||
for _, sta := range items {
|
||
var b strings.Builder
|
||
for _, f := range facArr {
|
||
b.WriteString(sta.Groups[f] + "+")
|
||
}
|
||
sta.Label = strings.TrimRight(b.String(), "+")
|
||
gpItems, _ := groups[sta.Label]
|
||
groups[sta.Label] = append(gpItems, sta)
|
||
end := sta.Sample.Offset + dayNum
|
||
if end > len(sta.Ret.Arr) {
|
||
end = len(sta.Ret.Arr)
|
||
}
|
||
// 计算未来分数，并进行分组打分
|
||
futures := sta.Ret.Arr[sta.Sample.Offset:end]
|
||
if outBy == "profit" {
|
||
sta.OutScore = floats.Sum(futures)
|
||
} else if outBy == "sharpe" {
|
||
sta.OutScore, err_ = utils2.SharpeRatioBy(futures, 0.02, 365, true)
|
||
if err_ != nil {
|
||
return err_
|
||
}
|
||
} else if outBy == "smart" || outBy == "" {
|
||
sharpe, err_ := utils2.SharpeRatioBy(futures, 0.02, 365, true)
|
||
if err_ != nil {
|
||
return err_
|
||
}
|
||
profit := floats.Sum(futures)
|
||
sta.OutScore = math.Log2(profit) * sharpe
|
||
} else {
|
||
return errors.New("unknown outBy: " + outBy)
|
||
}
|
||
}
|
||
// 按输出分数降序
|
||
sort.Slice(items, func(i, j int) bool {
|
||
return items[i].OutScore < items[j].OutScore
|
||
})
|
||
for i, it := range items {
|
||
it.OutID = i
|
||
}
|
||
// 计算每个目标块应包含的FacState数量
|
||
blockNum := int(math.Ceil(float64(len(items)) / float64(outNum)))
|
||
for key, gpList := range groups {
|
||
// 遍历分组中的每个FacState,根据其OutID确定所属块
|
||
blocks, ok := groupMeas[key]
|
||
if !ok {
|
||
blocks = make([]*GpState, outNum)
|
||
for i := 0; i < outNum; i++ {
|
||
blocks[i] = &GpState{}
|
||
}
|
||
groupMeas[key] = blocks
|
||
}
|
||
for _, state := range gpList {
|
||
blockID := state.OutID / blockNum // 确定目标块ID
|
||
if blockID >= outNum {
|
||
blockID = outNum - 1 // 处理边界情况
|
||
}
|
||
|
||
// 更新该块的统计信息
|
||
block := blocks[blockID]
|
||
block.count++
|
||
block.sumScore += utils2.NanInfTo(state.OutScore, 0)
|
||
}
|
||
}
|
||
}
|
||
|
||
countDsList := make([]*opt.ChartDs, len(groupMeas)) // 用于绘制数量分布图
|
||
scoreDsList := make([]*opt.ChartDs, len(groupMeas)) // 用于绘制平均分数图
|
||
// 生成x轴标签
|
||
xLabels := make([]string, outNum)
|
||
for i := 0; i < outNum; i++ {
|
||
xLabels[i] = fmt.Sprintf("Block%d", i+1)
|
||
}
|
||
|
||
keys := utils2.KeysOfMap(groupMeas)
|
||
slices.Sort(keys)
|
||
for i, key := range keys {
|
||
counts := make([]float64, outNum)
|
||
for j := 0; j < outNum; j++ {
|
||
counts[j] = groupMeas[key][j].count
|
||
}
|
||
countDsList[i] = &opt.ChartDs{
|
||
Label: key,
|
||
Data: counts,
|
||
}
|
||
scores := make([]float64, outNum)
|
||
for j := 0; j < outNum; j++ {
|
||
bk := groupMeas[key][j]
|
||
scores[j] = bk.sumScore / bk.count
|
||
}
|
||
scoreDsList[i] = &opt.ChartDs{
|
||
Label: key,
|
||
Data: scores,
|
||
}
|
||
}
|
||
|
||
facNames := strings.Join(facArr, "+")
|
||
|
||
// 生成数量分布折线图
|
||
countPath := filepath.Join(config.GetDataDir(), fmt.Sprintf("fac_count_%dM.html", monNum))
|
||
err = opt.DumpChart(countPath, "Distribution: "+facNames, xLabels, 5, nil, countDsList)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
// 生成平均分数折线图
|
||
scorePath := filepath.Join(config.GetDataDir(), fmt.Sprintf("fac_score_%dM.html", monNum))
|
||
err = opt.DumpChart(scorePath, "Average Scores: "+facNames, xLabels, 5, nil, scoreDsList)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
log.Info(fmt.Sprintf("generate in BanDataDir: fac_score_%dM.html fac_count_%dM.html", monNum, monNum))
|
||
return nil
|
||
}
|
||
|
||
func facMarketCap(items []*FacState) ([]int, []string) {
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Sample.MarketCap - a.Sample.MarketCap) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facVol1M(items []*FacState) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, len(it.Sample.Vols)-30)
|
||
it.Tmp = floats.Sum(it.Sample.Vols[start:])
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facVolStd(items []*FacState) ([]int, []string) {
|
||
for _, it := range items {
|
||
arr := it.Sample.Vols
|
||
data := make([]float64, 0, len(arr))
|
||
for i, v := range arr[1:] {
|
||
data = append(data, v/arr[i])
|
||
}
|
||
it.Tmp = utils2.StdDevVolatility(data, 1)
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facMont1(items []*FacState, unitDays int) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, it.Sample.Offset-unitDays)
|
||
it.Tmp = floats.Sum(it.Ret.Arr[start:it.Sample.Offset])
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facMont2(items []*FacState, unitDays int) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, it.Sample.Offset-unitDays*2)
|
||
it.Tmp = floats.Sum(it.Ret.Arr[start:it.Sample.Offset])
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facMont1Y(items []*FacState, unitDays int) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, it.Sample.Offset-unitDays*6)
|
||
it.Tmp = floats.Sum(it.Ret.Arr[start:it.Sample.Offset])
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facSharpe1Y(items []*FacState, unitDays int) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, it.Sample.Offset-unitDays*6)
|
||
arr := it.Ret.Arr[start:it.Sample.Offset]
|
||
sharpe, err := utils2.SharpeRatioBy(arr, 0.02, 365, true)
|
||
if err != nil {
|
||
log.Info("calc sharpe fail", zap.Error(err))
|
||
}
|
||
it.Tmp = sharpe
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|
||
|
||
func facScore1Y(items []*FacState, unitDays int) ([]int, []string) {
|
||
for _, it := range items {
|
||
start := max(0, it.Sample.Offset-unitDays*6)
|
||
arr := it.Ret.Arr[start:it.Sample.Offset]
|
||
sharpe, err_ := utils2.SharpeRatioBy(arr, 0.02, 365, true)
|
||
if err_ != nil {
|
||
log.Error("calc sharpe fail", zap.Error(err_))
|
||
}
|
||
profit := floats.Sum(arr)
|
||
it.Tmp = math.Log2(profit) * sharpe
|
||
}
|
||
slices.SortFunc(items, func(a, b *FacState) int {
|
||
return int((b.Tmp - a.Tmp) / 1000)
|
||
})
|
||
totalNum := float64(len(items))
|
||
bigNum := int(totalNum * 0.3)
|
||
midEnd := int(totalNum*0.4) + bigNum
|
||
return []int{bigNum, midEnd}, []string{"big", "mid", "sml"}
|
||
}
|