init
This commit is contained in:
348
arb/stat_prof.go
Normal file
348
arb/stat_prof.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package arb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/banbox/banbot/biz"
|
||||
"github.com/banbox/banbot/config"
|
||||
"github.com/banbox/banbot/core"
|
||||
"github.com/banbox/banbot/exg"
|
||||
"github.com/banbox/banbot/orm"
|
||||
"github.com/banbox/banbot/strat"
|
||||
"github.com/banbox/banbot/utils"
|
||||
"github.com/banbox/banexg"
|
||||
"github.com/banbox/banexg/errs"
|
||||
"github.com/banbox/banexg/log"
|
||||
"github.com/banbox/banstagy/com"
|
||||
"go.uber.org/zap"
|
||||
"gonum.org/v1/gonum/floats"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type IJV struct {
|
||||
I int
|
||||
J int
|
||||
V float64
|
||||
}
|
||||
|
||||
func ProfTuples() *errs.Error {
|
||||
exgId := "binance"
|
||||
tf := "1m"
|
||||
endMS := config.TimeRange.EndMS
|
||||
// 生成待订阅标的列表
|
||||
jobs := make(map[string]*strat.StratJob)
|
||||
exsMap := orm.GetExSymbols(exgId, banexg.MarketLinear)
|
||||
exsList, barList, jobs, err := getExsBars(exg.Default, exsMap, "1h", 0, endMS, 720)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 先从1h级别过滤，筛选比率较稳定的，取前10%，最多2k个
|
||||
maLen := 17
|
||||
stepMa := 8 //对应480分钟
|
||||
ts := make([]*IJV, 0, len(exsList)*2)
|
||||
for i := 0; i < len(exsList); i++ {
|
||||
barsA := barList[i]
|
||||
for j := i + 1; j < len(exsList); j++ {
|
||||
barsB := barList[j]
|
||||
barsRes := getAlignBars(barsA, barsB)
|
||||
if len(barsRes) == 0 {
|
||||
continue
|
||||
}
|
||||
a, b := barsRes[0], barsRes[1]
|
||||
if len(a) <= 100 {
|
||||
continue
|
||||
}
|
||||
slopes := make([]float64, 0, 90)
|
||||
sum := float64(0)
|
||||
avgs := make([]float64, 0, len(a))
|
||||
for m := 0; m < len(a); m++ {
|
||||
sum += a[m].Close / b[m].Close
|
||||
if m >= maLen {
|
||||
sum -= a[m-maLen].Close / b[m-maLen].Close
|
||||
avg := sum / float64(maLen)
|
||||
avgs = append(avgs, avg)
|
||||
if len(avgs)%stepMa == 1 && len(avgs) > stepMa {
|
||||
slopes = append(slopes, math.Abs(avg/avgs[len(avgs)-stepMa-1]-1)*100)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(slopes) < 20 {
|
||||
continue
|
||||
}
|
||||
avgSlope := floats.Sum(slopes) / float64(len(slopes))
|
||||
if avgSlope < 0.05 {
|
||||
continue
|
||||
}
|
||||
ts = append(ts, &IJV{I: i, J: j, V: avgSlope})
|
||||
}
|
||||
}
|
||||
sort.Slice(ts, func(i, j int) bool {
|
||||
return ts[i].V < ts[j].V
|
||||
})
|
||||
tops := ts[:min(2000, int(float64(len(ts))*0.1))]
|
||||
log.Info("cut top tuples", zap.Int("num", len(tops)), zap.Int("all", len(ts)))
|
||||
var tuples = make([][]string, 0, len(tops))
|
||||
its := make([]*com.CodeVal, 0, len(tops))
|
||||
for _, t := range tops {
|
||||
ca, cb := exsList[t.I].Symbol, exsList[t.J].Symbol
|
||||
tuples = append(tuples, []string{ca, cb})
|
||||
codeA, _, _, _ := core.SplitSymbol(ca)
|
||||
codeB, _, _, _ := core.SplitSymbol(cb)
|
||||
if codeA > codeB {
|
||||
codeA, codeB = codeB, codeA
|
||||
}
|
||||
its = append(its, &com.CodeVal{Code: codeA + "/" + codeB, Val: t.V})
|
||||
}
|
||||
printCodeVals(its)
|
||||
sta := newStatTwo(jobs, tuples)
|
||||
barNum := 0
|
||||
// 输出csv信息
|
||||
var datas = make(map[string][]float64)
|
||||
// 每行表示一对；每列表示一个时间帧
|
||||
var rows = make(map[string][]float64)
|
||||
colCount := 0 // 列数量，即时间采样数量
|
||||
// 进度条
|
||||
totalSecs := (endMS - config.TimeRange.StartMS) / 1000
|
||||
pbar := utils.NewPrgBar(int(totalSecs), "Run")
|
||||
defer pbar.Close()
|
||||
onStaBatch := func(arr []*Tuple) {
|
||||
if barNum%500 != 0 && barNum < 1800 {
|
||||
return
|
||||
}
|
||||
colCount += 1
|
||||
for _, it := range arr {
|
||||
code := it.env.Symbol
|
||||
if math.IsNaN(it.maSlope) {
|
||||
continue
|
||||
}
|
||||
absVal := math.Abs(it.maSlope * 100)
|
||||
|
||||
old, _ := datas[code]
|
||||
datas[code] = append(old, absVal)
|
||||
|
||||
col, _ := rows[code]
|
||||
if len(col) < colCount {
|
||||
col = append(col, make([]float64, colCount-len(col))...)
|
||||
}
|
||||
rows[code] = append(col, absVal)
|
||||
}
|
||||
}
|
||||
// Kline回调
|
||||
onBar := func(b *orm.InfoKline, nexts []*orm.InfoKline) {
|
||||
if b.Time > sta.barMS {
|
||||
if !b.IsWarmUp && sta.barMS > 0 {
|
||||
pbar.Add(int((b.Time - sta.barMS) / 1000))
|
||||
}
|
||||
barNum += 1
|
||||
sta.barMS = b.Time
|
||||
arr := sta.popTuples()
|
||||
if !b.IsWarmUp {
|
||||
onStaBatch(arr)
|
||||
}
|
||||
}
|
||||
sta.OnPrice(b.Symbol, b.Close)
|
||||
}
|
||||
var replayErr *errs.Error
|
||||
verCh := make(chan int, 1)
|
||||
onData := func(evt *orm.DataSeries, _ []*orm.DataSeries) {
|
||||
bar, err_ := orm.AsKline(evt)
|
||||
if err_ != nil {
|
||||
replayErr = errs.New(errs.CodeRunTime, err_)
|
||||
select {
|
||||
case verCh <- -1:
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
onBar(bar, nil)
|
||||
}
|
||||
err = biz.RunHistSeries(&biz.RunHistSeriesArgs{
|
||||
ExsList: exsList,
|
||||
Start: config.TimeRange.StartMS,
|
||||
End: config.TimeRange.EndMS,
|
||||
VerCh: verCh,
|
||||
OnData: onData,
|
||||
TfWarms: map[string]int{tf: 0}, // 无需预热，必须所有bar一同运行
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if replayErr != nil {
|
||||
return replayErr
|
||||
}
|
||||
// 计算平均大趋势斜率
|
||||
var items = make([]*com.CodeVal, 0, len(datas))
|
||||
var avgs = make(map[string]float64)
|
||||
for code, arr := range datas {
|
||||
avgVal := floats.Sum(arr) / float64(len(arr))
|
||||
items = append(items, &com.CodeVal{Code: code, Val: avgVal})
|
||||
avgs[code] = avgVal
|
||||
}
|
||||
printCodeVals(items)
|
||||
// 生成csv文件
|
||||
outRows := make([][]string, 0)
|
||||
for key, row := range rows {
|
||||
if len(row) < colCount {
|
||||
row = append(row, make([]float64, colCount-len(row))...)
|
||||
rows[key] = row
|
||||
}
|
||||
out := make([]string, 0, colCount+2)
|
||||
out = append(out, key, strconv.FormatFloat(avgs[key], 'f', 2, 64))
|
||||
for _, v := range row {
|
||||
out = append(out, strconv.FormatFloat(v, 'f', 2, 64))
|
||||
}
|
||||
outRows = append(outRows, out)
|
||||
}
|
||||
return utils.WriteCsvFile("D:/slopes.csv", outRows, false)
|
||||
}
|
||||
|
||||
func getExsBars(exchange banexg.BanExchange, exsMap map[int32]*orm.ExSymbol, tf string, startMS, endMS int64, limit int) ([]*orm.ExSymbol, [][]*banexg.Kline, map[string]*strat.StratJob, *errs.Error) {
|
||||
_, err := orm.LoadMarkets(exg.Default, false)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
err = orm.BulkDownOHLCV(exchange, exsMap, tf, startMS, endMS, limit, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
exsList := make([]*orm.ExSymbol, 0, len(exsMap))
|
||||
barList := make([][]*banexg.Kline, 0, len(exsMap))
|
||||
jobs := make(map[string]*strat.StratJob)
|
||||
for _, exs := range exsMap {
|
||||
_, quote, _, ident := core.SplitSymbol(exs.Symbol)
|
||||
if quote != "USDT" || ident != "" {
|
||||
continue
|
||||
}
|
||||
_, bars, err := orm.GetOHLCV(exs, tf, startMS, endMS, limit, false)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
exsList = append(exsList, exs)
|
||||
barList = append(barList, bars)
|
||||
jobs[exs.Symbol] = &strat.StratJob{Symbol: exs}
|
||||
}
|
||||
return exsList, barList, jobs, nil
|
||||
}
|
||||
|
||||
func getAlignBars(items ...[]*banexg.Kline) [][]*banexg.Kline {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(items) == 1 {
|
||||
return items
|
||||
}
|
||||
first := items[0]
|
||||
if len(first) == 0 {
|
||||
return nil
|
||||
}
|
||||
headLen, headStart, headStop := len(first), first[0].Time, first[len(first)-1].Time
|
||||
allAlign, minLen := true, headLen
|
||||
for _, a := range items {
|
||||
if len(a) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(a) != headLen || a[0].Time != headStart || a[len(a)-1].Time != headStop {
|
||||
allAlign = false
|
||||
}
|
||||
minLen = min(minLen, len(a))
|
||||
}
|
||||
if allAlign {
|
||||
return items
|
||||
}
|
||||
var resList = make([][]*banexg.Kline, len(items))
|
||||
var indexs = make([]int, len(items))
|
||||
started := false
|
||||
reachEnd := false
|
||||
for {
|
||||
stampCounts := make(map[int64]int)
|
||||
maxNum, mostTime, maxTime := 0, int64(0), int64(0)
|
||||
for i, idx := range indexs {
|
||||
b := items[i][idx]
|
||||
num, _ := stampCounts[b.Time]
|
||||
stampCounts[b.Time] = num + 1
|
||||
if num >= maxNum {
|
||||
maxNum, mostTime = num+1, b.Time
|
||||
}
|
||||
maxTime = max(maxTime, b.Time)
|
||||
}
|
||||
idxChange := false
|
||||
if len(stampCounts) > 1 {
|
||||
// 至少2个不同时间戳，将较旧的移动
|
||||
if maxNum <= len(indexs)/2 {
|
||||
mostTime = maxTime
|
||||
}
|
||||
for i, idx := range indexs {
|
||||
arr := items[i]
|
||||
for arr[idx].Time < mostTime {
|
||||
idx += 1
|
||||
if idx >= minLen {
|
||||
reachEnd = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx > indexs[i] {
|
||||
idxChange = true
|
||||
indexs[i] = idx
|
||||
}
|
||||
}
|
||||
} else if !started {
|
||||
started = true
|
||||
}
|
||||
if started && maxNum >= len(indexs)/2 {
|
||||
// 已开始，maxNum超过一半时有效
|
||||
for i, idx := range indexs {
|
||||
b := items[i][idx]
|
||||
res := resList[i]
|
||||
if b.Time > mostTime {
|
||||
b = res[len(res)-1]
|
||||
} else if b.Time < mostTime {
|
||||
panic("bad logic")
|
||||
} else {
|
||||
indexs[i] = idx + 1
|
||||
idxChange = true
|
||||
if idx+1 >= minLen {
|
||||
reachEnd = true
|
||||
}
|
||||
}
|
||||
resList[i] = append(res, b)
|
||||
}
|
||||
}
|
||||
if !idxChange {
|
||||
for i, idx := range indexs {
|
||||
arr := items[i]
|
||||
if arr[idx].Time <= mostTime {
|
||||
idx += 1
|
||||
indexs[i] = idx
|
||||
if idx >= minLen {
|
||||
reachEnd = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if reachEnd {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(resList[0]) == 0 {
|
||||
return nil
|
||||
}
|
||||
return resList
|
||||
}
|
||||
|
||||
func printCodeVals(items []*com.CodeVal) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
return items[i].Val < items[j].Val
|
||||
})
|
||||
var b strings.Builder
|
||||
for _, it := range items {
|
||||
b.WriteString(it.Code)
|
||||
b.WriteString(": ")
|
||||
b.WriteString(strconv.FormatFloat(it.Val, 'f', 2, 64))
|
||||
b.WriteString(", ")
|
||||
}
|
||||
fmt.Printf(b.String())
|
||||
fmt.Printf("\n")
|
||||
}
|
||||
Reference in New Issue
Block a user