update

cmd/cbpred-export/main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -64,12 +65,21 @@ func main() {
 			}
 			fail(err)
 		}
-		_, rows, err := orm.GetSeries(exs, *timeframe, *startMS, *endMS, 0, false)
+		_, rows, err := orm.GetSeries(exs, "1h", *startMS, *endMS, 0, false)
 		if err != nil {
 			fail(err)
 		}
+		bars := make([]*orm.SeriesOHLCV, 0, len(rows))
+		for _, row := range rows {
+			bar, err := row.OHLCV(exs)
+			if err != nil {
+				fail(err)
+			}
+			bars = append(bars, bar)
+		}
+		bars = aggregateHourly(bars, *timeframe)
 		name := fmt.Sprintf("%d_%s.csv", exs.ID, strings.NewReplacer("/", "_", ":", "_").Replace(pair))
-		if err := writeCSV(filepath.Join(*outDir, name), rows, exs); err != nil {
+		if err := writeCSV(filepath.Join(*outDir, name), bars); err != nil {
 			fail(err)
 		}
 	}
@@ -79,7 +89,47 @@ func isUnavailablePair(err *errs.Error) bool {
 	return err != nil && err.Code == errs.CodeNoMarketForPair
 }
 
-func writeCSV(path string, rows []*orm.DataSeries, exs *orm.ExSymbol) error {
+// aggregateHourly emits only complete UTC buckets with exactly one bar per hour.
+func aggregateHourly(rows []*orm.SeriesOHLCV, timeframe string) []*orm.SeriesOHLCV {
+	const hourMS int64 = 60 * 60 * 1000
+	hours := 8
+	if timeframe == "1d" {
+		hours = 24
+	}
+	stepMS := int64(hours) * hourMS
+	ordered := append([]*orm.SeriesOHLCV(nil), rows...)
+	sort.Slice(ordered, func(i, j int) bool { return ordered[i].Time < ordered[j].Time })
+	var output []*orm.SeriesOHLCV
+	for first := 0; first < len(ordered); {
+		bucket := ordered[first].Time / stepMS * stepMS
+		end := first + 1
+		for end < len(ordered) && ordered[end].Time/stepMS*stepMS == bucket {
+			end++
+		}
+		complete := end-first == hours
+		for i := first; complete && i < end; i++ {
+			complete = ordered[i].Time == bucket+int64(i-first)*hourMS
+		}
+		if complete {
+			bar := *ordered[first]
+			bar.Time, bar.EndMS, bar.TimeFrame, bar.Closed = bucket, bucket+stepMS, timeframe, true
+			for _, row := range ordered[first+1 : end] {
+				bar.High = max(bar.High, row.High)
+				bar.Low = min(bar.Low, row.Low)
+				bar.Close = row.Close
+				bar.Volume += row.Volume
+				bar.Quote += row.Quote
+				bar.BuyVolume += row.BuyVolume
+				bar.TradeNum += row.TradeNum
+			}
+			output = append(output, &bar)
+		}
+		first = end
+	}
+	return output
+}
+
+func writeCSV(path string, rows []*orm.SeriesOHLCV) error {
 	file, err := os.Create(path)
 	if err != nil {
 		return err
@@ -89,11 +139,7 @@ func writeCSV(path string, rows []*orm.DataSeries, exs *orm.ExSymbol) error {
 	if err := writer.Write(columns); err != nil {
 		return err
 	}
-	for _, row := range rows {
-		bar, err := row.OHLCV(exs)
-		if err != nil {
-			return err
-		}
+	for _, bar := range rows {
 		values := []string{
 			strconv.FormatInt(bar.Time, 10),
 			strconv.FormatFloat(bar.Open, 'f', -1, 64),

cmd/cbpred-export/main_test.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"encoding/csv"
+	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
 
+	"github.com/banbox/banbot/orm"
 	"github.com/banbox/banexg/errs"
 )
 
@@ -14,3 +19,93 @@ func TestIsUnavailablePair(t *testing.T) {
 		t.Fatal("non-market errors must not be skipped")
 	}
 }
+
+const testHourMS int64 = 3600000
+
+func hourlyBars(startHour, count int) []*orm.SeriesOHLCV {
+	rows := make([]*orm.SeriesOHLCV, 0, count)
+	for i := startHour; i < startHour+count; i++ {
+		rows = append(rows, &orm.SeriesOHLCV{
+			Time: int64(i) * testHourMS, EndMS: int64(i+1) * testHourMS,
+			Open: float64(i + 10), High: float64(i + 20), Low: float64(i), Close: float64(i + 12),
+			Volume: 2, Quote: 3, BuyVolume: 1, TradeNum: 4,
+		})
+	}
+	return rows
+}
+
+func TestAggregateHourlyDailyAndCSV(t *testing.T) {
+	// Reversed inputs still produce UTC-aligned daily history in time order.
+	rows := hourlyBars(0, 72)
+	for i, j := 0, len(rows)-1; i < j; i, j = i+1, j-1 {
+		rows[i], rows[j] = rows[j], rows[i]
+	}
+	bars := aggregateHourly(rows, "1d")
+	if len(bars) != 3 {
+		t.Fatalf("daily bars = %d, want 3", len(bars))
+	}
+	for i, bar := range bars {
+		if bar.Time != int64(i*24)*testHourMS || bar.EndMS != int64((i+1)*24)*testHourMS || bar.TimeFrame != "1d" || !bar.Closed {
+			t.Fatalf("unexpected daily metadata: %+v", bar)
+		}
+	}
+	path := filepath.Join(t.TempDir(), "daily.csv")
+	if err := writeCSV(path, bars); err != nil {
+		t.Fatal(err)
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	contents, err := csv.NewReader(file).ReadAll()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(contents[0], columns) {
+		t.Fatalf("unexpected CSV columns: %v", contents[0])
+	}
+	want := []string{"0", "10", "43", "0", "35", "48", "72", "24", "96"}
+	if !reflect.DeepEqual(contents[1], want) {
+		t.Fatalf("CSV first bar = %v, want %v", contents[1], want)
+	}
+}
+
+func TestAggregateHourlyEightHour(t *testing.T) {
+	bars := aggregateHourly(hourlyBars(0, 24), "8h")
+	if len(bars) != 3 {
+		t.Fatalf("eight-hour bars = %d, want 3", len(bars))
+	}
+	for i, bar := range bars {
+		if bar.Time != int64(i*8)*testHourMS || bar.Open != float64(i*8+10) || bar.Close != float64(i*8+19) || bar.Volume != 16 || bar.Quote != 24 || bar.BuyVolume != 8 || bar.TradeNum != 32 {
+			t.Fatalf("unexpected eight-hour bar: %+v", bar)
+		}
+	}
+}
+
+func TestAggregateHourlyRejectsIncompleteBuckets(t *testing.T) {
+	missing := hourlyBars(0, 48)
+	missing = append(missing[:12:12], missing[13:]...)
+	gapWithDuplicate := hourlyBars(0, 48)
+	gapWithDuplicate[12] = gapWithDuplicate[11]
+	duplicate := hourlyBars(0, 48)
+	duplicate = append(duplicate, duplicate[12])
+	for name, rows := range map[string][]*orm.SeriesOHLCV{
+		"missing hour":          missing,
+		"gap with duplicate":    gapWithDuplicate,
+		"extra duplicate":       duplicate,
+		"partial start and end": hourlyBars(1, 70),
+		"misaligned hour":       append(hourlyBars(0, 23), &orm.SeriesOHLCV{Time: 23*testHourMS + 1}),
+	} {
+		t.Run(name, func(t *testing.T) {
+			bars := aggregateHourly(rows, "1d")
+			want := 1
+			if name == "misaligned hour" {
+				want = 0
+			}
+			if len(bars) != want || (want == 1 && bars[0].Time != 24*testHourMS) {
+				t.Fatalf("unexpected complete buckets: %+v", bars)
+			}
+		})
+	}
+}