add unit test, csv/hst is working

bi5/bi5.go

@@ -38,9 +38,12 @@ type Bi5 struct {
 
 // New create an bi5 saver
 func New(day time.Time, symbol, dest string) *Bi5 {
+	y, m, d := day.Date()
+	dir := fmt.Sprintf("%s/%04d/%02d/%02d", symbol, y, m, d)
+
 	return &Bi5{
+		dest:   filepath.Join(dest, dir),
 		dayH:   day,
-		dest:   dest,
 		symbol: symbol,
 	}
 }
@@ -84,17 +87,13 @@ func (b *Bi5) Save(data []byte) error {
 		return nil
 	}
 
-	y, m, d := b.dayH.Date()
-	subDir := fmt.Sprintf("%s/%04d/%02d/%02d", b.symbol, y, m, d)
-
-	fpath := filepath.Join(b.dest, subDir)
-	if err := os.MkdirAll(fpath, 666); err != nil {
-		log.Error("Create folder (%s) failed: %v.", fpath, err)
+	if err := os.MkdirAll(b.dest, 666); err != nil {
+		log.Error("Create folder (%s) failed: %v.", b.dest, err)
 		return err
 	}
 
 	fname := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext)
-	fpath = filepath.Join(fpath, fname)
+	fpath := filepath.Join(b.dest, fname)
 
 	f, err := os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 666)
 	if err != nil {
@@ -115,8 +114,9 @@ func (b *Bi5) Save(data []byte) error {
 // Load bi5 data from file content
 //
 func (b *Bi5) Load() ([]byte, error) {
-	subpath := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext)
-	fpath := filepath.Join(b.dest, subpath)
+
+	fname := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext)
+	fpath := filepath.Join(b.dest, fname)
 
 	f, err := os.OpenFile(fpath, os.O_RDONLY, 666)
 	if err != nil {

bi5/bi5_test.go

@@ -0,0 +1,60 @@
+package bi5
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestLoadBi5(t *testing.T) {
+	//fname := `F:\00\EURUSD\2017\01\01\22h.bi5`
+	dest := `F:\00`
+
+	day, err := time.ParseInLocation("2006-01-02 15", "2017-01-01 22", time.UTC)
+	if err != nil {
+		t.Fatalf("Invalid date format\n")
+	}
+
+	fb := New(day, "EURUSD", dest)
+	bs, err := fb.Load()
+	if err != nil {
+		t.Fatalf("Load bi5 failed: %v.\n", err)
+	}
+
+	ticks, err := fb.Decode(bs[:])
+	if err != nil {
+		t.Fatalf("Decode bi5 failed: %v.\n", err)
+	}
+
+	for idx, tick := range ticks {
+		fmt.Printf("%d:  %v\n", idx, tick)
+	}
+}
+
+func TestDownloadBi5(t *testing.T) {
+	//fname := `F:\00\EURUSD\2017\01\01\22h.bi5`
+	dest := `F:\test01`
+
+	day, err := time.ParseInLocation("2006-01-02 15", "2017-01-01 22", time.UTC)
+	if err != nil {
+		t.Fatalf("Invalid date format\n")
+	}
+
+	fb := New(day, "EURUSD", dest)
+	bs, err := fb.Download()
+	if err != nil {
+		t.Fatalf("Load bi5 failed: %v.\n", err)
+	}
+
+	defer fb.Save(bs[:])
+
+	ticks, err := fb.Decode(bs[:])
+	if err != nil {
+		t.Fatalf("Decode bi5 failed: %v.\n", err)
+	}
+
+	for idx, tick := range ticks {
+		fmt.Printf("%d:  %v\n", idx, tick)
+	}
+
+}

convert.go → core/convert.go

@@ -1,19 +1,13 @@
-package main
+package core
 
 import (
 	"regexp"
 	"strconv"
-	"strings"
-
-	"github.com/adyzng/go-duka/core"
-	"github.com/adyzng/go-duka/csv"
-	"github.com/adyzng/go-duka/fxt4"
-	"github.com/adyzng/go-duka/hst"
 )
 
 var (
-	tfRegexp = regexp.MustCompile(`(M|H|D|W|MN)(\d+)`)
-	tfMinute = map[string]uint32{
+	TimeframeRegx = regexp.MustCompile(`(M|H|D|W|MN)(\d+)`)
+	tfMinute      = map[string]uint32{
 		"M":  1,
 		"H":  60,
 		"D":  24 * 60,
@@ -22,32 +16,6 @@ var (
 	}
 )
 
-func NewOutputs(opt *AppOption) []core.Converter {
-	outs := make([]core.Converter, 0)
-	for _, period := range strings.Split(opt.Periods, ",") {
-		var format core.Converter
-		timeframe, _ := ParseTimeframe(strings.Trim(period, " \t\r\n"))
-
-		switch opt.Format {
-		case "csv":
-			format = csv.New(opt.Start, opt.End, opt.CsvHeader, opt.Symbol, opt.Folder)
-			break
-		case "fxt":
-			format = fxt4.NewFxtFile(timeframe, opt.Spread, opt.Mode, opt.Folder, opt.Symbol)
-			break
-		case "hst":
-			format = hst.NewHST(timeframe, opt.Spread, opt.Symbol, opt.Folder)
-			break
-		default:
-			log.Error("unsupported format %s.", opt.Format)
-			return nil
-		}
-
-		outs = append(outs, NewTimeframe(period, opt.Symbol, format))
-	}
-	return outs
-}
-
 // Timeframe wrapper of tick data in timeframe like: M1, M5, M15, M30, H1, H4, D1, W1, MN
 //
 type Timeframe struct {
@@ -58,16 +26,16 @@ type Timeframe struct {
 	period         string // M1, M5, M15, M30, H1, H4, D1, W1, MN
 	symbol         string
 
-	chTicks chan *core.TickData
+	chTicks chan *TickData
 	close   chan struct{}
-	out     core.Converter
+	out     Converter
 }
 
 // ParseTimeframe from input string
 //
 func ParseTimeframe(period string) (uint32, string) {
 	// M15 => [M15 M 15]
-	if ss := tfRegexp.FindStringSubmatch(period); len(ss) == 3 {
+	if ss := TimeframeRegx.FindStringSubmatch(period); len(ss) == 3 {
 		n, _ := strconv.Atoi(ss[2])
 		for key, val := range tfMinute {
 			if key == ss[1] {
@@ -79,15 +47,15 @@ func ParseTimeframe(period string) (uint32, string) {
 }
 
 // NewTimeframe create an new timeframe
-func NewTimeframe(period, symbol string, out core.Converter) core.Converter {
+func NewTimeframe(period, symbol string, out Converter) Converter {
 	min, str := ParseTimeframe(period)
 	tf := &Timeframe{
 		deltaTimestamp: min * 60,
 		timeframe:      min,
 		period:         str,
 		symbol:         symbol,
 		out:            out,
-		chTicks:        make(chan *core.TickData, 1024),
+		chTicks:        make(chan *TickData, 1024),
 		close:          make(chan struct{}, 1),
 	}
 
@@ -96,7 +64,7 @@ func NewTimeframe(period, symbol string, out core.Converter) core.Converter {
 }
 
 // PackTicks receive original tick data
-func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*core.TickData) error {
+func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*TickData) error {
 	for _, tick := range ticks {
 		select {
 		case tf.chTicks <- tick:
@@ -116,7 +84,7 @@ func (tf *Timeframe) Finish() error {
 // worker thread
 func (tf *Timeframe) worker() error {
 	maxCap := 1024
-	barTicks := make([]*core.TickData, 0, maxCap)
+	barTicks := make([]*TickData, 0, maxCap)
 
 	defer func() {
 		log.Info("%s %s convert completed.", tf.symbol, tf.period)

core/tick.go

@@ -23,21 +23,9 @@ func (t *TickData) UTC() time.Time {
 	return tm.UTC()
 }
 
-// BarData means tick data within one Bar
+// Strings used to format into csv row
 //
-type BarData struct {
-	TickTimestamp uint32  // second
-	BarTimestamp  uint32  // second
-	Open          float64 // OLHCV
-	Low           float64 //
-	High          float64 //
-	Close         float64 //
-	Volume        uint64  //
-}
-
-// ToString used to format into csv row
-//
-func (t *TickData) ToString() []string {
+func (t *TickData) Strings() []string {
 	return []string{
 		t.UTC().Format("2006-01-02 15:04:05.000"),
 		fmt.Sprintf("%.5f", t.Ask),
@@ -46,3 +34,13 @@ func (t *TickData) ToString() []string {
 		fmt.Sprintf("%.2f", t.VolumeBid),
 	}
 }
+
+func (t *TickData) String() string {
+	return fmt.Sprintf("%s %.5f %.5f %.2f %.2f",
+		t.UTC().Format("2006-01-02 15:04:06.000"),
+		t.Ask,
+		t.Bid,
+		t.VolumeAsk,
+		t.VolumeBid,
+	)
+}

csv/csv.go

@@ -98,7 +98,7 @@ func (c *CsvDump) worker() error {
 
 	// write tick one by one
 	for tick := range c.chTicks {
-		if err = csv.Write(tick.ToString()); err != nil {
+		if err = csv.Write(tick.Strings()); err != nil {
 			log.Error("Write csv %s failed: %v.", fpath, err)
 			break
 		}

csv/csv_test.go

@@ -1,8 +1,11 @@
 package csv
 
 import (
+	"fmt"
 	"testing"
 	"time"
+
+	"github.com/adyzng/go-duka/bi5"
 )
 
 func TestCloseChan(t *testing.T) {
@@ -16,3 +19,37 @@ func TestCloseChan(t *testing.T) {
 	<-chClose
 	t.Logf("Receive close channel.\n")
 }
+
+func TestDumpCsv(t *testing.T) {
+	dest := `F:\00`
+	symbol := "EURUSD"
+
+	day, err := time.ParseInLocation("2006-01-02", "2017-01-02", time.UTC)
+	if err != nil {
+		t.Fatalf("Invalid date format\n")
+	}
+
+	csv := New(day, day, true, symbol, dest)
+	defer csv.Finish()
+
+	for h := 0; h < 24; h++ {
+		dayH := day.Add(time.Duration(h) * time.Hour)
+		fb := bi5.New(dayH, symbol, dest)
+
+		bs, err := fb.Load()
+		if err != nil {
+			t.Fatalf("Load bi5 failed: %v.\n", err)
+		}
+
+		ticks, err := fb.Decode(bs[:])
+		if err != nil {
+			t.Fatalf("Decode bi5 failed: %v.\n", err)
+		}
+
+		for idx, tick := range ticks {
+			fmt.Printf("%d:  %v\n", idx, tick)
+		}
+
+		csv.PackTicks(0, ticks)
+	}
+}

duka.go

@@ -112,9 +112,10 @@ func main() {
 	fmt.Printf("    Symbol: %s\n", opt.Symbol)
 	fmt.Printf("    Spread: %d\n", opt.Spread)
 	fmt.Printf("      Mode: %d\n", opt.Mode)
-	fmt.Printf(" Timeframe: %d\n", opt.Timeframe)
+	fmt.Printf(" Timeframe: %s\n", opt.Periods)
 	fmt.Printf("    Format: %s\n", opt.Format)
 	fmt.Printf(" CsvHeader: %t\n", opt.CsvHeader)
+	fmt.Printf(" LocalData: %t\n", opt.Convert)
 	fmt.Printf(" StartDate: %s\n", opt.Start.Format("2006-01-02:15H"))
 	fmt.Printf("   EndDate: %s\n", opt.End.Format("2006-01-02:15H"))
 

duka_test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"testing"
 
-	clog "gopkg.in/clog.v1"
+	clog "github.com/go-clog/clog"
 )
 
 func TestDukaApp(t *testing.T) {
@@ -14,7 +14,7 @@ func TestDukaApp(t *testing.T) {
 		Spread:  20,
 		Model:   0,
 		Symbol:  "EURUSD",
-		Output:  "g:\\00",
+		Output:  "f:\\00",
 		Format:  "csv",
 		Period:  "M1",
 		Start:   "2017-01-01",