From eec5d165dfcb0c5b43886cb0e0f9f32930cec7ec Mon Sep 17 00:00:00 2001 From: huan Date: Wed, 27 Dec 2017 22:24:43 +0800 Subject: [PATCH] add unit test, csv/hst is working --- bi5/bi5.go | 20 +++--- bi5/bi5_test.go | 60 +++++++++++++++++ convert.go => core/convert.go | 52 +++------------ core/tick.go | 26 ++++---- csv/csv.go | 2 +- csv/csv_test.go | 37 +++++++++++ duka.go | 3 +- duka_test.go | 4 +- app.go => dukapp.go | 83 +++++++++++++++++------ fxt4/{fxt.go => fxt4.go} | 58 ++++++---------- fxt4/fxt4_test.go | 63 ++++++++++++++++++ fxt4/fxt_test.go | 12 ---- fxt4/header.go | 121 +++++++++++----------------------- hst/header.go | 33 ++++++++-- hst/hst401.go | 11 ++-- hst/hst401_test.go | 97 +++++++++++++++++++++++++++ misc/bytes.go | 4 +- 17 files changed, 453 insertions(+), 233 deletions(-) create mode 100644 bi5/bi5_test.go rename convert.go => core/convert.go (64%) rename app.go => dukapp.go (79%) rename fxt4/{fxt.go => fxt4.go} (74%) create mode 100644 fxt4/fxt4_test.go delete mode 100644 fxt4/fxt_test.go create mode 100644 hst/hst401_test.go diff --git a/bi5/bi5.go b/bi5/bi5.go index 7884053..b0f6d01 100644 --- a/bi5/bi5.go +++ b/bi5/bi5.go @@ -38,9 +38,12 @@ type Bi5 struct { // New create an bi5 saver func New(day time.Time, symbol, dest string) *Bi5 { + y, m, d := day.Date() + dir := fmt.Sprintf("%s/%04d/%02d/%02d", symbol, y, m, d) + return &Bi5{ + dest: filepath.Join(dest, dir), dayH: day, - dest: dest, symbol: symbol, } } @@ -84,17 +87,13 @@ func (b *Bi5) Save(data []byte) error { return nil } - y, m, d := b.dayH.Date() - subDir := fmt.Sprintf("%s/%04d/%02d/%02d", b.symbol, y, m, d) - - fpath := filepath.Join(b.dest, subDir) - if err := os.MkdirAll(fpath, 666); err != nil { - log.Error("Create folder (%s) failed: %v.", fpath, err) + if err := os.MkdirAll(b.dest, 666); err != nil { + log.Error("Create folder (%s) failed: %v.", b.dest, err) return err } fname := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext) - fpath = filepath.Join(fpath, fname) + fpath := filepath.Join(b.dest, fname) f, err := os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 666) if err != nil { @@ -115,8 +114,9 @@ func (b *Bi5) Save(data []byte) error { // Load bi5 data from file content // func (b *Bi5) Load() ([]byte, error) { - subpath := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext) - fpath := filepath.Join(b.dest, subpath) + + fname := fmt.Sprintf("%02dh.%s", b.dayH.Hour(), ext) + fpath := filepath.Join(b.dest, fname) f, err := os.OpenFile(fpath, os.O_RDONLY, 666) if err != nil { diff --git a/bi5/bi5_test.go b/bi5/bi5_test.go new file mode 100644 index 0000000..dc23829 --- /dev/null +++ b/bi5/bi5_test.go @@ -0,0 +1,60 @@ +package bi5 + +import ( + "fmt" + "testing" + "time" +) + +func TestLoadBi5(t *testing.T) { + //fname := `F:\00\EURUSD\2017\01\01\22h.bi5` + dest := `F:\00` + + day, err := time.ParseInLocation("2006-01-02 15", "2017-01-01 22", time.UTC) + if err != nil { + t.Fatalf("Invalid date format\n") + } + + fb := New(day, "EURUSD", dest) + bs, err := fb.Load() + if err != nil { + t.Fatalf("Load bi5 failed: %v.\n", err) + } + + ticks, err := fb.Decode(bs[:]) + if err != nil { + t.Fatalf("Decode bi5 failed: %v.\n", err) + } + + for idx, tick := range ticks { + fmt.Printf("%d: %v\n", idx, tick) + } +} + +func TestDownloadBi5(t *testing.T) { + //fname := `F:\00\EURUSD\2017\01\01\22h.bi5` + dest := `F:\test01` + + day, err := time.ParseInLocation("2006-01-02 15", "2017-01-01 22", time.UTC) + if err != nil { + t.Fatalf("Invalid date format\n") + } + + fb := New(day, "EURUSD", dest) + bs, err := fb.Download() + if err != nil { + t.Fatalf("Load bi5 failed: %v.\n", err) + } + + defer fb.Save(bs[:]) + + ticks, err := fb.Decode(bs[:]) + if err != nil { + t.Fatalf("Decode bi5 failed: %v.\n", err) + } + + for idx, tick := range ticks { + fmt.Printf("%d: %v\n", idx, tick) + } + +} diff --git a/convert.go b/core/convert.go similarity index 64% rename from convert.go rename to core/convert.go index 7d69dcc..7006fd6 100644 --- a/convert.go +++ b/core/convert.go @@ -1,19 +1,13 @@ -package main +package core import ( "regexp" "strconv" - "strings" - - "github.com/adyzng/go-duka/core" - "github.com/adyzng/go-duka/csv" - "github.com/adyzng/go-duka/fxt4" - "github.com/adyzng/go-duka/hst" ) var ( - tfRegexp = regexp.MustCompile(`(M|H|D|W|MN)(\d+)`) - tfMinute = map[string]uint32{ + TimeframeRegx = regexp.MustCompile(`(M|H|D|W|MN)(\d+)`) + tfMinute = map[string]uint32{ "M": 1, "H": 60, "D": 24 * 60, @@ -22,32 +16,6 @@ var ( } ) -func NewOutputs(opt *AppOption) []core.Converter { - outs := make([]core.Converter, 0) - for _, period := range strings.Split(opt.Periods, ",") { - var format core.Converter - timeframe, _ := ParseTimeframe(strings.Trim(period, " \t\r\n")) - - switch opt.Format { - case "csv": - format = csv.New(opt.Start, opt.End, opt.CsvHeader, opt.Symbol, opt.Folder) - break - case "fxt": - format = fxt4.NewFxtFile(timeframe, opt.Spread, opt.Mode, opt.Folder, opt.Symbol) - break - case "hst": - format = hst.NewHST(timeframe, opt.Spread, opt.Symbol, opt.Folder) - break - default: - log.Error("unsupported format %s.", opt.Format) - return nil - } - - outs = append(outs, NewTimeframe(period, opt.Symbol, format)) - } - return outs -} - // Timeframe wrapper of tick data in timeframe like: M1, M5, M15, M30, H1, H4, D1, W1, MN // type Timeframe struct { @@ -58,16 +26,16 @@ type Timeframe struct { period string // M1, M5, M15, M30, H1, H4, D1, W1, MN symbol string - chTicks chan *core.TickData + chTicks chan *TickData close chan struct{} - out core.Converter + out Converter } // ParseTimeframe from input string // func ParseTimeframe(period string) (uint32, string) { // M15 => [M15 M 15] - if ss := tfRegexp.FindStringSubmatch(period); len(ss) == 3 { + if ss := TimeframeRegx.FindStringSubmatch(period); len(ss) == 3 { n, _ := strconv.Atoi(ss[2]) for key, val := range tfMinute { if key == ss[1] { @@ -79,7 +47,7 @@ func ParseTimeframe(period string) (uint32, string) { } // NewTimeframe create an new timeframe -func NewTimeframe(period, symbol string, out core.Converter) core.Converter { +func NewTimeframe(period, symbol string, out Converter) Converter { min, str := ParseTimeframe(period) tf := &Timeframe{ deltaTimestamp: min * 60, @@ -87,7 +55,7 @@ func NewTimeframe(period, symbol string, out core.Converter) core.Converter { period: str, symbol: symbol, out: out, - chTicks: make(chan *core.TickData, 1024), + chTicks: make(chan *TickData, 1024), close: make(chan struct{}, 1), } @@ -96,7 +64,7 @@ func NewTimeframe(period, symbol string, out core.Converter) core.Converter { } // PackTicks receive original tick data -func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*core.TickData) error { +func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*TickData) error { for _, tick := range ticks { select { case tf.chTicks <- tick: @@ -116,7 +84,7 @@ func (tf *Timeframe) Finish() error { // worker thread func (tf *Timeframe) worker() error { maxCap := 1024 - barTicks := make([]*core.TickData, 0, maxCap) + barTicks := make([]*TickData, 0, maxCap) defer func() { log.Info("%s %s convert completed.", tf.symbol, tf.period) diff --git a/core/tick.go b/core/tick.go index 8e8fd9e..4e4fbec 100644 --- a/core/tick.go +++ b/core/tick.go @@ -23,21 +23,9 @@ func (t *TickData) UTC() time.Time { return tm.UTC() } -// BarData means tick data within one Bar +// Strings used to format into csv row // -type BarData struct { - TickTimestamp uint32 // second - BarTimestamp uint32 // second - Open float64 // OLHCV - Low float64 // - High float64 // - Close float64 // - Volume uint64 // -} - -// ToString used to format into csv row -// -func (t *TickData) ToString() []string { +func (t *TickData) Strings() []string { return []string{ t.UTC().Format("2006-01-02 15:04:05.000"), fmt.Sprintf("%.5f", t.Ask), @@ -46,3 +34,13 @@ func (t *TickData) ToString() []string { fmt.Sprintf("%.2f", t.VolumeBid), } } + +func (t *TickData) String() string { + return fmt.Sprintf("%s %.5f %.5f %.2f %.2f", + t.UTC().Format("2006-01-02 15:04:06.000"), + t.Ask, + t.Bid, + t.VolumeAsk, + t.VolumeBid, + ) +} diff --git a/csv/csv.go b/csv/csv.go index 1bae432..5eb0c1e 100644 --- a/csv/csv.go +++ b/csv/csv.go @@ -98,7 +98,7 @@ func (c *CsvDump) worker() error { // write tick one by one for tick := range c.chTicks { - if err = csv.Write(tick.ToString()); err != nil { + if err = csv.Write(tick.Strings()); err != nil { log.Error("Write csv %s failed: %v.", fpath, err) break } diff --git a/csv/csv_test.go b/csv/csv_test.go index 2c529d2..4af1887 100644 --- a/csv/csv_test.go +++ b/csv/csv_test.go @@ -1,8 +1,11 @@ package csv import ( + "fmt" "testing" "time" + + "github.com/adyzng/go-duka/bi5" ) func TestCloseChan(t *testing.T) { @@ -16,3 +19,37 @@ func TestCloseChan(t *testing.T) { <-chClose t.Logf("Receive close channel.\n") } + +func TestDumpCsv(t *testing.T) { + dest := `F:\00` + symbol := "EURUSD" + + day, err := time.ParseInLocation("2006-01-02", "2017-01-02", time.UTC) + if err != nil { + t.Fatalf("Invalid date format\n") + } + + csv := New(day, day, true, symbol, dest) + defer csv.Finish() + + for h := 0; h < 24; h++ { + dayH := day.Add(time.Duration(h) * time.Hour) + fb := bi5.New(dayH, symbol, dest) + + bs, err := fb.Load() + if err != nil { + t.Fatalf("Load bi5 failed: %v.\n", err) + } + + ticks, err := fb.Decode(bs[:]) + if err != nil { + t.Fatalf("Decode bi5 failed: %v.\n", err) + } + + for idx, tick := range ticks { + fmt.Printf("%d: %v\n", idx, tick) + } + + csv.PackTicks(0, ticks) + } +} diff --git a/duka.go b/duka.go index 0fa4e84..76b4747 100644 --- a/duka.go +++ b/duka.go @@ -112,9 +112,10 @@ func main() { fmt.Printf(" Symbol: %s\n", opt.Symbol) fmt.Printf(" Spread: %d\n", opt.Spread) fmt.Printf(" Mode: %d\n", opt.Mode) - fmt.Printf(" Timeframe: %d\n", opt.Timeframe) + fmt.Printf(" Timeframe: %s\n", opt.Periods) fmt.Printf(" Format: %s\n", opt.Format) fmt.Printf(" CsvHeader: %t\n", opt.CsvHeader) + fmt.Printf(" LocalData: %t\n", opt.Convert) fmt.Printf(" StartDate: %s\n", opt.Start.Format("2006-01-02:15H")) fmt.Printf(" EndDate: %s\n", opt.End.Format("2006-01-02:15H")) diff --git a/duka_test.go b/duka_test.go index fa2fc52..1b75a32 100644 --- a/duka_test.go +++ b/duka_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - clog "gopkg.in/clog.v1" + clog "github.com/go-clog/clog" ) func TestDukaApp(t *testing.T) { @@ -14,7 +14,7 @@ func TestDukaApp(t *testing.T) { Spread: 20, Model: 0, Symbol: "EURUSD", - Output: "g:\\00", + Output: "f:\\00", Format: "csv", Period: "M1", Start: "2017-01-01", diff --git a/app.go b/dukapp.go similarity index 79% rename from app.go rename to dukapp.go index 000ba48..73906a7 100644 --- a/app.go +++ b/dukapp.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "path/filepath" @@ -11,6 +12,9 @@ import ( "github.com/adyzng/go-duka/bi5" "github.com/adyzng/go-duka/core" + "github.com/adyzng/go-duka/csv" + "github.com/adyzng/go-duka/fxt4" + "github.com/adyzng/go-duka/hst" "github.com/adyzng/go-duka/misc" ) @@ -29,25 +33,19 @@ type DukaApp struct { // AppOption download options // type AppOption struct { - Start time.Time - End time.Time - Symbol string - Format string - Folder string - Periods string - Spread uint32 - Mode uint32 - Timeframe uint32 + Start time.Time + End time.Time + Symbol string + Format string + Folder string + Periods string + Spread uint32 + Mode uint32 + //Timeframe uint32 Convert bool CsvHeader bool } -type hReader struct { - Bi5 *bi5.Bi5 - DayH time.Time - Data []byte -} - // ParseOption parse input command line // func ParseOption(args argsList) (*AppOption, error) { @@ -59,6 +57,7 @@ func ParseOption(args argsList) (*AppOption, error) { err = fmt.Errorf("Invalid symbol parameter") return nil, err } + // check format { bSupport, format := false, strings.ToLower(args.Format) for _, sformat := range supportsFormats { @@ -71,6 +70,7 @@ func ParseOption(args argsList) (*AppOption, error) { err = fmt.Errorf("not supported output format") return nil, err } + opt.Format = format } if opt.Start, err = time.ParseInLocation("2006-01-02", args.Start, time.UTC); err != nil { err = fmt.Errorf("invalid start parameter") @@ -95,7 +95,7 @@ func ParseOption(args argsList) (*AppOption, error) { if args.Period != "" { args.Period = strings.ToUpper(args.Period) - if !tfRegexp.MatchString(args.Period) { + if !core.TimeframeRegx.MatchString(args.Period) { err = fmt.Errorf("invalid timeframe value: %s", args.Period) return nil, err } @@ -104,6 +104,7 @@ func ParseOption(args argsList) (*AppOption, error) { opt.Symbol = strings.ToUpper(args.Symbol) opt.CsvHeader = args.Header + opt.Convert = args.Convert opt.Format = args.Format opt.Spread = uint32(args.Spread) opt.Mode = uint32(args.Model) @@ -111,6 +112,34 @@ func ParseOption(args argsList) (*AppOption, error) { return &opt, nil } +// NewOutputs create timeframe instance +// +func NewOutputs(opt *AppOption) []core.Converter { + outs := make([]core.Converter, 0) + for _, period := range strings.Split(opt.Periods, ",") { + var format core.Converter + timeframe, _ := core.ParseTimeframe(strings.Trim(period, " \t\r\n")) + + switch opt.Format { + case "csv": + format = csv.New(opt.Start, opt.End, opt.CsvHeader, opt.Symbol, opt.Folder) + break + case "fxt": + format = fxt4.NewFxtFile(timeframe, opt.Spread, opt.Mode, opt.Folder, opt.Symbol) + break + case "hst": + format = hst.NewHST(timeframe, opt.Spread, opt.Symbol, opt.Folder) + break + default: + log.Error("unsupported format %s.", opt.Format) + return nil + } + + outs = append(outs, core.NewTimeframe(period, opt.Symbol, format)) + } + return outs +} + // NewApp create an application instance by input arguments // func NewApp(opt *AppOption) *DukaApp { @@ -120,6 +149,12 @@ func NewApp(opt *AppOption) *DukaApp { } } +type hReader struct { + Bi5 *bi5.Bi5 + DayH time.Time + Data []byte +} + // Execute download source bi5 tick data from dukascopy // func (app *DukaApp) Execute() error { @@ -129,6 +164,11 @@ func (app *DukaApp) Execute() error { startTime = time.Now() ) + if len(app.outputs) < 1 { + log.Error("No valid output format") + return errors.New("no valid output format") + } + // // 创建输出目录 // @@ -209,12 +249,13 @@ func (app *DukaApp) fetchDay(day time.Time) <-chan *hReader { } if err != nil { - log.Error("%s, %s failed: %v.", str, dayH.Format("2006-01-02:15H")) + log.Error("%s, %s failed: %v.", str, dayH.Format("2006-01-02:15H"), err) return } if len(data) > 0 { select { case ch <- &hReader{Data: data[:], DayH: dayH, Bi5: bi5File}: + //log.Trace("%s %s", str, dayH.Format("2006-01-02:15H")) break } } @@ -222,7 +263,7 @@ func (app *DukaApp) fetchDay(day time.Time) <-chan *hReader { } wg.Wait() - log.Trace("%s %s download complete.", opt.Symbol, day.Format("2006-01-02")) + log.Trace("Bi5 %s %s complete.", opt.Symbol, day.Format("2006-01-02")) }() return ch @@ -246,9 +287,9 @@ func (app *DukaApp) sortAndOutput(day time.Time, ticks []*core.TickData) error { out.PackTicks(timestamp, ticks[:]) } - firstTick := ticks[0].Timestamp - tm := time.Unix(firstTick/1000, 0).UTC() - log.Trace("%s sort and output day %v.", app.option.Format, tm) + //firstTick := ticks[0].Timestamp + //tm := time.Unix(firstTick/1000, 0).UTC() + //log.Trace("%s sort and output day %v.", app.option.Format, tm) return nil } diff --git a/fxt4/fxt.go b/fxt4/fxt4.go similarity index 74% rename from fxt4/fxt.go rename to fxt4/fxt4.go index 5890507..f262429 100644 --- a/fxt4/fxt.go +++ b/fxt4/fxt4.go @@ -15,30 +15,6 @@ var ( log = misc.NewLogger("FXT", 3) ) -type fxtTick struct { - BarTimestamp int32 - padding int32 - Open float64 - High float64 - Low float64 - Close float64 - Volume uint64 - TickTimestamp int32 - LaunchExpert int32 -} - -func convertToFxtTick(tick *core.TickData) *fxtTick { - return &fxtTick{ - BarTimestamp: int32(tick.Timestamp / 1000), - TickTimestamp: int32(tick.Timestamp / 1000), - Open: tick.Bid, - High: tick.Bid, - Low: tick.Bid, - Close: tick.Bid, - Volume: uint64(tick.VolumeBid), - } -} - // FxtFile define fxt file format // // Refer: https://github.com/EA31337/MT-Formats @@ -49,14 +25,17 @@ func convertToFxtTick(tick *core.TickData) *fxtTick { // type FxtFile struct { fpath string + symbol string + model uint32 header *FXTHeader - firstUniBar *fxtTick - lastUniBar *fxtTick + firstUniBar *FxtTick + lastUniBar *FxtTick deltaTimestamp uint32 endTimestamp uint32 + timeframe uint32 barCount int32 tickCount int64 - chTicks chan *fxtTick + chTicks chan *FxtTick chClose chan struct{} } @@ -66,9 +45,12 @@ func NewFxtFile(timeframe, spread, model uint32, dest, symbol string) *FxtFile { fxt := &FxtFile{ header: NewHeader(405, symbol, timeframe, spread, model), fpath: filepath.Join(dest, fn), - chTicks: make(chan *fxtTick, 1024), + chTicks: make(chan *FxtTick, 1024), chClose: make(chan struct{}, 1), deltaTimestamp: timeframe * 60, + timeframe: timeframe, + symbol: symbol, + model: model, } go fxt.worker() @@ -78,7 +60,7 @@ func NewFxtFile(timeframe, spread, model uint32, dest, symbol string) *FxtFile { func (f *FxtFile) worker() error { defer func() { close(f.chClose) - log.Info("Saved Bar: %d, Ticks: %d.", f.barCount, f.tickCount) + log.Info("M5d Saved Bar: %d, Ticks: %d.", f.timeframe, f.barCount, f.tickCount) }() fxt, err := os.OpenFile(f.fpath, os.O_CREATE|os.O_TRUNC, 666) @@ -88,7 +70,7 @@ func (f *FxtFile) worker() error { } defer fxt.Close() - bu := bytes.NewBuffer(make([]byte, 4096)) + bu := bytes.NewBuffer(make([]byte, 0, 4096)) // // write FXT header @@ -123,9 +105,9 @@ func (f *FxtFile) worker() error { func (f *FxtFile) PackTicks(barTimestemp uint32, ticks []*core.TickData) error { for _, tick := range ticks { - f.chTicks <- &fxtTick{ - BarTimestamp: int32(barTimestemp), - TickTimestamp: int32(tick.Timestamp / 1000), + f.chTicks <- &FxtTick{ + BarTimestamp: uint32(barTimestemp), + TickTimestamp: uint32(tick.Timestamp / 1000), Open: tick.Bid, High: tick.Bid, Low: tick.Bid, @@ -152,9 +134,9 @@ func (f *FxtFile) adjustHeader() error { // first part if _, err := fxt.Seek(216, os.SEEK_SET); err == nil { d := struct { - BarCount int32 // Total bar count - BarStartTimestamp int32 // Modelling start date - date of the first tick. - BarEndTimestamp int32 // Modelling end date - date of the last tick. + BarCount int32 // Total bar count + BarStartTimestamp uint32 // Modelling start date - date of the first tick. + BarEndTimestamp uint32 // Modelling end date - date of the last tick. }{ f.barCount, f.firstUniBar.BarTimestamp, @@ -177,8 +159,8 @@ func (f *FxtFile) adjustHeader() error { // end part if _, err := fxt.Seek(472, os.SEEK_SET); err == nil { d := struct { - BarStartTimestamp int32 // Tester start date - date of the first tick. - BarEndTimestamp int32 // Tester end date - date of the last tick. + BarStartTimestamp uint32 // Tester start date - date of the first tick. + BarEndTimestamp uint32 // Tester end date - date of the last tick. }{ f.firstUniBar.BarTimestamp, f.lastUniBar.BarTimestamp, diff --git a/fxt4/fxt4_test.go b/fxt4/fxt4_test.go new file mode 100644 index 0000000..87dc093 --- /dev/null +++ b/fxt4/fxt4_test.go @@ -0,0 +1,63 @@ +package fxt4 + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "testing" + + "github.com/adyzng/go-duka/core" +) + +func TestFxtFile(t *testing.T) { + fxt := NewFxtFile(1, 20, 0, "D:\\Data", "EURUSD") + fxt.PackTicks(0, []*core.TickData{&core.TickData{}}) +} + +func TestHeader(t *testing.T) { + //fname := `F:\tester-ok\EURUSD1_0.fxt` + fname := `E:\test\EURUSD5_0.fxt` + + fh, err := os.OpenFile(fname, os.O_RDONLY, 666) + if err != nil { + t.Fatalf("Open fxt file failed: %v.\n", err) + } + defer fh.Close() + + bs := make([]byte, headerSize) + n, err := fh.Read(bs[:]) + if err != nil || n != headerSize { + t.Fatalf("Read fxt header failed: %v.\n", err) + } + + var h FXTHeader + err = binary.Read(bytes.NewBuffer(bs[:]), binary.LittleEndian, &h) + if err != nil { + t.Fatalf("Decode fxt header failed: %v.\n", err) + } + + tickBs := make([]byte, tickSize) + for { + n, err = fh.Read(tickBs[:tickSize]) + if err == io.EOF { + break + } + + if n != tickSize || err != nil { + t.Errorf("Read tick data failed: %v.\n", err) + break + } + + var tick FxtTick + err = binary.Read(bytes.NewBuffer(tickBs[:]), binary.LittleEndian, &tick) + if err != nil { + t.Errorf("Decode tick data failed: %v.\n", err) + break + } + + fmt.Println(tick) + } + fmt.Printf("Header:\n%+v\n", h) +} diff --git a/fxt4/fxt_test.go b/fxt4/fxt_test.go deleted file mode 100644 index 69d1735..0000000 --- a/fxt4/fxt_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package fxt4 - -import ( - "testing" - - "github.com/adyzng/go-duka/core" -) - -func TestFxtFile(t *testing.T) { - fxt := NewFxtFile(1, 20, 0, "D:\\Data", "EURUSD") - fxt.AddTicks([]*core.TickData{&core.TickData{}}) -} diff --git a/fxt4/header.go b/fxt4/header.go index f660f9c..da1ba49 100644 --- a/fxt4/header.go +++ b/fxt4/header.go @@ -1,85 +1,16 @@ package fxt4 import ( + "fmt" + "time" + "github.com/adyzng/go-duka/misc" ) -/* -struct FXT_HEADER { // -- offset ---- size --- description ---------------------------------------------------------------------------- - UINT version; // 0 4 header version: 405 - char description[64]; // 4 64 copyright/description (szchar) - char serverName[128]; // 68 128 account server name (szchar) - char symbol[MAX_SYMBOL_LENGTH+1]; // 196 12 symbol (szchar) - UINT period; // 208 4 timeframe in minutes - UINT modelType; // 212 4 0=EveryTick|1=ControlPoints|2=BarOpen - UINT modeledBars; // 216 4 number of modeled bars (w/o prolog) - UINT firstBarTime; // 220 4 bar open time of first tick (w/o prolog) - UINT lastBarTime; // 224 4 bar open time of last tick (w/o prolog) - BYTE reserved_1[4]; // 228 4 (alignment to the next double) - double modelQuality; // 232 8 max. 99.9 - - // common parameters // ---------------------------------------------------------------------------------------------------------------- - char baseCurrency[MAX_SYMBOL_LENGTH+1]; // 240 12 base currency (szchar) = StringLeft(symbol, 3) - UINT spread; // 252 4 spread in points: 0=zero spread = MarketInfo(MODE_SPREAD) - UINT digits; // 256 4 digits = MarketInfo(MODE_DIGITS) - BYTE reserved_2[4]; // 260 4 (alignment to the next double) - double pointSize; // 264 8 resolution, ie. 0.0000'1 = MarketInfo(MODE_POINT) - UINT minLotsize; // 272 4 min lot size in centi lots (hundredths) = MarketInfo(MODE_MINLOT) * 100 - UINT maxLotsize; // 276 4 max lot size in centi lots (hundredths) = MarketInfo(MODE_MAXLOT) * 100 - UINT lotStepsize; // 280 4 lot stepsize in centi lots (hundredths) = MarketInfo(MODE_LOTSTEP) * 100 - UINT stopsLevel; // 284 4 orders stop distance in points = MarketInfo(MODE_STOPLEVEL) - BOOL pendingsGTC; // 288 4 close pending orders at end of day or GTC - BYTE reserved_3[4]; // 292 4 (alignment to the next double) - - // profit calculation parameters // ---------------------------------------------------------------------------------------------------------------- - double contractSize; // 296 8 ie. 100000 = MarketInfo(MODE_LOTSIZE) - double tickValue; // 304 8 tick value in quote currency (empty) = MarketInfo(MODE_TICKVALUE) - double tickSize; // 312 8 tick size (empty) = MarketInfo(MODE_TICKSIZE) - UINT profitCalculationMode; // 320 4 0=Forex|1=CFD|2=Futures = MarketInfo(MODE_PROFITCALCMODE) - - // swap calculation parameters // ---------------------------------------------------------------------------------------------------------------- - BOOL swapEnabled; // 324 4 if swaps are to be applied - UINT swapCalculationMode; // 328 4 0=Points|1=BaseCurrency|2=Interest|3=MarginCurrency = MarketInfo(MODE_SWAPTYPE) - BYTE reserved_4[4]; // 332 4 (alignment to the next double) - double swapLongValue; // 336 8 long overnight swap value = MarketInfo(MODE_SWAPLONG) - double swapShortValue; // 344 8 short overnight swap values = MarketInfo(MODE_SWAPSHORT) - UINT tripleRolloverDay; // 352 4 weekday of triple swaps = WEDNESDAY (3) - - // margin calculation parameters // ---------------------------------------------------------------------------------------------------------------- - UINT accountLeverage; // 356 4 account leverage = AccountLeverage() - UINT freeMarginCalculationType; // 360 4 free margin calculation type = AccountFreeMarginMode() - UINT marginCalculationMode; // 364 4 margin calculation mode = MarketInfo(MODE_MARGINCALCMODE) - UINT marginStopoutLevel; // 368 4 margin stopout level = AccountStopoutLevel() - UINT marginStopoutType; // 372 4 margin stopout type = AccountStopoutMode() - double marginInit; // 376 8 initial margin requirement (in units) = MarketInfo(MODE_MARGININIT) - double marginMaintenance; // 384 8 maintainance margin requirement (in units) = MarketInfo(MODE_MARGINMAINTENANCE) - double marginHedged; // 392 8 hedged margin requirement (in units) = MarketInfo(MODE_MARGINHEDGED) - double marginDivider; // 400 8 leverage calculation @see example in struct SYMBOL - char marginCurrency[MAX_SYMBOL_LENGTH+1]; // 408 12 = AccountCurrency() - BYTE reserved_5[4]; // 420 4 (alignment to the next double) - - // commission calculation parameters // ---------------------------------------------------------------------------------------------------------------- - double commissionValue; // 424 8 commission rate - UINT commissionCalculationMode; // 432 4 0=Money|1=Pips|2=Percent @see COMMISSION_MODE_* - UINT commissionType; // 436 4 0=RoundTurn|1=PerDeal @see COMMISSION_TYPE_* - - // later additions // ---------------------------------------------------------------------------------------------------------------- - UINT firstBar; // 440 4 bar number/index??? of first bar (w/o prolog) or 0 for first bar - UINT lastBar; // 444 4 bar number/index??? of last bar (w/o prolog) or 0 for last bar - UINT startPeriodM1; // 448 4 bar index where modeling started using M1 bars - UINT startPeriodM5; // 452 4 bar index where modeling started using M5 bars - UINT startPeriodM15; // 456 4 bar index where modeling started using M15 bars - UINT startPeriodM30; // 460 4 bar index where modeling started using M30 bars - UINT startPeriodH1; // 464 4 bar index where modeling started using H1 bars - UINT startPeriodH4; // 468 4 bar index where modeling started using H4 bars - UINT testerSettingFrom; // 472 4 begin date from tester settings - UINT testerSettingTo; // 476 4 end date from tester settings - UINT freezeDistance; // 480 4 order freeze level in points = MarketInfo(MODE_FREEZELEVEL) - UINT modelErrors; // 484 4 number of errors during model generation (FIX ERRORS SHOWING UP HERE BEFORE TESTING) - BYTE reserved_6[240]; // 488 240 unused -}; - -*/ +var ( + headerSize = 728 + tickSize = 56 +) // FXTHeader ... // @@ -94,7 +25,7 @@ type FXTHeader struct { ModeledBars uint32 // 216 4 number of modeled bars (w/o prolog) FirstBarTime uint32 // 220 4 bar open time of first tick (w/o prolog) LastBarTime uint32 // 224 4 bar open time of last tick (w/o prolog) - Reserved1 [4]byte // 228 4 (alignment to the next double) + _ [4]byte // 228 4 (alignment to the next double) ModelQuality float64 // 232 8 max. 99.9 // common parameters----------------------------------------------------------------------------------------------------------------------- @@ -108,7 +39,7 @@ type FXTHeader struct { LotStepsize uint32 // 280 4 lot stepsize in centi lots (hundredths) = MarketInfo(MODE_LOTSTEP) * 100 StopsLevel uint32 // 284 4 orders stop distance in points = MarketInfo(MODE_STOPLEVEL) PendingsGTC uint32 // 288 4 close pending orders at end of day or GTC - Reserved3 [4]byte // 292 4 (alignment to the next double) + _ [4]byte // 292 4 (alignment to the next double) // profit calculation parameters------------------------------------------------------------------------------------------------------------- ContractSize float64 // 296 8 ie. 100000 = MarketInfo(MODE_LOTSIZE) @@ -119,7 +50,7 @@ type FXTHeader struct { // swap calculation parameters ------------------------------------------------------------------------------------------------------------- SwapEnabled uint32 // 324 4 if swaps are to be applied SwapCalculationMode int32 // 328 4 0=Points|1=BaseCurrency|2=Interest|3=MarginCurrency = MarketInfo(MODE_SWAPTYPE) - Reserved4 [4]byte // 332 4 (alignment to the next double) + _ [4]byte // 332 4 (alignment to the next double) SwapLongValue float64 // 336 8 long overnight swap value = MarketInfo(MODE_SWAPLONG) SwapShortValue float64 // 344 8 short overnight swap values = MarketInfo(MODE_SWAPSHORT) TripleRolloverDay uint32 // 352 4 weekday of triple swaps = WEDNESDAY (3) @@ -135,7 +66,7 @@ type FXTHeader struct { MarginHedged float64 // 392 8 hedged margin requirement (in units) = MarketInfo(MODE_MARGINHEDGED) MarginDivider float64 // 400 8 leverage calculation @see example in struct SYMBOL MarginCurrency [12]byte // 408 12 = AccountCurrency() - Reserved5 [4]byte // 420 4 (alignment to the next double) + _ [4]byte // 420 4 (alignment to the next double) // commission calculation parameters ---------------------------------------------------------------------------------------------------------- CommissionValue float64 // 424 8 commission rate @@ -155,7 +86,35 @@ type FXTHeader struct { TesterSettingTo uint32 // 476 4 end date from tester settings FreezeDistance uint32 // 480 4 order freeze level in points = MarketInfo(MODE_FREEZELEVEL) ModelErrors uint32 // 484 4 number of errors during model generation (FIX ERRORS SHOWING UP HERE BEFORE TESTING - Reserved6 [240]byte // 488 240 unused + _ [240]byte // 488 240 unused +} + +// FxtTick +// +type FxtTick struct { + BarTimestamp uint32 + _ uint32 + Open float64 + High float64 + Low float64 + Close float64 + Volume uint64 + TickTimestamp uint32 + LaunchExpert uint32 +} + +func (t FxtTick) String() string { + bt := time.Unix(int64(t.BarTimestamp), 0).UTC() + tt := time.Unix(int64(t.TickTimestamp), 0).UTC() + return fmt.Sprintf("%s %s %f %f %f %f %d", + bt.Format("2006-01-02 15:04:05"), + tt.Format("2006-01-02 15:04:05"), + t.Open, + t.High, + t.Low, + t.Close, + t.Volume, + ) } // NewHeader return an predefined FXT header diff --git a/hst/header.go b/hst/header.go index c3cc084..0b3ebef 100644 --- a/hst/header.go +++ b/hst/header.go @@ -1,6 +1,7 @@ package hst import ( + "fmt" "time" "github.com/adyzng/go-duka/misc" @@ -22,7 +23,7 @@ type Header struct { Digits uint32 // 84 4 The amount of digits after decimal point in the symbol TimeSign uint32 // 88 4 Time of sign (database creation) LastSync uint32 // 92 4 Time of last synchronization - unused [13]uint32 // 96 52 unused + _ [13]uint32 // 96 52 unused } // BarData wrap the bar data inside hst (60 Bytes) @@ -30,9 +31,9 @@ type Header struct { type BarData struct { CTM uint32 // 0 4 current time in seconds _ uint32 // 4 4 for padding only - Open float64 // 8 8 OLHCV - Low float64 // 16 8 L + Open float64 // 8 8 OHLCV High float64 // 24 8 H + Low float64 // 16 8 L Close float64 // 32 8 C Volume uint64 // 40 8 V Spread uint32 // 48 4 @@ -50,7 +51,7 @@ func NewHeader(timeframe uint32, symbol string) *Header { } misc.ToFixBytes(h.Symbol[:], symbol) - misc.ToFixBytes(h.Copyright[:], "(C)opyright 2017, MetaQuotes Software Corp.") + misc.ToFixBytes(h.Copyright[:], "##(C)opyright 2017, MetaQuotes Software Corp.") return h } @@ -75,3 +76,27 @@ func (b *BarData) ToBytes() ([]byte, error) { } return bs, err } + +func (b *BarData) String() string { + tm := time.Unix(int64(b.CTM), 0).UTC() + return fmt.Sprintf("%s %f %f %f %f %d", + tm.Format("2006-01-02 15:04"), + b.Open, + b.High, + b.Low, + b.Close, + b.Volume, + ) +} + +func (b *BarData) Strings() []string { + tm := time.Unix(int64(b.CTM), 0).UTC() + return []string{ + tm.Format("2006.01.02,15:04"), + fmt.Sprintf("%.5f", b.Open), + fmt.Sprintf("%.5f", b.High), + fmt.Sprintf("%.5f", b.Low), + fmt.Sprintf("%.5f", b.Close), + fmt.Sprintf("%d", b.Volume), + } +} diff --git a/hst/hst401.go b/hst/hst401.go index 67b8972..f2d71ae 100644 --- a/hst/hst401.go +++ b/hst/hst401.go @@ -59,7 +59,7 @@ func (h *HST401) worker() error { defer func() { f.Close() close(h.chClose) - log.Info("Saved Bar: %d.", h.barCount) + log.Info("M%d Saved Bar: %d.", h.timefame, h.barCount) }() // write HST header @@ -69,14 +69,14 @@ func (h *HST401) worker() error { log.Error("Pack HST Header (%v) failed: %v.", h.header, err) return err } - if _, err = f.Write(bs); err != nil { + if _, err = f.Write(bs[:]); err != nil { log.Error("Write HST Header (%v) failed: %v.", h.header, err) return err } for bar := range h.chBars { if bs, err = bar.ToBytes(); err == nil { - if _, err = f.Write(bs); err != nil { + if _, err = f.Write(bs[:]); err != nil { log.Error("Write BarData(%v) failed: %v.", bar, err) } } else { @@ -100,7 +100,7 @@ func (h *HST401) PackTicks(barTimestamp uint32, ticks []*core.TickData) error { } bar := &BarData{ - CTM: uint32(ticks[0].Timestamp / 1000), + CTM: barTimestamp, //uint32(ticks[0].Timestamp / 1000), Open: ticks[0].Bid, Low: ticks[0].Bid, High: ticks[0].Bid, @@ -111,11 +111,12 @@ func (h *HST401) PackTicks(barTimestamp uint32, ticks []*core.TickData) error { bar.Close = tick.Bid bar.Low = math.Min(tick.Bid, bar.Low) bar.High = math.Max(tick.Bid, bar.High) - bar.Volume = bar.Volume + uint64(tick.VolumeAsk+tick.VolumeBid) + bar.Volume = bar.Volume + uint64(math.Max(tick.VolumeAsk+tick.VolumeBid, 1)) } select { case h.chBars <- bar: + log.Trace("Bar %d: %v.", h.barCount, bar) h.barCount++ break //case <-h.close: diff --git a/hst/hst401_test.go b/hst/hst401_test.go new file mode 100644 index 0000000..885b3e8 --- /dev/null +++ b/hst/hst401_test.go @@ -0,0 +1,97 @@ +package hst + +import ( + "bytes" + "encoding/binary" + "encoding/csv" + "io" + "os" + "testing" +) + +func TestHSTHeader(t *testing.T) { + header := NewHeader(1, "EURUSD") + + bs, _ := header.ToBytes() + if len(bs) != headerBytes { + t.Errorf("Encode header failed: %d, %x.\n", len(bs), bs) + } else { + var h Header + err := binary.Read(bytes.NewBuffer(bs), binary.LittleEndian, &h) + + if err != nil { + t.Errorf("Decode header failed: %v.\n", err) + } + + t.Logf("%+v.\n", h) + t.Logf("Copyright: %s, Symbol: %s\n", string(h.Copyright[:]), string(h.Symbol[:])) + } +} + +func TestLoadHst(t *testing.T) { + //fname := "F:\\00\\EURUSD1.hst" + fcsv := "F:\\00\\EURUSD30-correct.csv" + fname := "C:\\Users\\huan\\AppData\\Roaming\\MetaQuotes\\Terminal\\1DAFD9A7C67DC84FE37EAA1FC1E5CF75\\history\\ICMarkets-Demo01\\00\\EURUSD30.hst" + + f, err := os.OpenFile(fname, os.O_RDONLY, 666) + if err != nil { + t.Fatalf("Open file error: %v.\n", err) + } + + defer f.Close() + bs := make([]byte, headerBytes) + + n, err := f.Read(bs[:]) + if n != headerBytes || err != nil { + t.Fatalf("Load file header failed: %v.\n", err) + } + + var h Header + err = binary.Read(bytes.NewBuffer(bs[:]), binary.LittleEndian, &h) + if err != nil { + t.Errorf("Decode header failed: %v.\n", err) + } + + t.Logf("%+v.\n", h) + t.Logf("Copyright: %s, Symbol: %s\n", string(h.Copyright[:]), string(h.Symbol[:])) + + // open csv + fc, err := os.OpenFile(fcsv, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 666) + if err != nil { + t.Errorf("Failed to create file %s, error %v.\n", fcsv, err) + } + + wc := csv.NewWriter(fc) + + defer func() { + fc.Close() + wc.Flush() + }() + + barCount := 0 + barBs := make([]byte, barBytes) + for { + barBs = barBs[:barBytes] + n, err := f.Read(barBs[:]) + + if err == io.EOF { + break + } + + if n != barBytes || err != nil { + t.Errorf("Read bar data failed: %d:%v.\n", n, err) + break + } + + var bar BarData + err = binary.Read(bytes.NewBuffer(barBs[:]), binary.LittleEndian, &bar) + if err != nil { + t.Errorf("Decode bar data failed: %v.\n", err) + break + } + + wc.Write(bar.Strings()) + //t.Logf("Bar %d: %+v\n", barCount, bar) + barCount++ + } +} diff --git a/misc/bytes.go b/misc/bytes.go index 1f26d76..948525c 100644 --- a/misc/bytes.go +++ b/misc/bytes.go @@ -20,7 +20,7 @@ func PackLittleEndian(size int, v interface{}) ([]byte, error) { if size == 0 || v == nil { return nil, errors.New("invalid arguments") } - bu := bytes.NewBuffer(make([]byte, size)) + bu := bytes.NewBuffer(make([]byte, 0, size)) if err := binary.Write(bu, binary.LittleEndian, v); err != nil { return nil, err } @@ -33,7 +33,7 @@ func PackBigEndian(size int, v interface{}) ([]byte, error) { if size == 0 || v == nil { return nil, errors.New("invalid arguments") } - bu := bytes.NewBuffer(make([]byte, size)) + bu := bytes.NewBuffer(make([]byte, 0, size)) if err := binary.Write(bu, binary.BigEndian, v); err != nil { return nil, err }