fxt tick type change

bi5/bi5.go

@@ -121,8 +121,8 @@ func (b *Bi5) Load() ([]byte, error) {
 	f, err := os.OpenFile(fpath, os.O_RDONLY, 666)
 	if err != nil {
 		if os.IsNotExist(err) {
-			log.Trace("Bi5 (%s) not exist.", fpath)
-			return emptBytes, nil
+			log.Trace("Bi5 (%s) not exist, try to download from dukascopy", fpath)
+			return b.Download()
 		}
 		log.Error("Open file %s failed: %v.", fpath, err)
 		return nil, err
@@ -145,25 +145,25 @@ func (b *Bi5) Download() ([]byte, error) {
 	link := fmt.Sprintf(core.DukaTmplURL, b.symbol, year, month-1, day, b.dayH.Hour())
 
 	if data, err = httpDownld.Download(link); err != nil {
-		log.Error("Download %s: %s failed: %v.", b.symbol, b.dayH.Format("2006-01-02:15H"), err)
+		log.Error("%s %s download failed: %v.", b.symbol, b.dayH.Format("2006-01-02:15H"), err)
 		return emptBytes, err
 	}
 
 	if len(data) > 0 {
-		log.Trace("Downloaded %s: %s.", b.symbol, b.dayH.Format("2006-01-02:15H"))
+		log.Trace("%s %s downloaded.", b.symbol, b.dayH.Format("2006-01-02:15H"))
 		b.save = true
 		return data, err
 	}
 
-	log.Warn("Empty %s: %s.", b.symbol, b.dayH.Format("2006-01-02:15H"))
+	log.Warn("%s %s empty.", b.symbol, b.dayH.Format("2006-01-02:15H"))
 	return emptBytes, nil
 }
 
 // decodeTickData from input data bytes array.
 // the valid data array should be at size `TICK_BYTES`.
 //
 //  struck.unpack(!IIIff)
-//  date, ask / point, bid / point, round(volume_ask * 1000000), round(volume_bid * 1000000)
+//  date, ask / point, bid / point, round(volume_ask * 100000), round(volume_bid * 100000)
 //
 func (b *Bi5) decodeTickData(data []byte, symbol string, timeH time.Time) (*core.TickData, error) {
 	raw := struct {

core/fetch.go

@@ -43,12 +43,12 @@ func (h *HTTPDownload) Download(URL string) ([]byte, error) {
 		defer resp.Body.Close()
 
 		if resp.StatusCode != http.StatusOK {
-			log.Warn("[%d] Download %s response %d:%s.", retry, URL, resp.StatusCode, resp.Status)
+			log.Warn("[%d] Download %s failed %d:%s.", retry, URL, resp.StatusCode, resp.Status)
 			if resp.StatusCode == http.StatusNotFound {
 				// 404
 				break
 			}
-			err = fmt.Errorf("http response %d:%s", resp.StatusCode, resp.Status)
+			err = fmt.Errorf("http error %d:%s", resp.StatusCode, resp.Status)
 			continue
 		}
 

core/tickdata.go

@@ -12,8 +12,8 @@ type TickData struct {
 	Timestamp int64   // 时间戳(ms)
 	Ask       float64 // 卖价
 	Bid       float64 // 买价
-	VolumeAsk float64 // 单位：MIO(百万)
-	VolumeBid float64 // 单位：MIO(百万)
+	VolumeAsk float64 // 单位：通常是按10万美元为一手，最小0.01手
+	VolumeBid float64 // 单位：...
 }
 
 // UTC convert timestamp to UTC time

dukapp.go

@@ -195,7 +195,7 @@ func (app *DukaApp) Execute() error {
 			break
 		}
 
-		log.Info("Finished %s %s.", opt.Symbol, day.Format("2006-01-02"))
+		log.Info("%s %s finished.", opt.Symbol, day.Format("2006-01-02"))
 	}
 
 	//
@@ -261,7 +261,7 @@ func (app *DukaApp) fetchDay(day time.Time) <-chan *hReader {
 		}
 
 		wg.Wait()
-		log.Trace("Bi5 %s %s complete.", opt.Symbol, day.Format("2006-01-02"))
+		log.Trace("%s %s loaded.", opt.Symbol, day.Format("2006-01-02"))
 	}()
 
 	return ch
@@ -332,6 +332,6 @@ func (app *DukaApp) saveData(day time.Time, chData <-chan *hReader) error {
 		app.sortAndOutput(day, dayTicks[:])
 	}
 
-	log.Trace("%s %s convert complete.", opt.Symbol, day.Format("2006-01-02"))
+	log.Trace("%s %s converted.", opt.Symbol, day.Format("2006-01-02"))
 	return err
 }

fxt4/fxt4.go

@@ -1,9 +1,11 @@
 package fxt4
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"io"
 	"math"
 	"os"
 	"path/filepath"
@@ -88,7 +90,7 @@ func (f *FxtFile) worker() error {
 
 	for tick := range f.chTicks {
 
-		if tick.BarTimestamp > tick.TickTimestamp {
+		if tick.BarTimestamp > uint64(tick.TickTimestamp) {
 			log.Fatal("Tick(%v)", tick)
 		}
 
@@ -128,15 +130,15 @@ func (f *FxtFile) PackTicks(barTimestemp uint32, ticks []*core.TickData) error {
 
 	for _, tick := range ticks {
 		ft := &FxtTick{
-			BarTimestamp:  uint32(barTimestemp),
+			BarTimestamp:  uint64(barTimestemp),
 			TickTimestamp: uint32(tick.Timestamp / 1000),
 			Open:          op,
 			High:          math.Max(tick.Bid, hi),
 			Low:           math.Min(tick.Bid, lo),
 			Close:         tick.Bid,
-			Volume:        uint64(vo),
+			Volume:        uint64(math.Max(tick.VolumeBid*100, 1)),
+			LaunchExpert:  3,
 			//RealSpread:    uint32(tick.Ask - tick.Bid/f.header.PointSize),
-			//LaunchExpert:  3,
 		}
 		vo = vo + tick.VolumeBid
 		f.chTicks <- ft
@@ -169,8 +171,8 @@ func (f *FxtFile) adjustHeader() error {
 			BarEndTimestamp   uint32 // Modelling end date - date of the last tick.
 		}{
 			f.barCount,
-			f.firstUniBar.BarTimestamp,
-			f.lastUniBar.BarTimestamp,
+			uint32(f.firstUniBar.BarTimestamp),
+			uint32(f.lastUniBar.BarTimestamp),
 		}
 
 		bu := new(bytes.Buffer)
@@ -192,8 +194,8 @@ func (f *FxtFile) adjustHeader() error {
 			BarStartTimestamp uint32 // Tester start date - date of the first tick.
 			BarEndTimestamp   uint32 // Tester end date - date of the last tick.
 		}{
-			f.firstUniBar.BarTimestamp,
-			f.lastUniBar.BarTimestamp,
+			uint32(f.firstUniBar.BarTimestamp),
+			uint32(f.lastUniBar.BarTimestamp),
 		}
 		bu := new(bytes.Buffer)
 		if err = binary.Write(bu, binary.LittleEndian, &d); err == nil {
@@ -216,3 +218,62 @@ func (f *FxtFile) Finish() error {
 	<-f.chClose
 	return f.adjustHeader()
 }
+
+// DumpFile dump fxt file into txt format
+//
+func DumpFile(fname string, header bool, w io.Writer) {
+	fh, err := os.OpenFile(fname, os.O_RDONLY, 666)
+	if err != nil {
+		log.Error("Open fxt file failed: %v.", err)
+		return
+	}
+	defer fh.Close()
+
+	bs := make([]byte, headerSize)
+	n, err := fh.Read(bs[:])
+	if err != nil || n != headerSize {
+		log.Error("Read fxt header failed: %v.", err)
+		return
+	}
+
+	var h FXTHeader
+	err = binary.Read(bytes.NewBuffer(bs[:]), binary.LittleEndian, &h)
+	if err != nil {
+		log.Error("Decode fxt header failed: %v.", err)
+		return
+	}
+
+	if w == nil {
+		w = os.Stdout
+	}
+	bw := bufio.NewWriter(w)
+	bw.WriteString(fmt.Sprintf("Header: %+v\n", h))
+	defer bw.Flush()
+
+	if header {
+		// only header
+		return
+	}
+
+	tickBs := make([]byte, tickSize)
+	for {
+		n, err = fh.Read(tickBs[:tickSize])
+		if err == io.EOF {
+			break
+		}
+
+		if n != tickSize || err != nil {
+			log.Error("Read tick data failed: %v.", err)
+			break
+		}
+
+		var tick FxtTick
+		err = binary.Read(bytes.NewBuffer(tickBs[:]), binary.LittleEndian, &tick)
+		if err != nil {
+			log.Error("Decode tick data failed: %v.", err)
+			break
+		}
+
+		bw.WriteString(fmt.Sprintf("%s\n", &tick))
+	}
+}

fxt4/fxt4_test.go

@@ -20,9 +20,9 @@ func TestFxtFile(t *testing.T) {
 }
 
 func TestHeader(t *testing.T) {
-	fname := `F:\201710\EURUSD1_0.fxt`
+	fname := `F:\201209\EURUSD15_0.fxt`
 	//fname := `F:\201710\EURUSD1.fxt`
-	//fname := `C:\Users\huan\AppData\Roaming\MetaQuotes\Terminal\1DAFD9A7C67DC84FE37EAA1FC1E5CF75\tester\history\EURUSD1_0.fxt`
+	//fname := `C:\Users\huan\AppData\Roaming\MetaQuotes\Terminal\1DAFD9A7C67DC84FE37EAA1FC1E5CF75\tester\history\org\EURUSD15_0.fxt`
 
 	fh, err := os.OpenFile(fname, os.O_RDONLY, 666)
 	if err != nil {
@@ -63,7 +63,7 @@ func TestHeader(t *testing.T) {
 			break
 		}
 
-		fmt.Println(&tick)
+		fmt.Printf("%+v\n", tick)
 		//break
 	}
 }

fxt4/header.go

@@ -12,8 +12,17 @@ var (
 	tickSize   = 56
 )
 
-// FXTHeader ...
+// FXTHeader History Files in FXT Format
+// https://www.metatrader4.com/en/trading-platform/help/autotrading/tester/tester_fxt
 //
+// Documentation on the format can be found in terminal Help (Client terminal - Auto Trading - Strategy Testing - History Files FXT).
+// However the obtained data shows that the data does not match the declared format.
+// In the eye catches the fact that the work is carried out over time in both formats: the new and the old MQL4.
+// So, members of fromdate and todate structure TestHistoryHeader , and ctm structure TestHistory use the old (4 hbaytny) date / time format, but a member of otm structure TestHistory written in the new (8-byte) date / time format.
+// It is unclear whether the correct type of selected members unknown.
+// The FXT as teak prices recorded only Bid, but its spread is written in the Volume field.
+// By breaking MT4 is obtained to ensure that the MT4-tester figured on each tick Ask, how the Bid + the Volume (that's the trick).
+// Source: https://forum.mql4.com/ru/64199/page3
 type FXTHeader struct {
 	//Header layout--------------------- offset --- size --- description ------------------------------------------------------------------------
 	Version      uint32    //  				  0        4     header version:
@@ -92,27 +101,14 @@ type FXTHeader struct {
 // FxtTick ...
 //
 type FxtTick struct {
-	BarTimestamp  uint32  //   0  4   bar timestamp align with timeframe, unit seconds
-	_             uint32  //   4  4   for padding
+	BarTimestamp  uint64  //   0  8   Bar datetime, align with timeframe, unit seconds
 	Open          float64 //   8  8
 	High          float64 //  16  8
 	Low           float64 //  24  8
 	Close         float64 //  32  8
-	Volume        uint64  //  40  8
+	Volume        uint64  //  40  8   Volume (documentation says it's a double, though it's stored as a long int)
 	TickTimestamp uint32  //  48  4   tick data timestamp in seconds
-	LaunchExpert  uint32  //  52  4
-}
-
-type FxtTick_ struct {
-	BarTimestamp  uint32  //   0  4
-	Open          float64 //   4  8
-	High          float64 //  12  8
-	Low           float64 //  20  8
-	Close         float64 //  28  8
-	Volume        uint64  //  36  8
-	RealSpread    uint32  //  44  4
-	TickTimestamp uint32  //  48  4
-	LaunchExpert  uint32  //  52  4
+	LaunchExpert  uint32  //  52  4   Flag to launch an expert (0 - bar will be modified, but the expert will not be launched).
 }
 
 // NewHeader return an predefined FXT header
@@ -170,7 +166,7 @@ func NewHeader(version uint32, symbol string, timeframe, spread, model uint32) *
 	misc.ToFixBytes(h.ServerName[:], "Beijing MoreU Tech.")
 	misc.ToFixBytes(h.Symbol[:], symbol)
 	misc.ToFixBytes(h.BaseCurrency[:], symbol[:3])
-	misc.ToFixBytes(h.MarginCurrency[:], symbol[3:])
+	misc.ToFixBytes(h.MarginCurrency[:], symbol[:3])
 
 	return h
 }
@@ -196,7 +192,7 @@ func (t *FxtTick) ToBytes() ([]byte, error) {
 func (t *FxtTick) String() string {
 	bt := time.Unix(int64(t.BarTimestamp), 0).UTC()
 	tt := time.Unix(int64(t.TickTimestamp), 0).UTC()
-	return fmt.Sprintf("%s %s %f %f %f %f %d",
+	return fmt.Sprintf("%s %s %f %f %f %f %v",
 		bt.Format("2006-01-02 15:04:05"),
 		tt.Format("2006-01-02 15:04:05"),
 		t.Open,

hst/header.go

@@ -29,8 +29,7 @@ type Header struct {
 // BarData wrap the bar data inside hst (60 Bytes)
 //
 type BarData struct {
-	CTM        uint32  //   0   4   current time in seconds
-	_          uint32  //   4   4   for padding only
+	CTM        uint64  //   0   8   current time in seconds, MQL4 datetime
 	Open       float64 //   8   8   OHLCV
 	High       float64 //  24   8   H
 	Low        float64 //  16   8   L

hst/hst401.go

@@ -100,19 +100,21 @@ func (h *HST401) PackTicks(barTimestamp uint32, ticks []*core.TickData) error {
 	}
 
 	bar := &BarData{
-		CTM:   barTimestamp, //uint32(ticks[0].Timestamp / 1000),
+		CTM:   uint64(barTimestamp), //uint32(ticks[0].Timestamp / 1000),
 		Open:  ticks[0].Bid,
 		Low:   ticks[0].Bid,
 		High:  ticks[0].Bid,
 		Close: ticks[0].Bid,
 	}
 
+	var totalVol float64
 	for _, tick := range ticks {
 		bar.Close = tick.Bid
 		bar.Low = math.Min(tick.Bid, bar.Low)
 		bar.High = math.Max(tick.Bid, bar.High)
-		bar.Volume = bar.Volume + uint64(math.Max(tick.VolumeAsk+tick.VolumeBid, 1))
+		totalVol = totalVol + tick.VolumeBid /*+tick.VolumeAsk*/
 	}
+	bar.Volume = uint64(math.Max(totalVol, 1))
 
 	select {
 	case h.chBars <- bar:

main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"flag"
 	"fmt"
+	"path/filepath"
 	"time"
 
+	"github.com/adyzng/go-duka/fxt4"
 	"github.com/go-clog/clog"
 )
 
@@ -42,6 +44,7 @@ type argsList struct {
 	Local   bool
 	Spread  uint
 	Model   uint
+	Dump    string
 	Symbol  string
 	Output  string
 	Format  string
@@ -54,7 +57,9 @@ func main() {
 	args := argsList{}
 	start := time.Now().Format("2006-01-02")
 	end := time.Now().Add(24 * time.Hour).Format("2006-01-02")
-
+	flag.StringVar(&args.Dump,
+		"dump", "",
+		"dump given file format")
 	flag.StringVar(&args.Period,
 		"timeframe", "M1",
 		"timeframe values: M1, M5, M15, M30, H1, H4, D1, W1, MN")
@@ -102,6 +107,15 @@ func main() {
 		})
 	}
 
+	if args.Dump != "" {
+		if filepath.Ext(args.Dump) == ".fxt" {
+			fxt4.DumpFile(args.Dump, args.Header, nil)
+		} else {
+			fmt.Println("invalid file ext", filepath.Ext(args.Dump))
+		}
+		return
+	}
+
 	opt, err := ParseOption(args)
 	if err != nil {
 		fmt.Println(err)