Skip to content

Commit

Permalink
add fxt header
Browse files Browse the repository at this point in the history
  • Loading branch information
@adyzng
adyzng committed Dec 20, 2017
1 parent 24e20d9 commit 351d0b7
Show file tree
Hide file tree
Showing 10 changed files with 374 additions and 46 deletions.
14 changes: 8 additions & 6 deletions app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import (
"sync"
"time"

"github.com/adyzng/duka/bi5"
"github.com/adyzng/duka/csv"
"github.com/adyzng/duka/download"
"github.com/adyzng/duka/misc"
"github.com/adyzng/go-duka/bi5"
"github.com/adyzng/go-duka/core"
"github.com/adyzng/go-duka/csv"
"github.com/adyzng/go-duka/misc"
)

var (
Expand All @@ -34,7 +34,7 @@ func App(opt DukaOption) {
}

// dukascopy downloader
duka := download.NewDukaDownloader()
duka := core.NewDukaDownloader()
startTime := time.Now()

for day := opt.Start; day.Unix() < opt.End.Unix(); day = day.Add(24 * time.Hour) {
Expand All @@ -53,7 +53,9 @@ func App(opt DukaOption) {
wg.Add(1)
go func(hour int) {
defer wg.Done()
URL := fmt.Sprintf(download.DukaTmplURL, opt.Symbol, y, m, d, hour)

// !! caution: month - 1
URL := fmt.Sprintf(core.DukaTmplURL, opt.Symbol, y, m-1, d, hour)
if data, err := duka.Download(URL); err == nil {
chReaders <- &hReader{
Data: data,
Expand Down
23 changes: 10 additions & 13 deletions bi5/bi5.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"path/filepath"
"time"

"github.com/adyzng/duka/misc"
"github.com/adyzng/duka/parse"
"github.com/adyzng/go-duka/core"
"github.com/adyzng/go-duka/misc"
"github.com/kjk/lzma"
)

Expand All @@ -18,28 +18,26 @@ var (
)

type Bi5 struct {
day time.Time
hour int
timeH time.Time
dest string
symbol string
}

// New create an bi5 saver
func New(day time.Time, hour int, symbol, dest string) *Bi5 {
return &Bi5{
day: day,
hour: hour,
timeH: day.Add(time.Duration(hour) * time.Hour),
dest: dest,
symbol: symbol,
}
}

func (b *Bi5) Decode(r io.Reader) ([]*parse.TickData, error) {
func (b *Bi5) Decode(r io.Reader) ([]*core.TickData, error) {
dec := lzma.NewReader(r)
defer dec.Close()

ticksArr := make([]*parse.TickData, 0)
bytesArr := make([]byte, parse.TICK_BYTES)
ticksArr := make([]*core.TickData, 0)
bytesArr := make([]byte, core.TICK_BYTES)

for {
n, err := dec.Read(bytesArr[:])
Expand All @@ -48,26 +46,25 @@ func (b *Bi5) Decode(r io.Reader) ([]*parse.TickData, error) {
break
}

if n != parse.TICK_BYTES || err != nil {
if n != core.TICK_BYTES || err != nil {
log.Error("LZMA decode failed: %d: %v.", n, err)
break
}

t, err := parse.DecodeTickData(bytesArr[:], b.symbol)
t, err := core.DecodeTickData(bytesArr[:], b.symbol, b.timeH)
if err != nil {
log.Error("Decode tick data failed: %v.", err)
break
}

t.Time += time.Duration(b.hour) * time.Hour
ticksArr = append(ticksArr, t)
}

return ticksArr, nil
}

func (b *Bi5) Save(r io.Reader) error {
subpath := fmt.Sprintf("%02dh.%s", b.hour, ext)
subpath := fmt.Sprintf("%02dh.%s", b.timeH.Hour(), ext)
fpath := filepath.Join(b.dest, subpath)

f, err := os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 666)
Expand Down
2 changes: 1 addition & 1 deletion download/downloader.go → core/downloader.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package download
package core

// Downloader interface...
type Downloader interface {
Expand Down
4 changes: 2 additions & 2 deletions download/dukascopy.go → core/fetch.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package download
package core

import (
"fmt"
"io/ioutil"
"net/http"
"time"

"github.com/adyzng/duka/misc"
"github.com/adyzng/go-duka/misc"
)

const (
Expand Down
2 changes: 1 addition & 1 deletion parse/parser.go → core/parser.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package parse
package core

import (
"io"
Expand Down
34 changes: 17 additions & 17 deletions parse/tick.go → core/tick.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package parse
package core

import (
"bytes"
Expand All @@ -22,16 +22,16 @@ var (
// date, ask / point, bid / point, round(volume_ask * 1000000), round(volume_bid * 1000000)
type TickData struct {
Symbol string
Time time.Duration
Ask float32
Bid float32
VolumeAsk int32
VolumeBid int32
Time time.Time
Ask float64
Bid float64
VolumeAsk int64
VolumeBid int64
}

func (t *TickData) ToString(day time.Time) []string {
func (t *TickData) ToString() []string {
return []string{
day.Add(t.Time).Format("2006-01-02 15:04:05.000"),
t.Time.Format("2006-01-02 15:04:05.000"),
fmt.Sprintf("%.6f", t.Ask),
fmt.Sprintf("%.6f", t.Bid),
fmt.Sprintf("%d", t.VolumeAsk),
Expand All @@ -41,7 +41,7 @@ func (t *TickData) ToString(day time.Time) []string {

// DecodeTickData from input data bytes array.
// the valid data array should be at size `TICK_BYTES`.
func DecodeTickData(data []byte, symbol string) (*TickData, error) {
func DecodeTickData(data []byte, symbol string, timeH time.Time) (*TickData, error) {
raw := struct {
TimeMs int32
Ask int32
Expand Down Expand Up @@ -72,26 +72,26 @@ func DecodeTickData(data []byte, symbol string) (*TickData, error) {
}
*/

var point float32 = 100000
var point float64 = 100000
for _, sym := range normSymbols {
if symbol == sym {
point = 1000
break
}
}

round := func(f float32) int32 {
round := func(f float64) int64 {
f += 0.5
return int32(math.Floor(float64(f)))
return int64(math.Floor(f))
}

t := TickData{
Symbol: symbol,
Time: time.Duration(raw.TimeMs) * time.Millisecond,
Ask: float32(raw.Ask) / point,
Bid: float32(raw.Bid) / point,
VolumeAsk: round(raw.VolumeAsk * 1000000),
VolumeBid: round(raw.VolumeBid * 1000000),
Time: timeH.Add(time.Duration(raw.TimeMs) * time.Millisecond),
Ask: float64(raw.Ask) / point,
Bid: float64(raw.Bid) / point,
VolumeAsk: round(float64(raw.VolumeAsk) * 1000000),
VolumeBid: round(float64(raw.VolumeBid) * 1000000),
}

return &t, nil
Expand Down
12 changes: 6 additions & 6 deletions csv/csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (
"sort"
"time"

"github.com/adyzng/duka/misc"
"github.com/adyzng/duka/parse"
"github.com/adyzng/go-duka/core"
"github.com/adyzng/go-duka/misc"
)

var (
Expand All @@ -23,7 +23,7 @@ type CsvDump struct {
day time.Time
dest string
symbol string
ticks []*parse.TickData
ticks []*core.TickData
}

func New(day time.Time, symbol, dest string) *CsvDump {
Expand All @@ -50,11 +50,11 @@ func (c *CsvDump) Save(r io.Reader) error {

// sort by time
sort.Slice(c.ticks, func(i, j int) bool {
return c.ticks[i].Time < c.ticks[j].Time
return c.ticks[i].Time.Before(c.ticks[j].Time)
})

for _, tick := range c.ticks {
if err := csv.Write(tick.ToString(c.day)); err != nil {
if err := csv.Write(tick.ToString()); err != nil {
log.Error("Write CSV %s failed: %v.", fpath, err)
return err
}
Expand All @@ -64,7 +64,7 @@ func (c *CsvDump) Save(r io.Reader) error {
return nil
}

func (c *CsvDump) AddTicks(ticks []*parse.TickData) {
func (c *CsvDump) AddTicks(ticks []*core.TickData) {
if len(ticks) > 0 {
c.ticks = append(c.ticks, ticks...)
}
Expand Down
77 changes: 77 additions & 0 deletions fxt/FXT_HEADER_405.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/**
* MT4 structure FXT_HEADER version 405 (tick file header).
*/
struct FXT_HEADER { // -- offset ---- size --- description ----------------------------------------------------------------------------
UINT version; // 0 4 header version: 405
char description[64]; // 4 64 copyright/description (szchar)
char serverName[128]; // 68 128 account server name (szchar)
char symbol[MAX_SYMBOL_LENGTH+1]; // 196 12 symbol (szchar)
UINT period; // 208 4 timeframe in minutes
UINT modelType; // 212 4 0=EveryTick|1=ControlPoints|2=BarOpen
UINT modeledBars; // 216 4 number of modeled bars (w/o prolog)
UINT firstBarTime; // 220 4 bar open time of first tick (w/o prolog)
UINT lastBarTime; // 224 4 bar open time of last tick (w/o prolog)
BYTE reserved_1[4]; // 228 4 (alignment to the next double)
double modelQuality; // 232 8 max. 99.9

// common parameters // ----------------------------------------------------------------------------------------------------------------
char baseCurrency[MAX_SYMBOL_LENGTH+1]; // 240 12 base currency (szchar) = StringLeft(symbol, 3)
UINT spread; // 252 4 spread in points: 0=zero spread = MarketInfo(MODE_SPREAD)
UINT digits; // 256 4 digits = MarketInfo(MODE_DIGITS)
BYTE reserved_2[4]; // 260 4 (alignment to the next double)
double pointSize; // 264 8 resolution, ie. 0.0000'1 = MarketInfo(MODE_POINT)
UINT minLotsize; // 272 4 min lot size in centi lots (hundredths) = MarketInfo(MODE_MINLOT) * 100
UINT maxLotsize; // 276 4 max lot size in centi lots (hundredths) = MarketInfo(MODE_MAXLOT) * 100
UINT lotStepsize; // 280 4 lot stepsize in centi lots (hundredths) = MarketInfo(MODE_LOTSTEP) * 100
UINT stopsLevel; // 284 4 orders stop distance in points = MarketInfo(MODE_STOPLEVEL)
BOOL pendingsGTC; // 288 4 close pending orders at end of day or GTC
BYTE reserved_3[4]; // 292 4 (alignment to the next double)

// profit calculation parameters // ----------------------------------------------------------------------------------------------------------------
double contractSize; // 296 8 ie. 100000 = MarketInfo(MODE_LOTSIZE)
double tickValue; // 304 8 tick value in quote currency (empty) = MarketInfo(MODE_TICKVALUE)
double tickSize; // 312 8 tick size (empty) = MarketInfo(MODE_TICKSIZE)
UINT profitCalculationMode; // 320 4 0=Forex|1=CFD|2=Futures = MarketInfo(MODE_PROFITCALCMODE)

// swap calculation parameters // ----------------------------------------------------------------------------------------------------------------
BOOL swapEnabled; // 324 4 if swaps are to be applied
UINT swapCalculationMode; // 328 4 0=Points|1=BaseCurrency|2=Interest|3=MarginCurrency = MarketInfo(MODE_SWAPTYPE)
BYTE reserved_4[4]; // 332 4 (alignment to the next double)
double swapLongValue; // 336 8 long overnight swap value = MarketInfo(MODE_SWAPLONG)
double swapShortValue; // 344 8 short overnight swap values = MarketInfo(MODE_SWAPSHORT)
UINT tripleRolloverDay; // 352 4 weekday of triple swaps = WEDNESDAY (3)

// margin calculation parameters // ----------------------------------------------------------------------------------------------------------------
UINT accountLeverage; // 356 4 account leverage = AccountLeverage()
UINT freeMarginCalculationType; // 360 4 free margin calculation type = AccountFreeMarginMode()
UINT marginCalculationMode; // 364 4 margin calculation mode = MarketInfo(MODE_MARGINCALCMODE)
UINT marginStopoutLevel; // 368 4 margin stopout level = AccountStopoutLevel()
UINT marginStopoutType; // 372 4 margin stopout type = AccountStopoutMode()
double marginInit; // 376 8 initial margin requirement (in units) = MarketInfo(MODE_MARGININIT)
double marginMaintenance; // 384 8 maintainance margin requirement (in units) = MarketInfo(MODE_MARGINMAINTENANCE)
double marginHedged; // 392 8 hedged margin requirement (in units) = MarketInfo(MODE_MARGINHEDGED)
double marginDivider; // 400 8 leverage calculation @see example in struct SYMBOL
char marginCurrency[MAX_SYMBOL_LENGTH+1]; // 408 12 = AccountCurrency()
BYTE reserved_5[4]; // 420 4 (alignment to the next double)

// commission calculation parameters // ----------------------------------------------------------------------------------------------------------------
double commissionValue; // 424 8 commission rate
UINT commissionCalculationMode; // 432 4 0=Money|1=Pips|2=Percent @see COMMISSION_MODE_*
UINT commissionType; // 436 4 0=RoundTurn|1=PerDeal @see COMMISSION_TYPE_*

// later additions // ----------------------------------------------------------------------------------------------------------------
UINT firstBar; // 440 4 bar number/index??? of first bar (w/o prolog) or 0 for first bar
UINT lastBar; // 444 4 bar number/index??? of last bar (w/o prolog) or 0 for last bar
UINT startPeriodM1; // 448 4 bar index where modeling started using M1 bars
UINT startPeriodM5; // 452 4 bar index where modeling started using M5 bars
UINT startPeriodM15; // 456 4 bar index where modeling started using M15 bars
UINT startPeriodM30; // 460 4 bar index where modeling started using M30 bars
UINT startPeriodH1; // 464 4 bar index where modeling started using H1 bars
UINT startPeriodH4; // 468 4 bar index where modeling started using H4 bars
UINT testerSettingFrom; // 472 4 begin date from tester settings
UINT testerSettingTo; // 476 4 end date from tester settings
UINT freezeDistance; // 480 4 order freeze level in points = MarketInfo(MODE_FREEZELEVEL)
UINT modelErrors; // 484 4 number of errors during model generation (FIX ERRORS SHOWING UP HERE BEFORE TESTING)
BYTE reserved_6[240]; // 488 240 unused
}; // ----------------------------------------------------------------------------------------------------------------
// = 728
29 changes: 29 additions & 0 deletions fxt/fxt.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package fxt

import (
"github.com/adyzng/go-duka/core"
)

type fxtTick struct {
BarTimestamp int32
padding int32
Open float64
High float64
Low float64
Close float64
Volume uint64
TickTimestamp int32
LaunchExpert int32
}

func convertToTxtTick(tick *core.TickData) *fxtTick {
return &fxtTick{
BarTimestamp: int32(tick.Time.Unix()),
TickTimestamp: int32(tick.Time.Unix()),
Open: tick.Bid,
High: tick.Bid,
Low: tick.Bid,
Close: tick.Bid,
Volume: tick.VolumeBid,
}
}
Loading

0 comments on commit 351d0b7

Please sign in to comment.