Feature: Create Ticks API.

 * Automatically download requested tick data from dukascopy
   bound by start and end time
 * Iterator style API

api/tickdata/stream/stream_test.go

@@ -43,7 +43,7 @@ func createEmptyDir(t *testing.T) string {
 		t.FailNow()
 	}
 	t.Cleanup(func() {
-		os.RemoveAll(dir)
+		_ = os.RemoveAll(dir)
 	})
 	return dir
 }

api/tickdata/tickdata.go

@@ -14,18 +14,23 @@ type TickData struct {
 	Bid       float64 // 买价
 	VolumeAsk float64 // 单位：通常是按10万美元为一手，最小0.01手
 	VolumeBid float64 // 单位：...
+
+	timestampInUTC time.Time
 }
 
 // UTC convert timestamp to UTC time
 //
 const timeMillisecond = int64(time.Millisecond)
 
-func (t TickData) UTC() time.Time {
-	return t.TimeInLocation(time.UTC)
+func (t *TickData) UTC() time.Time {
+	if t.timestampInUTC.IsZero() {
+		t.timestampInUTC = time.Unix(t.Timestamp/1000, (t.Timestamp%1000)*timeMillisecond).In(time.UTC)
+	}
+	return t.timestampInUTC
 }
 
 func (t TickData) TimeInLocation(location *time.Location) time.Time {
-	return time.Unix(t.Timestamp/1000, (t.Timestamp%1000)*timeMillisecond).In(location)
+	return t.UTC().In(location)
 }
 
 func (t TickData) String() string {

api/tickdata/ticks/ticks.go

@@ -0,0 +1,138 @@
+package ticks
+
+import (
+	"github.com/ed-fx/go-duka/api/tickdata"
+	"github.com/ed-fx/go-duka/internal/bi5"
+	"time"
+	"unknwon.dev/clog/v2"
+)
+
+type Ticks struct {
+	symbol             string
+	start              time.Time
+	end                time.Time
+	downloadFolderPath string
+
+	currTick     *tickdata.TickData
+	ticksIdx     int
+	ticks        []*tickdata.TickData
+	ticksDayHour time.Time
+	isCompleted  bool
+}
+
+func (t Ticks) Start() time.Time {
+	return t.start
+}
+
+func (t Ticks) End() time.Time {
+	return t.end
+}
+
+func (t Ticks) Current() *tickdata.TickData {
+	return t.currTick
+}
+
+func (t Ticks) IsCompleted() bool {
+	return t.isCompleted
+}
+
+func (t *Ticks) Next() (isSuccess bool, err error) {
+	if t.isCompleted {
+		return
+	}
+
+	// IF ticks loaded, let's check whether it's between boundaries
+	if !t.ticksDayHour.IsZero() {
+		nextTickIdx := t.ticksIdx + 1
+		if nextTickIdx < len(t.ticks) {
+			nextTick := t.ticks[nextTickIdx]
+			nextTickTime := nextTick.UTC()
+			if nextTickTime.Before(t.end) || nextTickTime.Equal(t.end) {
+				t.currTick = nextTick
+				t.ticksIdx = nextTickIdx
+
+				isSuccess = true
+				return
+			} else {
+				t.complete()
+				return
+			}
+		}
+	}
+
+	start := t.nextDownloadHour()
+	for currTime := start; currTime.Before(t.end); currTime = currTime.Add(time.Hour) {
+		bi := bi5.New(currTime, t.symbol, t.downloadFolderPath)
+
+		// Download might return errors when there's no tick data during weekend or holiday
+		if bi.Download() == nil {
+			t.ticks, err = bi.Ticks()
+			if err != nil {
+				t.complete()
+				return
+			} else if len(t.ticks) != 0 {
+				t.ticksDayHour = currTime
+				t.ticksIdx = t.searchTickIdx()
+				t.currTick = nil
+
+				return t.Next()
+			}
+		}
+	}
+
+	t.End()
+	return
+}
+
+func (t *Ticks) complete() {
+	t.isCompleted = true
+	t.ticksIdx = -1
+	t.ticks = nil
+	t.currTick = nil
+}
+
+func (t Ticks) nextDownloadHour() time.Time {
+	var next time.Time
+	if t.currTick == nil {
+		next = t.start.UTC()
+	} else {
+		next = t.currTick.UTC().Add(time.Hour)
+	}
+
+	return time.Date(next.Year(), next.Month(), next.Day(), next.Hour(), 0, 0, 0, time.UTC)
+}
+
+func (t Ticks) searchTickIdx() (idx int) {
+	count := len(t.ticks)
+	for idx = 0; idx < count; idx++ {
+		tick := t.ticks[idx]
+		if !tick.UTC().Before(t.start) {
+			break
+		}
+	}
+
+	return idx - 1
+}
+
+var isLogSetup = false
+
+// time are in UTC
+func New(symbol string, start time.Time, end time.Time, downloadFolderPath string) *Ticks {
+	if !isLogSetup {
+		isLogSetup = true
+		clog.NewConsole(0, clog.ConsoleConfig{
+			Level: clog.LevelInfo,
+		})
+	}
+
+	return &Ticks{
+		symbol:             symbol,
+		start:              start,
+		end:                end,
+		downloadFolderPath: downloadFolderPath,
+
+		ticksDayHour: time.Time{},
+		ticksIdx:     -1,
+		isCompleted:  false,
+	}
+}

api/tickdata/ticks/ticks_test.go

@@ -0,0 +1,52 @@
+package ticks
+
+import (
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+	_ "time/tzdata" // Ensure that custom timezone is included
+)
+
+func TestTick_IncludingWeekend(t *testing.T) {
+	start := time.Date(2017, time.January, 6, 21, 0, 0, 0, time.UTC)
+	end := time.Date(2017, time.January, 8, 22, 59, 0, 0, time.UTC)
+	ticks := New("GBPJPY", start, end, createEmptyDir(t))
+
+	for {
+		isSuccess, nErr := ticks.Next()
+		assert.NoError(t, nErr)
+
+		if !ticks.IsCompleted() {
+			assert.True(t, isSuccess)
+			tick := ticks.Current()
+
+			assert.NotNil(t, tick)
+			assertTime(t, tick.UTC(), start, end)
+			break
+		} else {
+			assert.False(t, isSuccess)
+		}
+	}
+}
+
+func createEmptyDir(t *testing.T) string {
+	dir, err := ioutil.TempDir(".", "test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+	return dir
+}
+
+func assertTime(t *testing.T, time time.Time, start time.Time, end time.Time) {
+	if !assert.True(t, start.Before(time) || start.Equal(time)) {
+		t.FailNow()
+	}
+	if !assert.True(t, end.Equal(time) || end.After(time)) {
+		t.FailNow()
+	}
+}

internal/app/dukapp_test.go

@@ -2,12 +2,17 @@ package app
 
 import (
 	"fmt"
+	"os"
 	"testing"
 
 	"unknwon.dev/clog/v2"
 )
 
 func TestDukaApp(t *testing.T) {
+	t.Cleanup(func() {
+		_ = os.RemoveAll("EURUSD-2017-01-01-2017-01-03.CSV")
+		_ = os.RemoveAll("download")
+	})
 	args := ArgsList{
 		Verbose: true,
 		Header:  true,

internal/bi5/bi5.go

@@ -29,6 +29,7 @@ const (
 // Bi5 from dukascopy
 type Bi5 struct {
 	dayHour        time.Time
+	endDayHour     time.Time
 	symbol         string
 	metadata       *instrument.Metadata
 	targetFilePath string
@@ -50,9 +51,12 @@ func New(dayHour time.Time, symbol, downloadFolderPath string) *Bi5 {
 	biFilePath := filepath.FromSlash(fmt.Sprintf("%s/download/%s/%04d/%02d/%02d/%02dh_ticks.%s", downloadFolderPath, symbol, y, m, d, dayHour.Hour(), ext))
 	metadata := instrument.GetMetadata(symbol)
 
+	beginHour := time.Date(y, m, d, dayHour.Hour(), 0, 0, 0, time.UTC)
+	endHour := beginHour.Add(time.Hour).Add(-1)
 	return &Bi5{
 		targetFilePath: biFilePath,
-		dayHour:        dayHour,
+		dayHour:        beginHour,
+		endDayHour:     endHour,
 		symbol:         symbol,
 		metadata:       metadata,
 	}
@@ -64,23 +68,67 @@ type TickDataResult struct {
 }
 
 func (b Bi5) Ticks() ([]*tickdata.TickData, error) {
-	ticksArr := make([]*tickdata.TickData, 0)
-	var ierr error
-	b.EachTick(func(tick *tickdata.TickData, err error) bool {
-		isNoError := err == nil
-		if isNoError {
-			ticksArr = append(ticksArr, tick)
+	return b.TicksBetween(time.Time{}, time.Time{})
+}
+
+func (b Bi5) TicksBetween(from time.Time, to time.Time) (r []*tickdata.TickData, err error) {
+	r = make([]*tickdata.TickData, 0)
+
+	location := b.dayHour.Location()
+	from, err = b.sanitizeFrom(from, location)
+	if err != nil {
+		return
+	}
+	to, err = b.sanitizeTo(to, location)
+	if err != nil {
+		return
+	}
+
+	b.EachTick(func(tick *tickdata.TickData, terr error) (isContinue bool) {
+		if terr == nil {
+			t := tick.UTC()
+			if !(t.Before(from) || t.After(to)) {
+				r = append(r, tick)
+			}
+			isContinue = to.After(t)
 		} else {
-			ierr = err
+			err = terr
 		}
 
-		return isNoError
+		return
 	})
 
-	if ierr != nil {
-		ticksArr = nil
+	if err != nil {
+		r = []*tickdata.TickData{}
+	}
+	return
+}
+
+func (b Bi5) sanitizeFrom(from time.Time, location *time.Location) (time.Time, error) {
+	if from.IsZero() {
+		return b.dayHour, nil
+	}
+	fromSanitize := from.In(location)
+	if fromSanitize.Before(b.dayHour) || fromSanitize.Equal(b.dayHour) {
+		return b.dayHour, nil
+	}
+	if fromSanitize.Before(b.endDayHour) || fromSanitize.Equal(b.endDayHour) {
+		return fromSanitize, nil
 	}
-	return ticksArr, ierr
+
+	return time.Time{}, errors.New("From [" + from.String() + "] is after [" + b.endDayHour.String() + "]")
+}
+
+func (b Bi5) sanitizeTo(to time.Time, location *time.Location) (time.Time, error) {
+	if to.IsZero() {
+		return b.endDayHour, nil
+	}
+	toSanitize := to.In(location)
+	if toSanitize.Before(b.dayHour) || toSanitize.Equal(b.dayHour) {
+		return time.Time{}, errors.New("To [" + to.String() + "] is before [" + b.dayHour.String() + "]")
+	}
+
+	return toSanitize, nil
 }
 
 // decodeTickData from input data bytes array.