forked from leifcr/go-duka
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Feature: Create streaming API for tick data.
Currently the download is controlled by day instead of hour. Future enhancement can reduce the download to minimum hours * Feature: Streaming API allows stop iterating at any point
- Loading branch information
1 parent
857a9cb
commit 315de21
Showing
9 changed files
with
369 additions
and
163 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
package stream | ||
|
||
import ( | ||
"ed-fx/go-duka/api/tickdata" | ||
iTickdata "ed-fx/go-duka/internal/tickdata" | ||
"time" | ||
"unknwon.dev/clog/v2" | ||
) | ||
|
||
// time is in UTC | ||
type Iterator func(time time.Time, tick *tickdata.TickData, err error) bool | ||
|
||
type Stream struct { | ||
symbol string | ||
start time.Time | ||
end time.Time | ||
downloadFolderPath string | ||
} | ||
|
||
func (s Stream) Start() time.Time { | ||
return s.start | ||
} | ||
|
||
func (s Stream) End() time.Time { | ||
return s.end | ||
} | ||
|
||
func (s Stream) EachTick(it Iterator) { | ||
start := s.start | ||
loc := start.Location() | ||
end := s.end.In(loc) | ||
|
||
dEnd := downloadEnd(s.end) | ||
var isContinue = true | ||
for t := downloadStart(start); t.Before(dEnd) && isContinue; t = t.Add(24 * time.Hour) { | ||
day, err := iTickdata.FetchDay(s.symbol, t, s.downloadFolderPath) | ||
if err != nil && !it(t, nil, err) { | ||
return | ||
} | ||
day.EachTick(func(tick *tickdata.TickData, err error) bool { | ||
tickTime := tick.TimeInLocation(loc) | ||
if (start.Equal(tickTime) || start.Before(tickTime)) && | ||
(end.Equal(tickTime) || end.After(tickTime)) { | ||
isContinue = it(tickTime, tick, err) | ||
} | ||
return isContinue | ||
}) | ||
} | ||
} | ||
|
||
func downloadStart(start time.Time) time.Time { | ||
dStart := start.UTC() | ||
dStart = time.Date(dStart.Year(), dStart.Month(), dStart.Day(), 0, 0, 0, 0, time.UTC) | ||
return dStart | ||
} | ||
|
||
func downloadEnd(end time.Time) time.Time { | ||
dEnd := end.UTC() | ||
dEnd = time.Date(dEnd.Year(), dEnd.Month(), dEnd.Day(), 23, 59, 59, 0, time.UTC) | ||
return dEnd | ||
} | ||
|
||
var isLogSetup = false | ||
|
||
// time are in UTC | ||
func NewStream(symbol string, start time.Time, end time.Time, downloadFolderPath string) *Stream { | ||
if !isLogSetup { | ||
clog.NewConsole(0, clog.ConsoleConfig{ | ||
Level: clog.LevelInfo, | ||
}) | ||
} | ||
|
||
return &Stream{ | ||
symbol: symbol, | ||
start: start, | ||
end: end, | ||
downloadFolderPath: downloadFolderPath, | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
package stream | ||
|
||
import ( | ||
"ed-fx/go-duka/api/tickdata" | ||
"github.com/stretchr/testify/assert" | ||
"io/ioutil" | ||
"os" | ||
"testing" | ||
"time" | ||
_ "time/tzdata" // Ensure that custom timezone is included | ||
) | ||
|
||
func TestStream_EachTick_AlwaysContinue(t *testing.T) { | ||
start := time.Date(2017, time.January, 10, 22, 0, 0, 0, time.UTC) | ||
end := start.Add(1 * time.Hour) | ||
stream := NewStream("GBPJPY", start, end, createEmptyDir(t)) | ||
isRun := false | ||
|
||
stream.EachTick(func(time time.Time, tick *tickdata.TickData, err error) bool { | ||
isRun = true | ||
if !assert.NoError(t, err) { | ||
t.FailNow() | ||
} | ||
assertTime(t, time, start, end) | ||
assert.NotNil(t, tick) | ||
return true | ||
}) | ||
assert.True(t, isRun) | ||
} | ||
|
||
func assertTime(t *testing.T, time time.Time, start time.Time, end time.Time) { | ||
if !assert.True(t, start.Before(time) || start.Equal(time)) { | ||
t.FailNow() | ||
} | ||
if !assert.True(t, end.Equal(time) || end.After(time)) { | ||
t.FailNow() | ||
} | ||
} | ||
|
||
func createEmptyDir(t *testing.T) string { | ||
dir, err := ioutil.TempDir(".", "test") | ||
if !assert.NoError(t, err) { | ||
t.FailNow() | ||
} | ||
t.Cleanup(func() { | ||
os.RemoveAll(dir) | ||
}) | ||
return dir | ||
} | ||
|
||
func TestStream_EachTick_OnlyContinueTwice(t *testing.T) { | ||
location, err := time.LoadLocation("America/New_York") | ||
if !assert.NoError(t, err) { | ||
t.FailNow() | ||
} | ||
|
||
start := time.Date(2017, time.January, 10, 22, 0, 0, 0, location) | ||
end := start.Add(4 * 24 * time.Hour) | ||
stream := NewStream("GBPJPY", start, end, createEmptyDir(t)) | ||
|
||
isRun := false | ||
tickCount := 0 | ||
stream.EachTick(func(time time.Time, tick *tickdata.TickData, err error) bool { | ||
isRun = true | ||
tickCount++ | ||
if !assert.NoError(t, err) { | ||
t.FailNow() | ||
} | ||
assertTime(t, time, start, end) | ||
assert.NotNil(t, tick) | ||
return tickCount < 2 | ||
}) | ||
assert.True(t, isRun) | ||
assert.Equal(t, 2, tickCount) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.