* Move tickdata to API.

* CSV now print ask and bid price using instrument metadata decimal factor.

api/instrument/metadata.go

@@ -1,6 +1,11 @@
 package instrument
 
-import "strings"
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+)
 
 // This file is a port of https://github.com/Leo4815162342/dukascopy-tools/blob/master/packages/dukascopy-node/src/config/instruments-metadata.ts
 
@@ -13,6 +18,8 @@ type Metadata struct {
 	decimalFactor     float64
 	minStartDateDaily string
 	group             string
+
+	priceFormat string
 }
 
 func (m Metadata) Code() string {
@@ -43,6 +50,13 @@ func (m Metadata) Group() string {
 	return m.group
 }
 
+func (m *Metadata) PriceToString(price float64) string {
+	if len(m.priceFormat) == 0 {
+		m.priceFormat = "%." + strconv.Itoa(int(math.Log10(m.decimalFactor))) + "f"
+	}
+	return fmt.Sprintf(m.priceFormat, price)
+}
+
 // Returns instrument with requested code.
 //
 // Returns [nil] if not found

api/instrument/metadata_test.go

@@ -0,0 +1,27 @@
+package instrument
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestGetMetadata_happyPath(t *testing.T) {
+	m := GetMetadata("EURUSD")
+	assert.NotNil(t, m)
+	assert.Equal(t, "eurusd", m.Code())
+	assert.Equal(t, "EUR/USD", m.Name())
+	assert.Equal(t, "Euro vs US Dollar", m.Description())
+	assert.Equal(t, float64(100000), m.DecimalFactor())
+}
+
+func TestGetMetadata_blank(t *testing.T) {
+	m := GetMetadata("NOT_EXISTENT")
+	assert.Nil(t, m)
+}
+
+func TestMetadata_PriceToString(t *testing.T) {
+	m := GetMetadata("EURUSD")
+	assert.Equal(t, "1.22464", m.PriceToString(1.22464))
+	assert.Equal(t, "1.00000", m.PriceToString(1))
+	assert.Equal(t, "1.01000", m.PriceToString(1.01))
+}

internal/core/tickdata.go → api/tickdata/tickdata.go

@@ -1,4 +1,4 @@
-package core
+package tickdata
 
 import (
 	"fmt"
@@ -25,18 +25,6 @@ func (t *TickData) UTC() time.Time {
 	return tm.UTC()
 }
 
-// Strings used to format into csv row
-//
-func (t *TickData) Strings() []string {
-	return []string{
-		t.UTC().Format("2006-01-02 15:04:05.000"),
-		fmt.Sprintf("%.5f", t.Ask),
-		fmt.Sprintf("%.5f", t.Bid),
-		fmt.Sprintf("%.2f", t.VolumeAsk),
-		fmt.Sprintf("%.2f", t.VolumeBid),
-	}
-}
-
 func (t *TickData) String() string {
 	return fmt.Sprintf("%s %.5f %.5f %.2f %.2f",
 		t.UTC().Format("2006-01-02 15:04:06.000"),

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.4.0
 	github.com/ulikunitz/xz v0.5.9
 	unknwon.dev/clog/v2 v2.2.0
 )

internal/app/dukapp.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"ed-fx/go-duka/api/tickdata"
 	"fmt"
 	"github.com/pkg/errors"
 	"os"
@@ -298,7 +299,7 @@ func (app *DukaApp) fetchDay(day time.Time) <-chan *hReader {
 // sortAndOutput 按时间戳，从前到后排序当天tick数据
 // Sort the tick data of the day from front to back by timestamp
 //
-func (app *DukaApp) sortAndOutput(day time.Time, ticks []*core.TickData) error {
+func (app *DukaApp) sortAndOutput(day time.Time, ticks []*tickdata.TickData) error {
 	if len(ticks) == 0 {
 		return nil
 	}
@@ -328,12 +329,12 @@ func (app *DukaApp) saveData(day time.Time, chData <-chan *hReader) error {
 	)
 
 	nDay := -1
-	dayTicks := make([]*core.TickData, 0, 2048)
+	dayTicks := make([]*tickdata.TickData, 0, 2048)
 
 	for data := range chData {
 		// save bi5 by hour
 		bi5File := data.Bi5
-		var ticks []*core.TickData
+		var ticks []*tickdata.TickData
 
 		// 解析 bi5 成 TickData 数据
 		// Parsing bi5 into TickData data

internal/bi5/bi5.go

@@ -3,6 +3,7 @@ package bi5
 import (
 	"bytes"
 	"ed-fx/go-duka/api/instrument"
+	"ed-fx/go-duka/api/tickdata"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -54,15 +55,15 @@ func New(day time.Time, symbol, dest string) *Bi5 {
 
 // Decode bi5 to tick data array
 //
-func (b *Bi5) Decode(data []byte) ([]*core.TickData, error) {
+func (b *Bi5) Decode(data []byte) ([]*tickdata.TickData, error) {
 	dec, err := lzma.NewReader(bytes.NewBuffer(data[:]))
 	if err != nil {
 		log.Error("Failed to create a lzma reader", err)
 		return nil, err
 	}
 	//defer dec.Close()
 
-	ticksArr := make([]*core.TickData, 0)
+	ticksArr := make([]*tickdata.TickData, 0)
 	bytesArr := make([]byte, TICK_BYTES)
 
 	for {
@@ -173,7 +174,7 @@ func (b *Bi5) Download() ([]byte, error) {
 //  struck.unpack(!IIIff)
 //  date, ask / point, bid / point, round(volume_ask * 100000), round(volume_bid * 100000)
 //
-func (b *Bi5) decodeTickData(data []byte, symbol string, timeH time.Time) (*core.TickData, error) {
+func (b *Bi5) decodeTickData(data []byte, symbol string, timeH time.Time) (*tickdata.TickData, error) {
 	raw := struct {
 		TimeMs    int32 // millisecond offset of current hour
 		Ask       int32
@@ -192,7 +193,7 @@ func (b *Bi5) decodeTickData(data []byte, symbol string, timeH time.Time) (*core
 	}
 
 	var point = b.metadata.DecimalFactor()
-	t := core.TickData{
+	t := tickdata.TickData{
 		Symbol:    symbol,
 		Timestamp: timeH.Unix()*1000 + int64(raw.TimeMs), //timeH.Add(time.Duration(raw.TimeMs) * time.Millisecond),
 		Ask:       float64(raw.Ask) / point,

internal/core/parser.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"ed-fx/go-duka/api/tickdata"
 	"io"
 )
 
@@ -20,7 +21,7 @@ type Converter interface {
 	// PackTicks by timeframe M1,M5...
 	// `barTimestamp` is the timeframe in seconds
 	// `ticks` is all the ticks data within timeframe
-	PackTicks(barTimestamp uint32, ticks []*TickData) error
+	PackTicks(barTimestamp uint32, ticks []*tickdata.TickData) error
 	// Finish current timeframe
 	Finish() error
 }

internal/core/timeframe.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"ed-fx/go-duka/api/tickdata"
 	"regexp"
 	"strconv"
 )
@@ -26,7 +27,7 @@ type Timeframe struct {
 	period         string // M1, M5, M15, M30, H1, H4, D1, W1, MN
 	symbol         string
 
-	chTicks chan *TickData
+	chTicks chan *tickdata.TickData
 	close   chan struct{}
 	out     Converter
 }
@@ -55,7 +56,7 @@ func NewTimeframe(period, symbol string, out Converter) Converter {
 		period:         str,
 		symbol:         symbol,
 		out:            out,
-		chTicks:        make(chan *TickData, 1024),
+		chTicks:        make(chan *tickdata.TickData, 1024),
 		close:          make(chan struct{}, 1),
 	}
 
@@ -64,7 +65,7 @@ func NewTimeframe(period, symbol string, out Converter) Converter {
 }
 
 // PackTicks receive original tick data
-func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*TickData) error {
+func (tf *Timeframe) PackTicks(barTimestamp uint32, ticks []*tickdata.TickData) error {
 	for _, tick := range ticks {
 		select {
 		case tf.chTicks <- tick:
@@ -84,7 +85,7 @@ func (tf *Timeframe) Finish() error {
 // worker thread
 func (tf *Timeframe) worker() error {
 	maxCap := 1024
-	barTicks := make([]*TickData, 0, maxCap)
+	barTicks := make([]*tickdata.TickData, 0, maxCap)
 
 	defer func() {
 		log.Info("%s %s convert completed.", tf.symbol, tf.period)

internal/export/csv/csv.go

@@ -1,13 +1,14 @@
 package csv
 
 import (
+	"ed-fx/go-duka/api/instrument"
+	"ed-fx/go-duka/api/tickdata"
 	"encoding/csv"
 	"fmt"
 	"os"
 	"path/filepath"
 	"time"
 
-	"ed-fx/go-duka/internal/core"
 	"ed-fx/go-duka/internal/misc"
 )
 
@@ -26,7 +27,7 @@ type CsvDump struct {
 	header    bool
 	tickCount int64
 	chClose   chan struct{}
-	chTicks   chan *core.TickData
+	chTicks   chan *tickdata.TickData
 }
 
 // New Csv file
@@ -38,7 +39,7 @@ func New(start, end time.Time, header bool, symbol, dest string) *CsvDump {
 		symbol:  symbol,
 		header:  header,
 		chClose: make(chan struct{}, 1),
-		chTicks: make(chan *core.TickData, 1024),
+		chTicks: make(chan *tickdata.TickData, 1024),
 	}
 
 	go csv.worker()
@@ -55,7 +56,7 @@ func (c *CsvDump) Finish() error {
 
 // PackTicks handle ticks data
 //
-func (c *CsvDump) PackTicks(barTimestamp uint32, ticks []*core.TickData) error {
+func (c *CsvDump) PackTicks(barTimestamp uint32, ticks []*tickdata.TickData) error {
 	for _, tick := range ticks {
 		select {
 		case c.chTicks <- tick:
@@ -96,13 +97,25 @@ func (c *CsvDump) worker() error {
 		csv.Write(csvHeader)
 	}
 
+	m := instrument.GetMetadata(c.symbol)
 	// write tick one by one
 	for tick := range c.chTicks {
-		if err = csv.Write(tick.Strings()); err != nil {
+		row := c.toRow(m, tick)
+		if err = csv.Write(row); err != nil {
 			log.Error("Write csv %s failed: %v.", fpath, err)
 			break
 		}
 	}
 
 	return err
 }
+
+func (c CsvDump) toRow(m *instrument.Metadata, t *tickdata.TickData) []string {
+	return []string{
+		t.UTC().Format("2006-01-02 15:04:05.000"),
+		m.PriceToString(t.Ask),
+		m.PriceToString(t.Bid),
+		fmt.Sprintf("%.2f", t.VolumeAsk),
+		fmt.Sprintf("%.2f", t.VolumeBid),
+	}
+}

internal/export/fxt4/fxt4.go

@@ -3,15 +3,14 @@ package fxt4
 import (
 	"bufio"
 	"bytes"
+	"ed-fx/go-duka/api/tickdata"
 	"ed-fx/go-duka/internal/misc"
 	"encoding/binary"
 	"fmt"
 	"io"
 	"math"
 	"os"
 	"path/filepath"
-
-	"ed-fx/go-duka/internal/core"
 )
 
 var (
@@ -115,7 +114,7 @@ func (f *FxtFile) worker() error {
 	return err
 }
 
-func (f *FxtFile) PackTicks(barTimestemp uint32, ticks []*core.TickData) error {
+func (f *FxtFile) PackTicks(barTimestemp uint32, ticks []*tickdata.TickData) error {
 
 	if len(ticks) == 0 {
 		return nil

internal/export/fxt4/fxt4_test.go

@@ -2,21 +2,20 @@ package fxt4
 
 import (
 	"bytes"
+	"ed-fx/go-duka/api/tickdata"
 	"encoding/binary"
 	"fmt"
 	"io"
 	"os"
 	"testing"
-
-	"ed-fx/go-duka/internal/core"
 )
 
 func TestFxtFile(t *testing.T) {
 	fn := 1e-5 + 0.123
 	fmt.Println(fn)
 
 	fxt := NewFxtFile(1, 20, 0, "D:\\Data", "EURUSD")
-	fxt.PackTicks(0, []*core.TickData{&core.TickData{}})
+	fxt.PackTicks(0, []*tickdata.TickData{&tickdata.TickData{}})
 }
 
 func TestHeader(t *testing.T) {

internal/export/hst/hst401.go

@@ -1,12 +1,12 @@
 package hst
 
 import (
+	"ed-fx/go-duka/api/tickdata"
 	"fmt"
 	"math"
 	"os"
 	"path/filepath"
 
-	"ed-fx/go-duka/internal/core"
 	"ed-fx/go-duka/internal/misc"
 )
 
@@ -93,7 +93,7 @@ func (h *HST401) worker() error {
 
 // PackTicks aggregate ticks with timeframe
 //
-func (h *HST401) PackTicks(barTimestamp uint32, ticks []*core.TickData) error {
+func (h *HST401) PackTicks(barTimestamp uint32, ticks []*tickdata.TickData) error {
 	// Transform universal bar list to binary bar data (60 Bytes per bar)
 	if len(ticks) == 0 {
 		return nil