* Simplify code

  * Use resty to download bi5
* Fix concurrency issue on instrument metadata downloader

api/instrument/metadata.go

@@ -3,12 +3,12 @@ package instrument
 import (
 	"encoding/json"
 	"fmt"
-	"io"
+	"github.com/go-resty/resty/v2"
 	"log/slog"
 	"math"
-	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -56,22 +56,34 @@ func (m *Metadata) PriceToString(price float64) string {
 	return fmt.Sprintf(m.priceFormat, price)
 }
 
-var codeToInstrument map[string]*Metadata = nil
-var nameToInstrument map[string]*Metadata = nil
+var codeToInstrument = map[string]*Metadata{}
+var nameToInstrument = map[string]*Metadata{}
+
+var s sync.RWMutex
 
 // GetMetadata returns instrument with requested code.
 // Returns nil if not found
 func GetMetadata(code string) *Metadata {
-	LoadMetadataFromJson()
-	return codeToInstrument[strings.ToUpper(code)]
+	LoadMetadataFromJson(false)
+
+	s.RLock()
+	r := codeToInstrument[strings.ToUpper(code)]
+	s.RUnlock()
+
+	return r
 }
 
 func GetMetadataByName(name string) *Metadata {
-	LoadMetadataFromJson()
-	return nameToInstrument[name]
+	LoadMetadataFromJson(false)
+
+	s.RLock()
+	r := nameToInstrument[name]
+	s.RUnlock()
+
+	return r
 }
 
-type InstrumentJson struct {
+type Instrument struct {
 	Name                       string    `json:"name"`
 	Description                string    `json:"description"`
 	DecimalFactor              int       `json:"decimalFactor"`
@@ -83,42 +95,62 @@ type InstrumentJson struct {
 
 var URL = "https://raw.githubusercontent.com/Leo4815162342/dukascopy-node/master/src/utils/instrument-meta-data/generated/instrument-meta-data.json"
 
-func LoadMetadataFromJson() {
-	if codeToInstrument != nil {
+func LoadMetadataFromJson(isForce bool) {
+	if len(codeToInstrument) > 0 && !isForce {
 		return
 	}
 
-	codeToInstrument = map[string]*Metadata{}
-	nameToInstrument = map[string]*Metadata{}
+	resp, getErr := resty.New().R().
+		SetDoNotParseResponse(true).
+		Get(URL)
 
-	resp, err := http.Get(URL)
-	if err != nil {
-		slog.Warn("Failed to retrieve dukas instrument from [%s]", URL)
+	if getErr != nil {
+		slog.Warn(
+			"Failed to retrieve dukas instrument",
+			slog.String("url", URL),
+			slog.Any("error", getErr),
+		)
 
 		return
 	}
 
-	defer func(Body io.ReadCloser) {
-		_ = Body.Close()
-	}(resp.Body)
+	if resp.IsError() {
+		statusCode := resp.RawResponse.StatusCode
+		slog.Error(
+			"failed to retrieve metadata",
+			slog.Int("httpStatusCode", statusCode),
+			slog.String("rawResponse", string(resp.Body())),
+		)
 
-	var m map[string]InstrumentJson
-	if unmarshalErr := json.NewDecoder(resp.Body).Decode(&m); unmarshalErr != nil {
+		return
+	}
+
+	tCodeToInstrument := map[string]*Metadata{}
+	tNameToInstrument := map[string]*Metadata{}
+
+	instruments := map[string]Instrument{}
+	if unmarshalErr := json.NewDecoder(resp.RawBody()).Decode(&instruments); unmarshalErr != nil {
 		slog.Error(
-			"failed to unmarshal instrument file",
+			"failed to unmarshal dukas instrument",
+			slog.Any("error", unmarshalErr),
 			slog.String("url", URL),
-			slog.Any("error", err),
 		)
-	}
 
-	for instrumentCode, instrument := range m {
+		return
+	}
+	for instrumentCode, instrument := range instruments {
 		metadata := jsonToMetadata(instrumentCode, instrument)
-		codeToInstrument[metadata.Code()] = metadata
-		nameToInstrument[metadata.Name()] = metadata
+		tCodeToInstrument[metadata.Code()] = metadata
+		tNameToInstrument[metadata.Name()] = metadata
 	}
+
+	s.Lock()
+	codeToInstrument = tCodeToInstrument
+	nameToInstrument = tNameToInstrument
+	s.Unlock()
 }
 
-func jsonToMetadata(code string, instrument InstrumentJson) *Metadata {
+func jsonToMetadata(code string, instrument Instrument) *Metadata {
 	return &Metadata{
 		code:              strings.ToUpper(code),
 		name:              instrument.Name,

examples/stream/dd_finder_test.go

@@ -6,7 +6,8 @@ import (
 	"github.com/edward-yakop/go-duka/api/tickdata"
 	"github.com/edward-yakop/go-duka/api/tickdata/stream"
 	"github.com/stretchr/testify/assert"
-	"io/ioutil"
+	"github.com/stretchr/testify/require"
+	"log/slog"
 	"math"
 	"os"
 	"strconv"
@@ -15,6 +16,17 @@ import (
 )
 
 func Test_StreamExample_DDFinder(t *testing.T) {
+	slog.SetDefault(
+		slog.New(
+			slog.NewTextHandler(
+				os.Stdout,
+				&slog.HandlerOptions{
+					Level: slog.LevelDebug,
+				},
+			),
+		),
+	)
+
 	loc, _ := time.LoadLocation("EET")
 
 	im := instrument.GetMetadata("GBPJPY")
@@ -26,21 +38,21 @@ func Test_StreamExample_DDFinder(t *testing.T) {
 	openPriceDiff, maxDD, maxPositive, maxDDForMaxPositive, maxPositiveTime, closePriceDiff :=
 		buyDDFinder(t, im, openTime, closeTime, openPrice, closePrice)
 
-	println("Open price diff in [", strconv.Itoa(openPriceDiff), "] points")
-	println("Max DD [", strconv.Itoa(maxDD), "] points")
-	println("Max Positive [", strconv.Itoa(maxPositive), "] points")
-	println("Max Positive Time [", fmtTime(maxPositiveTime), "] Duration [", fmtDuration(maxPositiveTime.Sub(openTime)), "]")
-	println("Max DD for Max Positive [", strconv.Itoa(maxDDForMaxPositive), "] points")
-	println("Close price diff in [", strconv.Itoa(closePriceDiff), "] points")
+	t.Log("Open price diff in [", strconv.Itoa(openPriceDiff), "] points")
+	t.Log("Max DD [", strconv.Itoa(maxDD), "] points")
+	t.Log("Max Positive [", strconv.Itoa(maxPositive), "] points")
+	t.Log("Max Positive Time [", fmtTime(maxPositiveTime), "] Duration [", fmtDuration(maxPositiveTime.Sub(openTime)), "]")
+	t.Log("Max DD for Max Positive [", strconv.Itoa(maxDDForMaxPositive), "] points")
+	t.Log("Close price diff in [", strconv.Itoa(closePriceDiff), "] points")
 	profitInPoints := int(math.Round((closePrice - openPrice) * 1000))
-	println("Profit [", strconv.Itoa(profitInPoints), "] points Duration [", fmtDuration(closeTime.Sub(openTime)), "]")
+	t.Log("Profit [", strconv.Itoa(profitInPoints), "] points Duration [", fmtDuration(closeTime.Sub(openTime)), "]")
 
 	// Asserts
-	assert.Equal(t, 0, openPriceDiff)
-	assert.Equal(t, -240, maxDD)
-	assert.Equal(t, 392, maxPositive)
-	assert.Equal(t, -240, maxDDForMaxPositive)
-	assert.Equal(t, 400, profitInPoints)
+	assert.Equal(t, 0, openPriceDiff, "openPriceDiff")
+	assert.Equal(t, -240, maxDD, "maxDD")
+	assert.Equal(t, 392, maxPositive, "maxPositive")
+	assert.Equal(t, -240, maxDDForMaxPositive, "maxDDForMaxPositive")
+	assert.Equal(t, 400, profitInPoints, "profitInPoints")
 }
 
 func buyDDFinder(t *testing.T, instrument *instrument.Metadata, openTime time.Time, closeTime time.Time, openPrice float64, closePrice float64) (
@@ -60,7 +72,7 @@ func buyDDFinder(t *testing.T, instrument *instrument.Metadata, openTime time.Ti
 		}
 
 		if openPriceDiff == math.MaxInt32 {
-			printTick(" open", tickTime, tick)
+			logTick(t, " open", tickTime, tick)
 			openPriceDiff = int(math.Round((openPrice - tick.Ask) * 1000))
 		}
 
@@ -75,7 +87,7 @@ func buyDDFinder(t *testing.T, instrument *instrument.Metadata, openTime time.Ti
 		}
 
 		if closeTime.Sub(tickTime) <= 0 && closePriceDiff == math.MaxInt32 {
-			printTick("close", tickTime, tick)
+			logTick(t, "close", tickTime, tick)
 			closePriceDiff = int(math.Round((tick.Bid - closePrice) * 1000))
 			return false
 		}
@@ -86,21 +98,27 @@ func buyDDFinder(t *testing.T, instrument *instrument.Metadata, openTime time.Ti
 }
 
 func createEmptyDir(t *testing.T) string {
-	dir, err := ioutil.TempDir(".", "test")
+	dir, err := os.MkdirTemp(".", "test")
 	if !assert.NoError(t, err) {
 		t.FailNow()
 	}
 	t.Cleanup(func() {
-		_ = os.RemoveAll(dir)
+		require.NoError(t, os.RemoveAll(dir), "remove temporary dir failed")
 	})
 	return dir
 }
 
-func printTick(op string, tickTime time.Time, tick *tickdata.TickData) {
-	println(op,
-		"date [", fmtTime(tickTime), "] timestamp [", tick.Timestamp,
-		"] ask [", fmtPrice(tick.Ask), "] bid [", fmtPrice(tick.Bid), "]",
-	)
+func logTick(t *testing.T, op string, tickTime time.Time, tick *tickdata.TickData) {
+	if tick == nil {
+		return
+	}
+
+	t.Helper()
+	timestamp := tick.Timestamp
+	ask := tick.Ask
+	bid := tick.Bid
+
+	t.Logf("%s date [%s] timestamp [%d], ask [%.3f] bid [%.3f]", op, fmtTime(tickTime), timestamp, ask, bid)
 }
 
 func fmtDuration(d time.Duration) string {
@@ -116,7 +134,3 @@ func fmtDuration(d time.Duration) string {
 func fmtTime(tickTime time.Time) string {
 	return tickTime.Format("2006-01-02 15:04:05")
 }
-
-func fmtPrice(p float64) string {
-	return fmt.Sprintf("%.3f", p)
-}

go.mod

@@ -3,6 +3,7 @@ module github.com/edward-yakop/go-duka
 go 1.22
 
 require (
+	github.com/go-resty/resty/v2 v2.13.1
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	github.com/ulikunitz/xz v0.5.12
@@ -11,5 +12,6 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,5 +1,7 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g=
+github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -8,6 +10,53 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/bi5/downloader.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/edward-yakop/go-duka/internal/core"
 	"github.com/edward-yakop/go-duka/internal/misc"
+	"github.com/go-resty/resty/v2"
 	"github.com/pkg/errors"
 	"net/http"
 	"os"
@@ -12,6 +13,7 @@ import (
 )
 
 type Downloader struct {
+	client *resty.Client
 	folder string
 }
 
@@ -31,10 +33,10 @@ func (d Downloader) Download(instrumentCode string, t time.Time) error {
 		return nil
 	}
 
-	link := fmt.Sprintf(core.DukaTmplURL, instrumentCode, year, month-1, day, hour)
+	url := fmt.Sprintf(core.DukaTmplURL, instrumentCode, year, month-1, day, hour)
 
 	var httpStatusCode int
-	httpStatusCode, filesize, err := httpDownload.Download(link, targetFilePath)
+	httpStatusCode, filesize, err := httpDownload.Download(url, targetFilePath)
 	if err != nil {
 		symbolTime := d.symbolAndTime(instrumentCode, dayHour)
 		return errors.Wrap(err, "Failed to download tick data for ["+symbolTime+"]")