Skip to content

Commit

Permalink
Use Yahoo Finance with fallback to Alpha Vantage
Browse files Browse the repository at this point in the history
  • Loading branch information
@dude333
dude333 committed May 3, 2021
1 parent 5d7c59e commit 7c4abb6
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 34 deletions.
57 changes: 40 additions & 17 deletions fetch/fetch_stockquote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@ import (
"github.com/pkg/errors"
)

// API providers
const (
APInone = iota
APIalphavantage
APIyahoo
)

type StockFetch struct {
apiKey string
store rapina.StockStore
Expand All @@ -23,6 +30,7 @@ type StockFetch struct {
// NewStockFetch returns a new instance of *StockServer
//
func NewStockFetch(store rapina.StockStore, log rapina.Logger, apiKey string) *StockFetch {

return &StockFetch{
apiKey: apiKey,
store: store,
Expand All @@ -43,9 +51,14 @@ func (s StockFetch) Quote(code, date string) (float64, error) {
return val, nil // returning data found on db
}

err = s.stockQuoteFromAPIServer(code)
if err != nil {
return 0, err
err = s.stockQuoteFromAPIServer(code, date, APIyahoo)
if err != nil && s.apiKey != "" {
// Fallback to Alpha Vantage if Yahoo fails
s.log.Debug("Cotação não encontrada no Yahoo, tentando no Alpha Vantage")
err = s.stockQuoteFromAPIServer(code, date, APIalphavantage)
if err != nil {
return 0, err
}
}

return s.store.Quote(code, date)
Expand All @@ -56,9 +69,10 @@ func (s StockFetch) Quote(code, date string) (float64, error) {
// daily low, daily close, daily volume) of the global equity specified,
// covering 20+ years of historical data.
//
func (s StockFetch) stockQuoteFromAPIServer(code string) error {
if _, ok := s.cache[code]; ok {
return fmt.Errorf("cotação histórica para '%s' já foi feita", code)
func (s StockFetch) stockQuoteFromAPIServer(code, date string, apiProvider int) error {
if v := s.cache[code]; v == APIalphavantage && apiProvider == APIalphavantage {
// return fmt.Errorf("cotação histórica para '%s' já foi feita", code)
return nil // silent return if this fetch has been run already
}

s.log.Printf("[>] Baixando cotações de %s\n", code)
Expand All @@ -70,19 +84,21 @@ func (s StockFetch) stockQuoteFromAPIServer(code string) error {
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
u := apiURL(APIalphavantage, code, s.apiKey)
s.log.Debug("%s", u)
u := apiURL(apiProvider, s.apiKey, code, date)
if u == "" {
return errors.New("URL do API server")
}
resp, err := client.Get(u)
if err != nil {
return err
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s: %s", resp.Status, u)
return fmt.Errorf("%s", resp.Status)
}

s.cache[code] += 1 // mark map to avoid unnecessary downloads
s.cache[code] = apiProvider // mark map to avoid unnecessary downloads

// JSON means error response
if resp.Header.Get("Content-Type") == "application/json" {
Expand All @@ -105,12 +121,7 @@ func (s StockFetch) stockQuoteFromAPIServer(code string) error {
return err
}

const (
APIalphavantage int = iota + 1
APIyahoo
)

func apiURL(provider int, code, apiKey string) string {
func apiURL(provider int, apiKey, code, date string) string {
v := url.Values{}
switch provider {
case APIalphavantage:
Expand All @@ -122,7 +133,19 @@ func apiURL(provider int, code, apiKey string) string {
return "https://www.alphavantage.co/query?" + v.Encode()

case APIyahoo:
return ""
const layout = "2006-01-02 15:04:05 -0700 MST"
t1, err1 := time.Parse(layout, date+" 00:00:00 -0300 GMT")
t2, err2 := time.Parse(layout, date+" 23:59:59 -0300 GMT")
if err1 != nil || err2 != nil {
return ""
}
v.Set("period1", fmt.Sprint(t1.Unix()))
v.Add("period2", fmt.Sprint(t2.Unix()))
v.Add("interval", "1d")
v.Add("events", "history")
v.Add("includeAdjustedClose", "true")
return fmt.Sprintf("https://query1.finance.yahoo.com/v7/finance/download/%s.SA?%s",
code, v.Encode())
}

return ""
Expand Down
94 changes: 79 additions & 15 deletions parsers/parsers_stockquote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,23 @@ func NewStock(db *sql.DB, log rapina.Logger) *StockParser {
return s
}

//
// Quote returns the quote from DB.
//
func (s *StockParser) Quote(code, date string) (float64, error) {
query := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;`
var close float64
err := s.db.QueryRow(query, code, date).Scan(&close)
if err == sql.ErrNoRows {
return 0, errors.New("não encontrado no bd")
}
if err != nil {
return 0, errors.Wrapf(err, "lendo cotação de %s do bd", code)
}

return close, nil
}

//
// Save parses the 'stream', get the 'code' stock quotes and
// store it on 'db'. Returns the number of registers saved.
Expand All @@ -58,12 +75,26 @@ func (s *StockParser) Save(stream io.ReadCloser, code string) (int, error) {
defer s.close()

// Read stream, line by line
var count int
var count, prov int
isHeader := true
scanner := bufio.NewScanner(stream)
for scanner.Scan() {
line := scanner.Text()

q, err := quoteAlphaVantage(line, code)
if isHeader {
prov = provider(line)
isHeader = false
continue
}

var q *stockQuote
var err error
switch prov {
case alphaVantage:
q, err = parseAlphaVantage(line, code)
case yahoo:
q, err = parseYahoo(line, code)
}
if err != nil {
continue // ignore lines with error
}
Expand Down Expand Up @@ -136,12 +167,33 @@ func (s *StockParser) close() error {
return err
}

func quoteAlphaVantage(line, code string) (*stockQuote, error) {
// API providers.
const (
none int = iota
alphaVantage
yahoo
)

// provider returns stream type based on header
func provider(header string) int {
if header == "timestamp,open,high,low,close,volume" {
return alphaVantage
}
if header == "Date,Open,High,Low,Close,Adj Close,Volume" {
return yahoo
}
return none
}

// parseAlphaVantage parses lines downloaded from Alpha Vantage API server
// and returns *stockQuote for 'code'.
func parseAlphaVantage(line, code string) (*stockQuote, error) {
fields := strings.Split(line, ",")
if len(fields) != 6 {
return nil, errors.New("linha inválida") // ignore lines with error
}

// Columns: timestamp,open,high,low,close,volume
var err error
var floats [5]float64
for i := 1; i <= 5; i++ {
Expand All @@ -162,19 +214,31 @@ func quoteAlphaVantage(line, code string) (*stockQuote, error) {
}, nil
}

//
// Quote returns the quote from DB.
//
func (s *StockParser) Quote(code, date string) (float64, error) {
query := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;`
var close float64
err := s.db.QueryRow(query, code, date).Scan(&close)
if err == sql.ErrNoRows {
return 0, errors.New("não encontrado no bd")
// parseYahoo parses lines downloaded from Yahoo Finance API server
// and returns *stockQuote for 'code'.
func parseYahoo(line, code string) (*stockQuote, error) {
fields := strings.Split(line, ",")
if len(fields) != 7 {
return nil, errors.New("linha inválida") // ignore lines with error
}
if err != nil {
return 0, errors.Wrapf(err, "lendo cotação de %s do bd", code)

// Columns: Date,Open,High,Low,Close,Adj Close,Volume
var err error
var floats [6]float64
for i := 1; i <= 6; i++ {
floats[i-1], err = strconv.ParseFloat(fields[i], 64)
if err != nil {
return nil, errors.Wrap(err, "campo inválido")
}
}

return close, nil
return &stockQuote{
Stock: code,
Date: fields[0],
Open: floats[0],
High: floats[1],
Low: floats[2],
Close: floats[3],
Volume: floats[5],
}, nil
}
4 changes: 2 additions & 2 deletions reports/reports_fii.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func (t FIITerminalReport) PrintDividends(code string, n int) (*strings.Builder,

q, err := t.fetchStock.Quote(code, d.Date)
if err != nil {
t.log.Error("%s (%s): %v", code, d.Date, err)
t.log.Error("Cotação de %s (%s): %v", code, d.Date, err)
}
if q > 0 && err == nil {
i := d.Val / q
Expand All @@ -150,7 +150,7 @@ func (t FIITerminalReport) CsvDividends(code string, n int) (*strings.Builder, e

q, err := t.fetchStock.Quote(code, d.Date)
if err != nil {
t.log.Error("%s (%s): %v", code, d.Date, err)
t.log.Error("Cotação de %s (%s): %v", code, d.Date, err)
}
if q > 0 && err == nil {
i := d.Val / q
Expand Down

0 comments on commit 7c4abb6

Please sign in to comment.