From 7c4abb6fb1f890e8a17971c143d4466139ed46dd Mon Sep 17 00:00:00 2001 From: Adriano Date: Mon, 3 May 2021 01:34:26 -0300 Subject: [PATCH] Use Yahoo Finance with fallback to Alpha Vantage --- fetch/fetch_stockquote.go | 57 ++++++++++++++------- parsers/parsers_stockquote.go | 94 +++++++++++++++++++++++++++++------ reports/reports_fii.go | 4 +- 3 files changed, 121 insertions(+), 34 deletions(-) diff --git a/fetch/fetch_stockquote.go b/fetch/fetch_stockquote.go index 3a23a88..39780af 100644 --- a/fetch/fetch_stockquote.go +++ b/fetch/fetch_stockquote.go @@ -12,6 +12,13 @@ import ( "github.com/pkg/errors" ) +// API providers +const ( + APInone = iota + APIalphavantage + APIyahoo +) + type StockFetch struct { apiKey string store rapina.StockStore @@ -23,6 +30,7 @@ type StockFetch struct { // NewStockFetch returns a new instance of *StockServer // func NewStockFetch(store rapina.StockStore, log rapina.Logger, apiKey string) *StockFetch { + return &StockFetch{ apiKey: apiKey, store: store, @@ -43,9 +51,14 @@ func (s StockFetch) Quote(code, date string) (float64, error) { return val, nil // returning data found on db } - err = s.stockQuoteFromAPIServer(code) - if err != nil { - return 0, err + err = s.stockQuoteFromAPIServer(code, date, APIyahoo) + if err != nil && s.apiKey != "" { + // Fallback to Alpha Vantage if Yahoo fails + s.log.Debug("Cotação não encontrada no Yahoo, tentando no Alpha Vantage") + err = s.stockQuoteFromAPIServer(code, date, APIalphavantage) + if err != nil { + return 0, err + } } return s.store.Quote(code, date) @@ -56,9 +69,10 @@ func (s StockFetch) Quote(code, date string) (float64, error) { // daily low, daily close, daily volume) of the global equity specified, // covering 20+ years of historical data. // -func (s StockFetch) stockQuoteFromAPIServer(code string) error { - if _, ok := s.cache[code]; ok { - return fmt.Errorf("cotação histórica para '%s' já foi feita", code) +func (s StockFetch) stockQuoteFromAPIServer(code, date string, apiProvider int) error { + if v := s.cache[code]; v == APIalphavantage && apiProvider == APIalphavantage { + // return fmt.Errorf("cotação histórica para '%s' já foi feita", code) + return nil // silent return if this fetch has been run already } s.log.Printf("[>] Baixando cotações de %s\n", code) @@ -70,8 +84,10 @@ func (s StockFetch) stockQuoteFromAPIServer(code string) error { TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr} - u := apiURL(APIalphavantage, code, s.apiKey) - s.log.Debug("%s", u) + u := apiURL(apiProvider, s.apiKey, code, date) + if u == "" { + return errors.New("URL do API server") + } resp, err := client.Get(u) if err != nil { return err @@ -79,10 +95,10 @@ func (s StockFetch) stockQuoteFromAPIServer(code string) error { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("%s: %s", resp.Status, u) + return fmt.Errorf("%s", resp.Status) } - s.cache[code] += 1 // mark map to avoid unnecessary downloads + s.cache[code] = apiProvider // mark map to avoid unnecessary downloads // JSON means error response if resp.Header.Get("Content-Type") == "application/json" { @@ -105,12 +121,7 @@ func (s StockFetch) stockQuoteFromAPIServer(code string) error { return err } -const ( - APIalphavantage int = iota + 1 - APIyahoo -) - -func apiURL(provider int, code, apiKey string) string { +func apiURL(provider int, apiKey, code, date string) string { v := url.Values{} switch provider { case APIalphavantage: @@ -122,7 +133,19 @@ func apiURL(provider int, code, apiKey string) string { return "https://www.alphavantage.co/query?" + v.Encode() case APIyahoo: - return "" + const layout = "2006-01-02 15:04:05 -0700 MST" + t1, err1 := time.Parse(layout, date+" 00:00:00 -0300 GMT") + t2, err2 := time.Parse(layout, date+" 23:59:59 -0300 GMT") + if err1 != nil || err2 != nil { + return "" + } + v.Set("period1", fmt.Sprint(t1.Unix())) + v.Add("period2", fmt.Sprint(t2.Unix())) + v.Add("interval", "1d") + v.Add("events", "history") + v.Add("includeAdjustedClose", "true") + return fmt.Sprintf("https://query1.finance.yahoo.com/v7/finance/download/%s.SA?%s", + code, v.Encode()) } return "" diff --git a/parsers/parsers_stockquote.go b/parsers/parsers_stockquote.go index bbdf25d..4000821 100644 --- a/parsers/parsers_stockquote.go +++ b/parsers/parsers_stockquote.go @@ -40,6 +40,23 @@ func NewStock(db *sql.DB, log rapina.Logger) *StockParser { return s } +// +// Quote returns the quote from DB. +// +func (s *StockParser) Quote(code, date string) (float64, error) { + query := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;` + var close float64 + err := s.db.QueryRow(query, code, date).Scan(&close) + if err == sql.ErrNoRows { + return 0, errors.New("não encontrado no bd") + } + if err != nil { + return 0, errors.Wrapf(err, "lendo cotação de %s do bd", code) + } + + return close, nil +} + // // Save parses the 'stream', get the 'code' stock quotes and // store it on 'db'. Returns the number of registers saved. @@ -58,12 +75,26 @@ func (s *StockParser) Save(stream io.ReadCloser, code string) (int, error) { defer s.close() // Read stream, line by line - var count int + var count, prov int + isHeader := true scanner := bufio.NewScanner(stream) for scanner.Scan() { line := scanner.Text() - q, err := quoteAlphaVantage(line, code) + if isHeader { + prov = provider(line) + isHeader = false + continue + } + + var q *stockQuote + var err error + switch prov { + case alphaVantage: + q, err = parseAlphaVantage(line, code) + case yahoo: + q, err = parseYahoo(line, code) + } if err != nil { continue // ignore lines with error } @@ -136,12 +167,33 @@ func (s *StockParser) close() error { return err } -func quoteAlphaVantage(line, code string) (*stockQuote, error) { +// API providers. +const ( + none int = iota + alphaVantage + yahoo +) + +// provider returns stream type based on header +func provider(header string) int { + if header == "timestamp,open,high,low,close,volume" { + return alphaVantage + } + if header == "Date,Open,High,Low,Close,Adj Close,Volume" { + return yahoo + } + return none +} + +// parseAlphaVantage parses lines downloaded from Alpha Vantage API server +// and returns *stockQuote for 'code'. +func parseAlphaVantage(line, code string) (*stockQuote, error) { fields := strings.Split(line, ",") if len(fields) != 6 { return nil, errors.New("linha inválida") // ignore lines with error } + // Columns: timestamp,open,high,low,close,volume var err error var floats [5]float64 for i := 1; i <= 5; i++ { @@ -162,19 +214,31 @@ func quoteAlphaVantage(line, code string) (*stockQuote, error) { }, nil } -// -// Quote returns the quote from DB. -// -func (s *StockParser) Quote(code, date string) (float64, error) { - query := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;` - var close float64 - err := s.db.QueryRow(query, code, date).Scan(&close) - if err == sql.ErrNoRows { - return 0, errors.New("não encontrado no bd") +// parseYahoo parses lines downloaded from Yahoo Finance API server +// and returns *stockQuote for 'code'. +func parseYahoo(line, code string) (*stockQuote, error) { + fields := strings.Split(line, ",") + if len(fields) != 7 { + return nil, errors.New("linha inválida") // ignore lines with error } - if err != nil { - return 0, errors.Wrapf(err, "lendo cotação de %s do bd", code) + + // Columns: Date,Open,High,Low,Close,Adj Close,Volume + var err error + var floats [6]float64 + for i := 1; i <= 6; i++ { + floats[i-1], err = strconv.ParseFloat(fields[i], 64) + if err != nil { + return nil, errors.Wrap(err, "campo inválido") + } } - return close, nil + return &stockQuote{ + Stock: code, + Date: fields[0], + Open: floats[0], + High: floats[1], + Low: floats[2], + Close: floats[3], + Volume: floats[5], + }, nil } diff --git a/reports/reports_fii.go b/reports/reports_fii.go index 118a0a9..1772ca9 100644 --- a/reports/reports_fii.go +++ b/reports/reports_fii.go @@ -125,7 +125,7 @@ func (t FIITerminalReport) PrintDividends(code string, n int) (*strings.Builder, q, err := t.fetchStock.Quote(code, d.Date) if err != nil { - t.log.Error("%s (%s): %v", code, d.Date, err) + t.log.Error("Cotação de %s (%s): %v", code, d.Date, err) } if q > 0 && err == nil { i := d.Val / q @@ -150,7 +150,7 @@ func (t FIITerminalReport) CsvDividends(code string, n int) (*strings.Builder, e q, err := t.fetchStock.Quote(code, d.Date) if err != nil { - t.log.Error("%s (%s): %v", code, d.Date, err) + t.log.Error("Cotação de %s (%s): %v", code, d.Date, err) } if q > 0 && err == nil { i := d.Val / q