Get stock quotes from B3

cmd/rapina/fii_dividends.go

@@ -64,7 +64,7 @@ func FIIDividends(parms map[string]string, codes []string, n int) error {
 		return err
 	}
 
-	r, err := reports.NewFIITerminalReport(db, viper.GetString("apikey"))
+	r, err := reports.NewFIITerminalReport(db, viper.GetString("apikey"), dataDir)
 	if err != nil {
 		return err
 	}

fetch/fetch_fii.go

@@ -76,13 +76,11 @@ type fiiYeld struct {
 func (fii FII) Dividends(code string, n int) (*[]rapina.Dividend, error) {
 	dividends, months, err := fii.dividendsFromDB(code, n)
 	if err == nil {
-		fii.log.Debug("FROM DB (n=%v months=%v)", n, months)
 		if months >= n {
 			return dividends, err
 		}
 	}
 
-	fii.log.Debug("FROM SERVER (%v)", err)
 	dividends, err = fii.dividendsFromServer(code, n)
 
 	return dividends, err

fetch/fetch_stockquote.go

@@ -6,10 +6,13 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"time"
 
 	"github.com/dude333/rapina"
 	"github.com/pkg/errors"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/transform"
 )
 
 // API providers
@@ -20,22 +23,23 @@ const (
 )
 
 type StockFetch struct {
-	apiKey string
-	store  rapina.StockStore
-	cache  map[string]int
-	log    rapina.Logger
+	apiKey  string
+	store   rapina.StockStore
+	cache   map[string]int
+	dataDir string // working directory where files will be stored to be parsed
+	log     rapina.Logger
 }
 
 //
 // NewStockFetch returns a new instance of *StockServer
 //
-func NewStockFetch(store rapina.StockStore, log rapina.Logger, apiKey string) *StockFetch {
-
+func NewStockFetch(store rapina.StockStore, log rapina.Logger, apiKey, dataDir string) *StockFetch {
 	return &StockFetch{
-		apiKey: apiKey,
-		store:  store,
-		cache:  make(map[string]int),
-		log:    log,
+		apiKey:  apiKey,
+		store:   store,
+		cache:   make(map[string]int),
+		dataDir: dataDir,
+		log:     log,
 	}
 }
 
@@ -51,17 +55,62 @@ func (s StockFetch) Quote(code, date string) (float64, error) {
 		return val, nil // returning data found on db
 	}
 
-	err = s.stockQuoteFromAPIServer(code, date, APIyahoo)
+	// Load quotes from B3, fallback to Yahoo Finance and Alpha Vantage on error
+	err = s.stockQuoteFromB3(date)
+	if err != nil {
+		err = s.stockQuoteFromAPIServer(code, date, APIyahoo)
+	}
 	if err != nil && s.apiKey != "" {
-		// Fallback to Alpha Vantage if Yahoo fails
-		s.log.Debug("Cotação não encontrada no Yahoo, tentando no Alpha Vantage")
 		err = s.stockQuoteFromAPIServer(code, date, APIalphavantage)
+	}
+	if err != nil {
+		return 0, err
+	}
+
+	return s.store.Quote(code, date)
+}
+
+//
+// stockQuoteFromB3 downloads the quotes for all companies for the given date,
+// where 'date' format is YYYY-MM-DD.
+//
+func (s StockFetch) stockQuoteFromB3(date string) error {
+	dataDir := ".data"
+
+	// Convert date string from YYYY-MM-DD to DDMMYYYY
+	if len(date) != len("2021-05-03") {
+		return fmt.Errorf("data com formato inválido: %s", date)
+	}
+	conv := date[8:10] + date[5:7] + date[0:4]
+	url := fmt.Sprintf(`http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_D%s.ZIP`,
+		conv)
+	// Download ZIP file and unzips its files
+	zip := fmt.Sprintf("%s/COTAHIST_D%s.ZIP", dataDir, conv)
+	files, err := fetchFiles(url, dataDir, zip)
+	if err != nil {
+		return err
+	}
+
+	// Delete files on return
+	defer filesCleanup(files)
+
+	// Parse and store files content
+	for _, f := range files {
+		fh, err := os.Open(f)
 		if err != nil {
-			return 0, err
+			return errors.Wrapf(err, "abrindo arquivo %s", f)
+		}
+		defer fh.Close()
+
+		dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())
+
+		_, err = s.store.Save(dec, "")
+		if err != nil {
+			return err
 		}
 	}
 
-	return s.store.Quote(code, date)
+	return nil
 }
 
 //

fetch/fetch_stockquote_test.go

@@ -12,10 +12,14 @@ type MockStockFetch struct {
 	// store rapina.StockStore
 }
 
-func (m MockStockFetch) Save(stream io.ReadCloser, code string) (int, error) {
+func (m MockStockFetch) Save(stream io.Reader, code string) (int, error) {
 	return 1, nil
 }
 
+func (m MockStockFetch) SaveB3Quotes(filename string) error {
+	return nil
+}
+
 func (m MockStockFetch) Quote(code, date string) (float64, error) {
 	// fmt.Printf("calling mock Quote(%s, %s)\n", code, date)
 	return 123.45, nil

fetch/unzip.go

@@ -88,7 +88,7 @@ func valid(filename string) bool {
 		return false
 	}
 
-	list := []string{"_bpa_", "_bpp_", "_dfc_", "_dre_", "_dva_", "fre_"}
+	list := []string{"_bpa_", "_bpp_", "_dfc_", "_dre_", "_dva_", "fre_", "cotahist_"}
 
 	for _, item := range list {
 		if strings.Contains(n, item) {

interfaces.go

@@ -10,7 +10,8 @@ type FIIParser interface {
 }
 
 type StockStore interface {
-	Save(stream io.ReadCloser, code string) (int, error)
+	Save(stream io.Reader, code string) (int, error)
+	SaveB3Quotes(filename string) error
 	Quote(code, date string) (float64, error)
 }
 

parsers/parsers_stockquote.go

@@ -9,13 +9,17 @@ package parsers
 import (
 	"bufio"
 	"database/sql"
+	"fmt"
 	"io"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 
 	"github.com/dude333/rapina"
 	"github.com/pkg/errors"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/transform"
 )
 
 type stockQuote struct {
@@ -57,11 +61,141 @@ func (s *StockParser) Quote(code, date string) (float64, error) {
 	return close, nil
 }
 
+func (s *StockParser) SaveB3Quotes(filename string) error {
+	isNew, err := isNewFile(s.db, filename)
+	if !isNew && err == nil { // if error, process file
+		s.log.Info("%s já processado anteriormente", filename)
+		return errors.New("este arquivo de cotações já foi importado anteriormente")
+	}
+
+	if err := s.populateStockQuotes(filename); err != nil {
+		return err
+	}
+
+	storeFile(s.db, filename)
+
+	return nil
+}
+
+func (s *StockParser) populateStockQuotes(filename string) error {
+	fh, err := os.Open(filename)
+	if err != nil {
+		return errors.Wrapf(err, "abrindo arquivo %s", filename)
+	}
+	defer fh.Close()
+
+	// BEGIN TRANSACTION
+	tx, err := s.db.Begin()
+	if err != nil {
+		return errors.Wrap(err, "Failed to begin transaction")
+	}
+
+	dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())
+	scanner := bufio.NewScanner(dec)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		q, err := parseB3(line)
+		if err != nil {
+			continue // ignore line
+		}
+		fmt.Printf("%+v\n", q)
+	}
+
+	// END TRANSACTION
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "Failed to commit transaction")
+	}
+
+	if err := scanner.Err(); err != nil {
+		return errors.Wrapf(err, "lendo arquivo %s", filename)
+	}
+
+	return nil
+}
+
+// parseB3 parses the line based on this layout:
+// http://www.b3.com.br/data/files/33/67/B9/50/D84057102C784E47AC094EA8/SeriesHistoricas_Layout.pdf
+//
+//   CAMPO/CONTEÚDO  TIPO E TAMANHO  POS. INIC.	 POS. FINAL
+//   TIPREG “01”     N(02)           01          02
+//   DATA “AAAAMMDD” N(08)           03          10
+//   CODBDI          X(02)           11          12
+//   CODNEG          X(12)           13          24
+//   TPMERC          N(03)           25          27
+//   PREABE          (11)V99         57          69
+//   PREMAX          (11)V99         70          82
+//   PREMIN          (11)V99         83          95
+//   PREULT          (11)V99         109         121
+//   QUATOT          N18             153         170
+//   VOLTOT          (16)V99         171         188
+//
+// CODBDI:
+//   02 LOTE PADRÃO
+//   12 FUNDO IMOBILIÁRIO
+//
+// TPMERC:
+//   010 VISTA
+//   020 FRACIONÁRIO
+func parseB3(line string) (*stockQuote, error) {
+	if len(line) != 245 {
+		return nil, errors.New("linha deve conter 245 bytes")
+	}
+
+	recType := line[0:2]
+	if recType != "01" {
+		return nil, fmt.Errorf("registro %s ignorado", recType)
+	}
+
+	codBDI := line[10:12]
+	if codBDI != "02" && codBDI != "12" {
+		return nil, fmt.Errorf("BDI %s ignorado", codBDI)
+	}
+
+	tpMerc := line[24:27]
+	if tpMerc != "010" && tpMerc != "020" {
+		return nil, fmt.Errorf("tipo de mercado %s ignorado", tpMerc)
+	}
+
+	date := line[2:6] + "-" + line[6:8] + "-" + line[8:10]
+	code := strings.TrimSpace(line[12:24])
+
+	numRanges := [5]struct {
+		i, f int
+	}{
+		{56, 69},   // PREABE = open
+		{69, 82},   // PREMAX = high
+		{82, 95},   // PREMIN = low
+		{108, 121}, // PREULT = close
+		{170, 188}, // VOLTOT = volume
+	}
+	var vals [5]int
+	for i, r := range numRanges {
+		num, err := strconv.Atoi(line[r.i:r.f])
+		if err != nil {
+			return nil, err
+		}
+		vals[i] = num
+	}
+
+	return &stockQuote{
+		Stock:  code,
+		Date:   date,
+		Open:   float64(vals[0]) / 100,
+		High:   float64(vals[1]) / 100,
+		Low:    float64(vals[2]) / 100,
+		Close:  float64(vals[3]) / 100,
+		Volume: float64(vals[4]) / 100,
+	}, nil
+}
+
 //
 // Save parses the 'stream', get the 'code' stock quotes and
 // store it on 'db'. Returns the number of registers saved.
 //
-func (s *StockParser) Save(stream io.ReadCloser, code string) (int, error) {
+func (s *StockParser) Save(stream io.Reader, code string) (int, error) {
 	if s.db == nil {
 		return 0, errors.New("bd inválido")
 	}
@@ -90,10 +224,12 @@ func (s *StockParser) Save(stream io.ReadCloser, code string) (int, error) {
 		var q *stockQuote
 		var err error
 		switch prov {
-		case alphaVantage:
-			q, err = parseAlphaVantage(line, code)
+		case b3:
+			q, err = parseB3(line)
 		case yahoo:
 			q, err = parseYahoo(line, code)
+		case alphaVantage:
+			q, err = parseAlphaVantage(line, code)
 		}
 		if err != nil {
 			continue // ignore lines with error
@@ -172,6 +308,7 @@ const (
 	none int = iota
 	alphaVantage
 	yahoo
+	b3
 )
 
 // provider returns stream type based on header
@@ -182,6 +319,9 @@ func provider(header string) int {
 	if header == "Date,Open,High,Low,Close,Adj Close,Volume" {
 		return yahoo
 	}
+	if strings.HasPrefix(header, "00COTAHIST.") {
+		return b3
+	}
 	return none
 }