implemented skipping transactions instead of failing with fatal error…

… details of which cannot be fetched

refactoring
added interfaces for tests

README.md

@@ -146,7 +146,7 @@ Mar 28 12:02:09.453 [INFO] [id:xxxxxxx-xxxxx-xxxxx-xxxxx-xxxx] Fetching transact
 Mar 28 12:02:09.485 [INFO] [id:xxxxxxx-xxxxx-xxxxx-xxxxx-xxxx] Processing transaction details
 Mar 28 12:02:09.488 [INFO] [id:xxxxxxx-xxxxx-xxxxx-xxxxx-xxxx] Unsupported transaction skipped
 ...
-Mar 28 12:02:27.379 [INFO] All data processed
+Mar 28 12:02:27.379 [INFO] Completed: 200, skipped: 47
 ```
 
 ### CSV File Fields
@@ -188,8 +188,6 @@ Please create a pull request with your changes if you have something to contribu
 
 ## Closing Words
 
-This project
-
- and its contributors have no affiliation with Trade Republic Bank GmbH.
+This project and its contributors have no affiliation with Trade Republic Bank GmbH.
 
 Trade Republic is a registered trademark of Trade Republic Bank GmbH.

cmd/portfoliodownloader/app.go

@@ -9,22 +9,23 @@ import (
 
 	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/timeline/details"
 	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/timeline/transactions"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/websocket"
 	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/portfolio/transaction"
 )
 
 type App struct {
-	transactionsClient    transactions.Client
+	transactionsClient    transactions.ClientInterface
 	eventTypeResolver     transactions.EventTypeResolverInterface
-	timelineDetailsClient details.Client
-	transactionProcessor  transaction.Processor
+	timelineDetailsClient details.ClientInterface
+	transactionProcessor  transaction.ProcessorInterface
 	logger                *log.Logger
 }
 
 func NewApp(
-	transactionsClient transactions.Client,
+	transactionsClient transactions.ClientInterface,
 	eventTypeResolver transactions.EventTypeResolverInterface,
-	timelineDetailsClient details.Client,
-	transactionProcessor transaction.Processor,
+	timelineDetailsClient details.ClientInterface,
+	transactionProcessor transaction.ProcessorInterface,
 	logger *log.Logger,
 ) App {
 	return App{
@@ -37,59 +38,84 @@ func NewApp(
 }
 
 func (a App) Run() error {
-	a.logger.Info("Downloading transactions")
+	counter := 0
 
-	transactionResponses, err := a.transactionsClient.Get()
+	responses, err := a.GetTimelineTransactions()
 	if err != nil {
-		return fmt.Errorf("could not fetch transactions: %w", err)
+		return err
 	}
 
-	a.logger.Infof("%d transactions downloaded", len(transactionResponses))
+	for _, response := range responses {
+		if response.Action.Payload == "" {
+			continue
+		}
 
-	slices.Reverse(transactionResponses)
+		infoFields := log.Fields{"id": response.ID}
+
+		err := a.ProcessTransaction(response)
+
+		switch {
+		case err == nil:
+			counter++
+		case errors.Is(err, websocket.ErrErrorStateReceived):
+			a.logger.WithFields(infoFields).Error(err)
+
+			continue
+		case errors.Is(err, transactions.ErrUnsupportedEventType):
+			a.logger.WithFields(infoFields).Warn("Unsupported transaction skipped")
+
+			continue
+		case errors.Is(err, transaction.ErrUnsupportedType):
+			a.logger.WithFields(infoFields).Warn("Unsupported transaction skipped")
 
-	for _, transactionResponse := range transactionResponses {
-		if transactionResponse.Action.Payload == "" {
 			continue
 		}
+	}
 
-		id := transactionResponse.Action.Payload
-		infoFields := log.Fields{"id": id}
+	skippedCount := len(responses) - counter
 
-		a.logger.WithFields(infoFields).Info("Fetching transaction details")
+	a.logger.Infof("Completed: %d, skipped: %d", counter, skippedCount)
 
-		transactionDetails, err := a.timelineDetailsClient.Get(id)
-		if err != nil {
-			return fmt.Errorf("could not fetch transaction details: %w", err)
-		}
+	return nil
+}
 
-		eventType, err := a.eventTypeResolver.Resolve(transactionResponse)
-		if err != nil {
-			if errors.Is(err, transactions.ErrUnsupportedEventType) {
-				a.logger.WithFields(infoFields).Warn("Unsupported transaction skipped")
+func (a App) GetTimelineTransactions() ([]transactions.ResponseItem, error) {
+	a.logger.Info("Downloading transactions")
+
+	transactionResponses, err := a.transactionsClient.Get()
+	if err != nil {
+		return transactionResponses, fmt.Errorf("could not fetch transactions: %w", err)
+	}
 
-				continue
-			}
+	slices.Reverse(transactionResponses)
 
-			return fmt.Errorf("could not resolve transaction even type: %w", err)
-		}
+	a.logger.Infof("%d transactions downloaded", len(transactionResponses))
 
-		a.logger.WithFields(infoFields).Info("Processing transaction details")
+	return transactionResponses, nil
+}
 
-		if err := a.transactionProcessor.Process(eventType, transactionDetails); err != nil {
-			if errors.Is(err, transaction.ErrUnsupportedType) {
-				a.logger.WithFields(infoFields).Warn("Unsupported transaction skipped")
+func (a App) ProcessTransaction(response transactions.ResponseItem) error {
+	infoFields := log.Fields{"id": response.ID}
 
-				continue
-			}
+	a.logger.WithFields(infoFields).Info("Fetching transaction details")
 
-			return fmt.Errorf("could process transaction: %w", err)
-		}
+	transactionDetails, err := a.timelineDetailsClient.Get(response.Action.Payload)
+	if err != nil {
+		return fmt.Errorf("could not fetch transaction details: %w", err)
+	}
+
+	eventType, err := a.eventTypeResolver.Resolve(response)
+	if err != nil {
+		return fmt.Errorf("could not resolve transaction even type: %w", err)
+	}
+
+	a.logger.WithFields(infoFields).Info("Processing transaction details")
 
-		a.logger.WithFields(infoFields).Info("Transaction processed")
+	if err := a.transactionProcessor.Process(eventType, transactionDetails); err != nil {
+		return fmt.Errorf("could process transaction: %w", err)
 	}
 
-	a.logger.Info("All data processed")
+	a.logger.WithFields(infoFields).Info("Transaction processed")
 
 	return nil
 }

cmd/portfoliodownloader/app_test.go

@@ -0,0 +1,54 @@
+package portfoliodownloader_test
+
+import (
+	"fmt"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
+
+	"github.com/dhojayev/traderepublic-portfolio-downloader/cmd/portfoliodownloader"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/timeline/details"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/timeline/transactions"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/api/websocket"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/filesystem"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/portfolio"
+	"github.com/dhojayev/traderepublic-portfolio-downloader/internal/portfolio/transaction"
+)
+
+func TestApp_Run(t *testing.T) {
+	t.Parallel()
+
+	logger := log.New()
+	ctrl := gomock.NewController(t)
+	transactionsClientMock := transactions.NewMockClientInterface(ctrl)
+	detailsReaderMock := portfolio.NewMockReaderInterface(ctrl)
+	typeResolverMock := transactions.NewMockEventTypeResolverInterface(ctrl)
+	processorMock := transaction.NewMockProcessorInterface(ctrl)
+
+	detailsClient := details.NewClient(detailsReaderMock)
+	app := portfoliodownloader.NewApp(transactionsClientMock, typeResolverMock, detailsClient, processorMock, logger)
+
+	transactionsClientMock.
+		EXPECT().
+		Get().
+		Return([]transactions.ResponseItem{
+			{
+				Action: transactions.ResponseItemAction{
+					Payload: "0e5cf3cb-0f4d-4905-ae5f-ec0a530de6ca",
+					Type:    "timelineDetail",
+				},
+				ID: "0e5cf3cb-0f4d-4905-ae5f-ec0a530de6ca",
+			},
+		}, nil)
+
+	detailsReaderMock.
+		EXPECT().
+		Read(details.DataType, map[string]any{"id": "0e5cf3cb-0f4d-4905-ae5f-ec0a530de6ca"}).
+		Return(filesystem.OutputData{}, fmt.Errorf("could not fetch %s: %w", details.DataType, websocket.ErrErrorStateReceived))
+
+	err := app.Run()
+
+	assert.NoError(t, err)
+}

cmd/portfoliodownloader/dev/wire.go

@@ -46,8 +46,11 @@ var (
 		ProvideInstrumentRepository,
 		ProvideDocumentRepository,
 
+		wire.Bind(new(transactions.ClientInterface), new(transactions.Client)),
 		wire.Bind(new(transactions.EventTypeResolverInterface), new(transactions.EventTypeResolver)),
+		wire.Bind(new(details.ClientInterface), new(details.Client)),
 		wire.Bind(new(details.TypeResolverInterface), new(details.TypeResolver)),
+		wire.Bind(new(transaction.ProcessorInterface), new(transaction.Processor)),
 		wire.Bind(new(transaction.ModelBuilderFactoryInterface), new(transaction.ModelBuilderFactory)),
 		wire.Bind(new(document.ModelBuilderInterface), new(document.ModelBuilder)),
 		wire.Bind(new(transaction.RepositoryInterface), new(*database.Repository[*transaction.Model])),

cmd/portfoliodownloader/public/wire.go

@@ -54,8 +54,11 @@ var (
 		wire.Bind(new(console.AuthServiceInterface), new(*console.AuthService)),
 		wire.Bind(new(portfolio.ReaderInterface), new(*websocket.Reader)),
 
+		wire.Bind(new(transactions.ClientInterface), new(transactions.Client)),
 		wire.Bind(new(transactions.EventTypeResolverInterface), new(transactions.EventTypeResolver)),
+		wire.Bind(new(details.ClientInterface), new(details.Client)),
 		wire.Bind(new(details.TypeResolverInterface), new(details.TypeResolver)),
+		wire.Bind(new(transaction.ProcessorInterface), new(transaction.Processor)),
 		wire.Bind(new(transaction.ModelBuilderFactoryInterface), new(transaction.ModelBuilderFactory)),
 		wire.Bind(new(document.ModelBuilderInterface), new(document.ModelBuilder)),
 		wire.Bind(new(transaction.RepositoryInterface), new(*database.Repository[*transaction.Model])),

internal/api/timeline/details/client.go

@@ -1,3 +1,5 @@
+//go:generate go run -mod=mod go.uber.org/mock/mockgen -source=client.go -destination client_mock.go -package=details
+
 package details
 
 import (
@@ -8,29 +10,33 @@ import (
 )
 
 const (
-	dataType = "timelineDetailV2"
+	DataType = "timelineDetailV2"
 )
 
+type ClientInterface interface {
+	Get(eventID string) (Response, error)
+}
+
 type Client struct {
-	retriever portfolio.ReaderInterface
+	reader portfolio.ReaderInterface
 }
 
-func NewClient(retriever portfolio.ReaderInterface) Client {
+func NewClient(reader portfolio.ReaderInterface) Client {
 	return Client{
-		retriever: retriever,
+		reader: reader,
 	}
 }
 
-func (c *Client) Get(eventID string) (Response, error) {
+func (c Client) Get(eventID string) (Response, error) {
 	var response Response
 
-	msg, err := c.retriever.Read(dataType, map[string]any{"id": eventID})
+	msg, err := c.reader.Read(DataType, map[string]any{"id": eventID})
 	if err != nil {
-		return response, fmt.Errorf("could not fetch %s: %w", dataType, err)
+		return response, fmt.Errorf("could not fetch %s: %w", DataType, err)
 	}
 
 	if err = json.Unmarshal(msg.Data(), &response); err != nil {
-		return response, fmt.Errorf("could not unmarshal %s response: %w", dataType, err)
+		return response, fmt.Errorf("could not unmarshal %s response: %w", DataType, err)
 	}
 
 	return response, nil