chore: introduce load table total rows (#3851)

rudderlabs · Sep 29, 2023 · 1fcabdd · 1fcabdd
1 parent be24be3
commit 1fcabdd
Show file tree

Hide file tree

Showing 49 changed files with 5,709 additions and 1,889 deletions.
diff --git a/.github/tools/matrixchecker/main.go b/.github/tools/matrixchecker/main.go
@@ -19,6 +19,7 @@ var IgnorePackages = []string{
 	"warehouse/integrations/testhelper",
 	"warehouse/integrations/testdata",
 	"warehouse/integrations/config",
+	"warehouse/integrations/types",
 }
 
 func main() {

diff --git a/Makefile b/Makefile
@@ -31,7 +31,7 @@ endif
 test-warehouse-integration:
 	$(eval TEST_PATTERN = 'TestIntegration')
 	$(eval TEST_CMD = SLOW=1 go test)
-	$(eval TEST_OPTIONS = -v -p 8 -timeout 30m -count 1 -race -run $(TEST_PATTERN) -coverprofile=profile.out -covermode=atomic -coverpkg=./...)
+	$(eval TEST_OPTIONS = -v -p 8 -timeout 30m -count 1 -run $(TEST_PATTERN) -coverprofile=profile.out -covermode=atomic -coverpkg=./...)
 	$(TEST_CMD) $(TEST_OPTIONS) $(package) && touch $(TESTFILE) || true
 
 test-warehouse: test-warehouse-integration test-teardown

diff --git a/warehouse/client/client.go b/warehouse/client/client.go
@@ -82,10 +82,10 @@ func (cl *Client) bqQuery(statement string) (result warehouseutils.QueryResult,
 	for {
 		var row []bigquery.Value
 		err = it.Next(&row)
-		if err == iterator.Done {
-			break
-		}
 		if err != nil {
+			if errors.Is(err, iterator.Done) {
+				break
+			}
 			return
 		}
 		var stringRow []string

diff --git a/warehouse/identity/identity.go b/warehouse/identity/identity.go
@@ -4,6 +4,7 @@ import (
 	"compress/gzip"
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -243,7 +244,7 @@ func (idr *Identity) addRules(txn *sqlmiddleware.Tx, loadFileNames []string, gzW
 			var record []string
 			record, err = eventReader.Read(columnNames)
 			if err != nil {
-				if err == io.EOF {
+				if errors.Is(err, io.EOF) {
 					break
 				}
 				pkgLogger.Errorf("IDR: Error while reading merge rule file %s for loading in staging table locally:%s: %v", loadFileName, mergeRulesStagingTable, err)