Improve logging

db/db.go

@@ -2,6 +2,7 @@ package db
 
 import (
 	"database/sql"
+	"encoding/base64"
 	"fmt"
 	"net/url"
 	"strconv"
@@ -99,7 +100,10 @@ func (db *Database) stateS3toDB(state *terraform.State, path string, versionId s
 
 			for k, v := range r.Primary.Attributes {
 				if !isASCII(v) {
-					log.Infof("Attribute %s has non-ASCII value %v, skipping", k, v)
+					log.WithFields(log.Fields{
+						"key":          k,
+						"value_base64": base64.StdEncoding.EncodeToString([]byte(v)),
+					}).Info("Attribute has non-ASCII value, skipping")
 					continue
 				}
 				res.Attributes = append(res.Attributes, Attribute{
@@ -183,7 +187,9 @@ type SearchResult struct {
 }
 
 func (db *Database) SearchAttribute(query url.Values) (results []SearchResult, page int, total int) {
-	log.Infof("Searching for attribute with query=%v", query)
+	log.WithFields(log.Fields{
+		"query": query,
+	}).Info("Searching for attribute with query")
 
 	targetVersion := string(query.Get("versionid"))
 

main.go

@@ -59,27 +59,40 @@ func refreshDB(d *db.Database) {
 		log.Infof("Refreshing DB from S3")
 		states, err := s3.GetStates()
 		if err != nil {
-			log.Errorf("Failed to build cache: %s", err)
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("Failed to retrieve states from S3. Retrying in 1 minute.")
+			time.Sleep(1 * time.Minute)
+			continue
 		}
 
 		statesVersions := d.ListStatesVersions()
 		for _, st := range states {
 			versions, _ := s3.GetVersions(st)
 			for _, v := range versions {
 				if _, ok := statesVersions[*v.VersionId]; ok {
-					log.Debugf("Version %s is already in the DB", *v.VersionId)
+					log.WithFields(log.Fields{
+						"version_id": *v.VersionId,
+					}).Debug("Version is already in the database, skipping")
 				} else {
 					d.InsertVersion(v)
 				}
 
 				if isKnownStateVersion(statesVersions, *v.VersionId, st) {
-					log.Debugf("State %s/%s is already in the DB, skipping", st, *v.VersionId)
+					log.WithFields(log.Fields{
+						"path":       st,
+						"version_id": *v.VersionId,
+					}).Debug("State is already in the database, skipping")
 					continue
 				}
 				state, _ := s3.GetState(st, *v.VersionId)
 				d.InsertState(st, *v.VersionId, state)
 				if err != nil {
-					log.Errorf("Failed to insert state %s/%s: %v", st, *v.VersionId, err)
+					log.WithFields(log.Fields{
+						"path":       st,
+						"version_id": *v.VersionId,
+						"error":      err,
+					}).Error("Failed to insert state in the database")
 				}
 			}
 		}

s3/s3.go

@@ -113,7 +113,10 @@ func GetVersions(prefix string) (versions []*s3.ObjectVersion, err error) {
 }
 
 func GetState(st, versionId string) (state *terraform.State, err error) {
-	log.Infof("Retrieving %s/%s from S3", st, versionId)
+	log.WithFields(log.Fields{
+		"path":       st,
+		"version_id": versionId,
+	}).Info("Retrieving state from S3")
 	input := &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(st),
@@ -123,7 +126,11 @@ func GetState(st, versionId string) (state *terraform.State, err error) {
 	}
 	result, err := svc.GetObjectWithContext(context.Background(), input)
 	if err != nil {
-		log.Errorf("Error retrieving %s/%s from S3: %v", st, versionId, err)
+		log.WithFields(log.Fields{
+			"path":       st,
+			"version_id": versionId,
+			"error":      err,
+		}).Error("Error retrieving state from S3")
 		errObj := make(map[string]string)
 		errObj["error"] = fmt.Sprintf("State file not found: %v", st)
 		errObj["details"] = fmt.Sprintf("%v", err)
@@ -134,7 +141,11 @@ func GetState(st, versionId string) (state *terraform.State, err error) {
 
 	content, err := ioutil.ReadAll(result.Body)
 	if err != nil {
-		log.Errorf("Error reading %s/%s from S3: %v", st, versionId, err)
+		log.WithFields(log.Fields{
+			"path":       st,
+			"version_id": versionId,
+			"error":      err,
+		}).Error("Error reading state from S3")
 		errObj := make(map[string]string)
 		errObj["error"] = fmt.Sprintf("Failed to read S3 response: %v", st)
 		errObj["details"] = fmt.Sprintf("%v", err)