Merge pull request #1211 from tanbirali/feature/1205-Resource-fetchin…

…g-Improvement-OVH fix: Resource fetching performance improvement for OVH
tailwarden · Nov 16, 2023 · e53b263 · e53b263
2 parents 7865a15 + ebffd92
commit e53b263
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 18 deletions.
diff --git a/internal/internal.go b/internal/internal.go
@@ -256,7 +256,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien
 	case "GCP":
 		gcp.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "OVH":
-		ovh.FetchResources(ctx, client, db, telemetry, analytics)
+		ovh.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	}
 }
 

diff --git a/providers/ovh/ovh.go b/providers/ovh/ovh.go
@@ -42,24 +42,26 @@ func listOfSupportedServices() []providers.FetchDataFunction {
 	}
 }
 
-func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics) {
-	for _, fetchResources := range listOfSupportedServices() {
-		resources, err := fetchResources(ctx, client)
-		if err != nil {
-			log.Printf("[%s][OVH] %s", client.Name, err)
-		} else {
-			for _, resource := range resources {
-				_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
-				if err != nil {
-					logrus.WithError(err).Errorf("db trigger failed")
+func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) {
+	wp.SubmitTask(func() {
+		for _, fetchResources := range listOfSupportedServices() {
+			resources, err := fetchResources(ctx, client)
+			if err != nil {
+				log.Printf("[%s][OVH] %s", client.Name, err)
+			} else {
+				for _, resource := range resources {
+					_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
+					if err != nil {
+						logrus.WithError(err).Errorf("db trigger failed")
+					}
+				}
+				if telemetry {
+					analytics.TrackEvent("discovered_resources", map[string]interface{}{
+						"provider":  "OVH",
+						"resources": len(resources),
+					})
 				}
-			}
-			if telemetry {
-				analytics.TrackEvent("discovered_resources", map[string]interface{}{
-					"provider":  "OVH",
-					"resources": len(resources),
-				})
 			}
 		}
-	}
+	})
 }