increased the throttle rate by factor of 5 (#31)

petabridge · Apr 14, 2019 · 0331c1c · 0331c1c
1 parent d754c66
commit 0331c1c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/WebCrawler.Shared.IO/DownloadCoordinator.cs b/src/WebCrawler.Shared.IO/DownloadCoordinator.cs
@@ -50,7 +50,7 @@ public DownloadCoordinator(CrawlJob job, IActorRef commander, IActorRef download
             var selfDocSink = Sink.ActorRef<CompletedDocument>(Self, StreamCompleteTick.Instance);
             var selfImgSink = Sink.ActorRef<CompletedDocument>(Self, StreamCompleteTick.Instance);
             var htmlFlow = Flow.Create<CrawlDocument>().Via(DownloadFlow.SelectDocType())
-                .Throttle(30, TimeSpan.FromSeconds(1), 100, ThrottleMode.Shaping)
+                .Throttle(30, TimeSpan.FromSeconds(5), 100, ThrottleMode.Shaping)
                 .Via(DownloadFlow.ProcessHtmlDownloadFor(DefaultMaxConcurrentDownloads, HttpClientFactory.GetClient()));
 
             var imageFlow = Flow.Create<CrawlDocument>()