Merge pull request #89 from dantleech/gh-88-referrer-title

Ensure that URL is recorded if client throws exception when requestin…
dantleech · Jun 2, 2019 · 5cd4d02 · 5cd4d02
2 parents 720a106 + 3e6c4c6
commit 5cd4d02
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## develop
+
+- Record referrer title if client encounters error while requesting URL, fixes #88
+
 ## [0.8.0] 2019-04-07
 
 - Record the request timestamp #33

diff --git a/lib/Model/Crawler.php b/lib/Model/Crawler.php
@@ -25,11 +25,11 @@ public function __construct(Client $client)
     public function crawl(Url $documentUrl, UrlQueue $queue, ReportBuilder $report): Generator
     {
         $start = microtime(true);
+        $report->withReferringElement($documentUrl->referringElement());
         $response = yield $this->client->request($documentUrl->__toString());
         $time = (microtime(true) - $start) * 1E6;
 
         $report->withRequestTime((int) $time);
-        $report->withReferringElement($documentUrl->referringElement());
 
         assert($response instanceof Response);
         $report->withStatus($response->getStatus());

diff --git a/tests/EndToEnd/Command/CrawlCommandTest.php b/tests/EndToEnd/Command/CrawlCommandTest.php
@@ -365,6 +365,21 @@ public function testShowsTheReferringLinkText()
         $this->assertEquals('/html/body/ul/li[1]/a', $url['referrer_xpath']);
     }
 
+    public function testShowsReferrerIfExceptionEncountered()
+    {
+        $process = $this->execute([
+            self::EXAMPLE_URL,
+            '--output='.$this->workspace()->path('/out.json'),
+        ], 'malformed-host');
+        $this->assertProcessSuccess($process);
+        $rows = $this->parseResults($this->workspace()->path('/out.json'));
+
+        $this->assertCount(2, $rows);
+        $row = $rows[1];
+        $this->assertNotNull($row['exception']);
+        $this->assertEquals('This is a link', $row['referrer_title']);
+    }
+
     public function testAllowsDisplayCustomization()
     {
         $process = $this->execute([

diff --git a/tests/Example/malformed-host/index.html b/tests/Example/malformed-host/index.html
@@ -0,0 +1,3 @@
+<html>
+    <a href="https://test-not-resolving1234.com">This is a link</a>
+</html>