fix for exception being thrown when encountering a malformatted url

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 All notable changes to `spatie/crawler` will be documented in this file.
 
+## 3.0.1 - 2018-02-01
+- fix for exception being thrown when encountering a malformatted url
+
 ## 3.0.0 - 2017-12-22
 - use `\Psr\Http\Message\UriInterface` for all urls
 - use Puppeteer

src/Crawler.php

@@ -18,6 +18,7 @@
 use GuzzleHttp\Exception\RequestException;
 use Spatie\Crawler\CrawlQueue\CollectionCrawlQueue;
 use Symfony\Component\DomCrawler\Crawler as DomCrawler;
+use InvalidArgumentException;
 
 class Crawler
 {
@@ -327,8 +328,13 @@ protected function extractAllLinks(string $html, UriInterface $foundOnUrl): Coll
 
         return collect($domCrawler->filterXpath('//a')->links())
             ->map(function (Link $link) {
-                return new Uri($link->getUri());
-            });
+                try {
+                    return new Uri($link->getUri());
+                } catch (InvalidArgumentException $exception) {
+                    return null;
+                }
+            })
+            ->filter();
     }
 
     protected function normalizeUrl(UriInterface $url): UriInterface

tests/CrawlerTest.php

@@ -133,6 +133,26 @@ public function it_uses_crawl_profile_for_internal_urls()
         ]);
     }
 
+    /** @test */
+    public function it_can_handle_pages_with_invalid_urls()
+    {
+        $crawlProfile = new class implements CrawlProfile {
+            public function shouldCrawl(UriInterface $url): bool
+            {
+                return true;
+            }
+        };
+
+        Crawler::create()
+            ->setCrawlObserver(new CrawlLogger())
+            ->setCrawlProfile($crawlProfile)
+            ->startCrawling('localhost:8080/invalid-url');
+
+        $this->assertCrawledOnce([
+                ['url' => 'http://localhost:8080/invalid-url'],
+            ]);
+    }
+
     /** @test */
     public function it_respects_the_maximum_amount_of_urls_to_be_crawled()
     {

tests/server/server.js

@@ -34,6 +34,10 @@ app.get('/dir/subdir/link6', function (request, response) {
     response.end('You are on /dir/subdir/link6<a href="/link1">link 1</a>');
 });
 
+app.get('/invalid-url', function (request, response) {
+    response.end('There is an <a href="https:///AfyaVzw">invalid</a> url');
+});
+
 let server = app.listen(8080, function () {
     const host = 'localhost';
     const port = server.address().port;