Added capability of crawling links with rel= next or prev. (#207)

* Added capability of crawling links with rel= next or prev. * Removed extra closing head tag in test server.
spatie · Mar 11, 2019 · 64303c3 · 64303c3
1 parent 2ae1901
commit 64303c3
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 44 deletions.
diff --git a/src/LinkAdder.php b/src/LinkAdder.php
@@ -61,7 +61,7 @@ protected function extractLinksFromHtml(string $html, UriInterface $foundOnUrl)
     {
         $domCrawler = new DomCrawler($html, $foundOnUrl);
 
-        return collect($domCrawler->filterXpath('//a')->links())
+        return collect($domCrawler->filterXpath('//a | //link[@rel="next" or @rel="prev"]')->links())
             ->reject(function (Link $link) {
                 return $link->getNode()->getAttribute('rel') === 'nofollow';
             })

diff --git a/tests/CrawlerTest.php b/tests/CrawlerTest.php
@@ -392,6 +392,8 @@ protected function regularUrls(): array
         return [
             ['url' => 'http://localhost:8080/'],
             ['url' => 'http://localhost:8080/link1', 'foundOn' => 'http://localhost:8080/'],
+            ['url' => 'http://localhost:8080/link1-prev', 'foundOn' => 'http://localhost:8080/link1'],
+            ['url' => 'http://localhost:8080/link1-next', 'foundOn' => 'http://localhost:8080/link1'],
             ['url' => 'http://localhost:8080/link2', 'foundOn' => 'http://localhost:8080/'],
             ['url' => 'http://localhost:8080/link3', 'foundOn' => 'http://localhost:8080/link2'],
             ['url' => 'http://localhost:8080/notExists', 'foundOn' => 'http://localhost:8080/link3'],

diff --git a/tests/server/package-lock.json b/tests/server/package-lock.json
diff --git a/tests/server/server.js b/tests/server/server.js
@@ -7,13 +7,21 @@ app.get('/', function (request, response) {
 });
 
 app.get('/link1', function (request, response) {
-    response.end('<html><body><script>var url = \'/javascript\';document.body.innerHTML = document.body.innerHTML + "<a href=\'" + url + "\'>Javascript Link</a>"</script>You are on link1<a href="http://example.com/">External Link</a></body></html>');
+    response.end('<html><head><link rel="next" href="/link1-next"><link rel="prev" href="/link1-prev"></head><body><script>var url = \'/javascript\';document.body.innerHTML = document.body.innerHTML + "<a href=\'" + url + "\'>Javascript Link</a>"</script>You are on link1<a href="http://example.com/">External Link</a></body></html>');
 });
 
 app.get('/javascript', function (request, response) {
     response.end('This page can only be reached if JavaScript is being executed');
 });
 
+app.get('/link1-next', function (request, response) {
+    response.end('You are on link1-next. Next page of link1');
+});
+
+app.get('/link1-prev', function (request, response) {
+    response.end('You are on link1-prev. Previous page of link1');
+});
+
 app.get('/nofollow', function (request, response) {
     response.end('This page should not be crawled');
 });