Execute javascript before observers are run (#166)

* Execute javascript before observers are run * Apply fixes from StyleCI
spatie · Jun 25, 2018 · 8f63cde · 8f63cde
1 parent f55f259
commit 8f63cde
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 13 deletions.
diff --git a/src/Handlers/CrawlRequestFulfilled.php b/src/Handlers/CrawlRequestFulfilled.php
@@ -6,9 +6,11 @@
 use Spatie\Crawler\CrawlUrl;
 use Spatie\Crawler\LinkAdder;
 use Spatie\Crawler\CrawlerRobots;
+use Psr\Http\Message\UriInterface;
 use Spatie\Crawler\CrawlSubdomains;
 use Psr\Http\Message\StreamInterface;
 use Psr\Http\Message\ResponseInterface;
+use function GuzzleHttp\Psr7\stream_for;
 
 class CrawlRequestFulfilled
 {
@@ -35,6 +37,12 @@ public function __invoke(ResponseInterface $response, $index)
 
         $crawlUrl = $this->crawler->getCrawlQueue()->getUrlById($index);
 
+        if ($this->crawler->mayExecuteJavaScript()) {
+            $html = $this->getBodyAfterExecutingJavaScript($crawlUrl->url);
+
+            $response = $response->withBody(stream_for($html));
+        }
+
         $this->handleCrawled($response, $crawlUrl);
 
         if (! $this->crawler->getCrawlProfile() instanceof CrawlSubdomains) {
@@ -65,4 +73,13 @@ protected function convertBodyToString(StreamInterface $bodyStream, $readMaximum
 
         return $body;
     }
+
+    protected function getBodyAfterExecutingJavaScript(UriInterface $url): string
+    {
+        $browsershot = $this->crawler->getBrowsershot();
+
+        $html = $browsershot->setUrl((string) $url)->bodyHtml();
+
+        return html_entity_decode($html);
+    }
 }
diff --git a/src/LinkAdder.php b/src/LinkAdder.php
@@ -57,10 +57,6 @@ public function addFromHtml(string $html, UriInterface $foundOnUrl)
      */
     protected function extractLinksFromHtml(string $html, UriInterface $foundOnUrl)
     {
-        if ($this->crawler->mayExecuteJavaScript()) {
-            $html = $this->getBodyAfterExecutingJavaScript($foundOnUrl);
-        }
-
         $domCrawler = new DomCrawler($html, $foundOnUrl);
 
         return collect($domCrawler->filterXpath('//a')->links())
@@ -101,13 +97,4 @@ protected function shouldCrawl(Node $node): bool
 
         return $node->getDepth() <= $maximumDepth;
     }
-
-    protected function getBodyAfterExecutingJavaScript(UriInterface $foundOnUrl): string
-    {
-        $browsershot = $this->crawler->getBrowsershot();
-
-        $html = $browsershot->setUrl((string) $foundOnUrl)->bodyHtml();
-
-        return html_entity_decode($html);
-    }
 }