From d80d8b73cfb33d519d859920c4011f68075903b9 Mon Sep 17 00:00:00 2001 From: freek Date: Thu, 15 Dec 2016 13:22:08 +0100 Subject: [PATCH] wip --- CHANGELOG.md | 3 +++ src/Crawler.php | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 697a2cd..2380959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All Notable changes to `spatie/crawler` will be documented in this file. +## 2.0.5 - 2016-12-15 +- fix detection of redirects + ## 2.0.4 - 2016-12-15 - fix the default timeout of 5 seconds diff --git a/src/Crawler.php b/src/Crawler.php index ff04f8a..c13e09a 100644 --- a/src/Crawler.php +++ b/src/Crawler.php @@ -40,7 +40,10 @@ class Crawler */ public static function create(array $clientOptions = []) { - $client = new Client($clientOptions); + $client = new Client($clientOptions ?? [ + RequestOptions::ALLOW_REDIRECTS => false, + RequestOptions::COOKIES => true, + ]); return new static($client); } @@ -118,9 +121,8 @@ protected function startCrawlingQueue() $pool = new Pool($this->client, $this->getCrawlRequests(), [ 'concurrency' => $this->concurrency, 'options' => [ - RequestOptions::CONNECT_TIMEOUT => 5, - RequestOptions::TIMEOUT => 5, - + RequestOptions::CONNECT_TIMEOUT => 10, + RequestOptions::TIMEOUT => 10, ], 'fulfilled' => function (ResponseInterface $response, int $index) { $this->handleResponse($response, $index);