Fix no oikotieapartment cookies modal (#54)

* Continue if no modal * Bump version
jmyrberg · Nov 21, 2020 · f90399a · f90399a
1 parent 29d7d92
commit f90399a
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.1.0b
+0.1.1b
diff --git a/finscraper/scrapy_spiders/oikotieapartment.py b/finscraper/scrapy_spiders/oikotieapartment.py
@@ -11,6 +11,8 @@
 from scrapy.loader import ItemLoader
 from scrapy.loader.processors import TakeFirst, Identity, Compose
 
+from selenium.common.exceptions import NoSuchElementException
+
 from finscraper.request import SeleniumCallbackRequest
 from finscraper.text_utils import strip_join, drop_empty_elements, \
     paragraph_join
@@ -134,10 +136,12 @@ def start_requests(self):
     @staticmethod
     def _handle_start(request, spider, driver):
         driver.get(request.url)
-        policy_modal = driver.find_element_by_xpath(
-            '//div[contains(@class, "sccm-button-green")]')
-        if policy_modal:
-            policy_modal.click()
+        try:  # Accept cookies modal
+            modal = driver.find_element_by_xpath(
+                '//div[contains(@class, "sccm-button-green")]')
+            modal.click()
+        except NoSuchElementException:
+            pass
         return HtmlResponse(
             driver.current_url,
             body=driver.page_source.encode('utf-8'),