Ignore other domain images. Close #41

c4software · Sep 8, 2017 · 18683d0 · 18683d0
1 parent c49a98b
commit 18683d0
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/crawler.py b/crawler.py
@@ -191,6 +191,12 @@ def __crawling(self):
 				if not self.exclude_url(image_link):
 					continue
 
+				# Ignore other domain images
+				image_link_parsed = urlparse(image_link)
+				if image_link_parsed.netloc != self.target_domain:
+					continue
+
+
 				# Test if images as been already seen and not present in the
 				# robot file
 				if self.can_fetch(image_link):