Course project

.gitignore

@@ -588,3 +588,6 @@
 /venv/lib/python3.7/site-packages/pymongo/uri_parser.py
 /venv/lib/python3.7/site-packages/pymongo-3.9.0.dist-info/WHEEL
 /venv/lib/python3.7/site-packages/pymongo/write_concern.py
+/venv/
+/.idea/
+/.env

README.md

@@ -1 +1,32 @@
-# Data_Mining
+# Data_Mining
+
+В файле runner.py на вход подаются два айдишника юзеров вк.
+Алгоритм использует т.н. Breadth-first search вместо стандартного Depth-first search.
+
+На первом шаге идет проверка, не состоят ли переданные айдишники в друзьях,
+на втором: нет ли у переданных айдишников общих друзей,
+на третьем и последующих:   - идет запрос списка друзей юзера_1.
+                            - Список сплитится на списки по 100 юзеров (ограничение апи вк)
+                            - проверяется нет ли общих друзей у юзера_2 с переданным списком из 100 айдишников
+
+На выходе в монго записывается первая найденная цепочка друзей.
+
+Интуитивно, мне показался этот способ быстрее, чем прямой перебор друзей, хочу потестить так ли это, но до сдачи уже не успею
+
+Не успел написать приятный аутпут процессор, поэтому расшифрую результат на конкретном примере одного айтема:
+
+{'_id': ObjectId('5dacd5ec6967cc57cb1daafa'),
+ 'friends_chain': [19587588,                                <- юзер_1 (передан на входе)
+                   81771,                                   <- друг_1 в цепочке
+                   [{'common_count': 1,
+                     'common_friends': [178913514],         <- список общих друзей друга_2 и юзера_2
+                     'id': 11081576},                       <- друг_2 в цепочке
+                    {'common_count': 1,
+                     'common_friends': [152465],            <- список общих друзей другого_друга_2 и юзера_2
+                     'id': 11264606}],                      <- другой_друг_2 в цепочке
+                   20367747]}                               <- юзер_2 (передан на входе)
+
+Фактически, выше представлены 2 цепочки, отличающиеся 3 и 4 айдишниками:
+[19587588, 81771, 11081576, 178913514, 20367747]
+и
+[19587588, 81771, 11264606, 152465, 20367747]

find_connections_vk/items.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+from scrapy.loader.processors import MapCompose, TakeFirst
+
+
+class FindConnectionsVkItem(scrapy.Item):
+    _id = scrapy.Field()
+    friends_chain = scrapy.Field()

find_connections_vk/middlewares.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+from scrapy.downloadermiddlewares.retry import RetryMiddleware
+from scrapy.utils.response import response_status_message
+import time
+import json
+
+
+class TooManyRequestsRetryMiddleware(RetryMiddleware):
+
+    def __init__(self, crawler):
+        super(TooManyRequestsRetryMiddleware, self).__init__(crawler.settings)
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def process_response(self, request, response, spider):
+        if request.meta.get('dont_retry', False):
+            return response
+        elif json.loads(response.body).get('error'):
+            self.crawler.engine.pause()
+            time.sleep(1)  # If the rate limit is renewed in a minute, put 60 seconds, and so on.
+            self.crawler.engine.unpause()
+            reason = response_status_message(response.status)
+            return self._retry(request, reason, spider) or response
+        elif response.status in self.retry_http_codes:
+            reason = response_status_message(response.status)
+            return self._retry(request, reason, spider) or response
+        return response
+
+class FindConnectionsVkSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class FindConnectionsVkDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

find_connections_vk/pipelines.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+from pymongo import MongoClient
+
+
+class FindConnectionsVkPipeline(object):
+    def __init__(self):
+        client = MongoClient('localhost', 27017)
+        self.mongo_base = client.vk_connections
+
+    def process_item(self, item, spider):
+        mongo_coll_name = f"{spider.name} for {item.get('friends_chain')[0]} and {item.get('friends_chain')[-1]}"
+        collection = self.mongo_base[mongo_coll_name]
+        collection.insert_one(item)
+        return item

find_connections_vk/settings.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for find_connections_vk project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'find_connections_vk'
+
+SPIDER_MODULES = ['find_connections_vk.spiders']
+NEWSPIDER_MODULE = 'find_connections_vk.spiders'
+
+LOG_ENABLED = True
+LOG_LEVEL = 'DEBUG'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'find_connections_vk (+http://www.yourdomain.com)'
+USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 " \
+                          "(KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 "
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+CONCURRENT_REQUESTS = 25
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+DOWNLOAD_DELAY = 0.2
+# The download delay setting will honor only one of:
+CONCURRENT_REQUESTS_PER_DOMAIN = 25
+CONCURRENT_REQUESTS_PER_IP = 25
+
+DEPTH_PRIORITY = 1
+SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
+SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
+
+# Disable cookies (enabled by default)
+COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#    'find_connections_vk.middlewares.FindConnectionsVkSpiderMiddleware': 543,
+# }
+URLLENGTH_LIMIT = 80000
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+DOWNLOADER_MIDDLEWARES = {
+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
+    'find_connections_vk.middlewares.TooManyRequestsRetryMiddleware': 543,
+}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+   'find_connections_vk.pipelines.FindConnectionsVkPipeline': 300,
+}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

find_connections_vk/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.