Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DEV] Crawler config changes. Changes for validate_metadata test. #57

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[DEV] Crawler config changes. Changes for validate_metadata test. #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9ea5216
Added headers and cookies to config file. Changes for validate_metada…
Mar 15, 2021
185551c
added Config class
Apr 2, 2021
f13b1d7
test changes with Config usage
Apr 2, 2021
c27eb95
fix import
Apr 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions config/raw_metadata_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import re
import os
import html
import json
import os
import re
import unittest

import requests

from constants import ASSETS_PATH, CRAWLER_CONFIG_PATH
from crawler_config import Config


class RawDataValidator(unittest.TestCase):
Expand Down Expand Up @@ -42,13 +46,19 @@ def test_validate_sort(self):
msg="""Articles ids are not homogeneous. E.g. numbers are not from 1 to N""")

def test_validate_metadata(self):
config = Config(CRAWLER_CONFIG_PATH)
session = requests.Session()

session.headers.update(config['headers'])
session.cookies.update(config['cookies'])

# can i open this URL?
for metadata in self.metadata:
self.assertTrue(requests.get(metadata[1]['url']),
self.assertTrue(session.get(metadata[1]['url']),
msg="Can not open URL: <{}>. Check how you collect URLs".format(
metadata[1]['url']))

html_source = requests.get(metadata[1]['url']).text
html_source = html.unescape(session.get(metadata[1]['url']).text)

self.assertTrue(metadata[1]['title'] in
html_source,
Expand Down
4 changes: 3 additions & 1 deletion crawler_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{
"base_urls": [],
"total_articles_to_find_and_parse": 0,
"max_number_articles_to_get_from_one_seed": 0
"max_number_articles_to_get_from_one_seed": 0,
"headers": {},
"cookies": {}
}