Skip to content

Commit

Permalink
#15 fix proceeding rating + update example CSV file
Browse files Browse the repository at this point in the history
  • Loading branch information
schuellersa committed Jul 28, 2021
1 parent 44dac27 commit 80c1a23
Show file tree
Hide file tree
Showing 3 changed files with 205 additions and 141 deletions.
289 changes: 149 additions & 140 deletions claim_extractor/extractors/africacheck.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,14 @@ def get_all_claims(criteria):
if 0 < criteria.maxClaims <= len(urls_):
break

# https://africacheck.org/search?rt_bef_combine=created_DESC&sort_by=created&sort_order=DESC&page=
#url = "https://africacheck.org/latest-reports/page/" + str(page_number) + "/"
url = "https://africacheck.org/search?rt_bef_combine=created_DESC&sort_by=created&sort_order=DESC&page=" + str(page_number)

try:
page = requests.get(url, headers=headers, timeout=5)
soup = BeautifulSoup(page.text, "lxml")
soup.prettify()
links = soup.findAll("div", {"class": "article-content"})
#block-mainpagecontent > div > div > div.view-content > div > div:nth-child(1) > div:nth-child(1) > article > div > a > div > article

if (len(links) != 0) or (links != last_page):
for anchor in links:
anchor = anchor.find('a', href=True)
Expand Down Expand Up @@ -160,7 +158,7 @@ def retrieve_urls(self, parsed_listing_page: BeautifulSoup, listing_page_url: st
urls = self.extract_urls(parsed_listing_page)
for page_number in tqdm(range(0, number_of_pages)):
# each page 9 articles:
if (((page_number*9)-9) >= self.configuration.maxClaims):
if ((page_number*9) + 18 >= self.configuration.maxClaims):
break
#url = "https://africacheck.org/latest-reports/page/" + str(page_number) + "/"
url = "https://africacheck.org/search?rt_bef_combine=created_DESC&sort_by=created&sort_order=DESC&page=" + str(page_number)
Expand Down Expand Up @@ -198,39 +196,9 @@ def extract_claim_and_review(self, parsed_claim_review_page: BeautifulSoup, url:
global_date_str = ""
if date:
# global_date_str = search_dates(date['datetime'].split(" ")[0])[0][1].strftime("%Y-%m-%d")
# TODO FASTER without date()!
global_date_str = search_dates(date)[0][1].strftime("%Y-%m-%d")
claim.set_date(global_date_str)

# rating
global_truth_rating = ""
if parsed_claim_review_page.find("div", {"class": "verdict-stamp"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "verdict-stamp"}).get_text()
if parsed_claim_review_page.find("div", {"class": "verdict"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "verdict"}).get_text()
if parsed_claim_review_page.find("div", {"class": "indicator"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "indicator"}).get_text()
if parsed_claim_review_page.find("div", {"class": "indicator"}).find('span'):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "indicator"}).find(
'span').get_text()

# If still no rathing value, try to extract from picture name
if (global_truth_rating == ""):
filename =""
if parsed_claim_review_page.select( 'article > div > div > div > div > p > img' ):
for child in parsed_claim_review_page.select( 'article > div > div > div > div > p > img' ):
if (hasattr( child, 'src' )):
try:
filename=child['src']
continue
except KeyError:
print("KeyError: Skip")

if (filename != ""):
global_truth_rating = filename.split("_")[1].strip(".png")

claim.set_rating(str(re.sub('[^A-Za-z0-9 -]+', '', global_truth_rating)).lower().strip())

# author
author = parsed_claim_review_page.find("div", {"class": "author-details"})
if author:
Expand All @@ -252,124 +220,164 @@ def extract_claim_and_review(self, parsed_claim_review_page: BeautifulSoup, url:
except KeyError:
print("KeyError: Skip")

# when there is no json

# date = parsed_claim_review_page.find("time", {"class": "datetime"})
#if date:
# claim.set_date(date.get_text())

if (url == "https://africacheck.org/fact-checks/reports/state-emergency-would-not-strip-ramaphosas-powers-and-make-dlamini-zuma"):
print("ws")

# tags
tags = []

for tag in parsed_claim_review_page.findAll('meta', {"property": "article:tag"}):
tags.append(tag["content"])
claim.set_tags(", ".join(tags))

global_claim_text = ""
report_claim_div = parsed_claim_review_page.find("div", {"class": "report-claim"})
if report_claim_div:
claim.set_claim(report_claim_div.find("p").get_text())
else:
claim.set_claim(claim.title)

inline_ratings = parsed_claim_review_page.findAll("div", {"class", "inline-rating"})
entry_section = parsed_claim_review_page.find("section", {"class", "entry-content"}) # type: Tag
#block-mainpagecontent > article > div > div.cell.medium-8.article--main > div > div.clearfix.text-formatted.field.field--name-body.field--type-text-with-summary.field--label-hidden.field__item

# claim
entry_section = parsed_claim_review_page.find("section", {"class", "cell"})
verdict_box = parsed_claim_review_page.find("div", {"class", "article-details__verdict"})

#block-mainpagecontent > article > div > div.cell.medium-8.article--main > div > div

entry_section_full_text = entry_section.text

# There are several claims checked within the page. Common date, author, tags ,etc.
if inline_ratings and len(inline_ratings) > 0:
entry_contents = entry_section.contents # type : List[Tag]
current_index = 0

# First we extract the bit of text common to everything until we meed a sub-section
body_text, links, current_index = get_text_and_links_until_next_header(entry_contents, current_index)
claim.set_body(body_text)
claim.set_refered_links(links)

while current_index < len(entry_contents):
current_index = forward_until_inline_rating(entry_contents, current_index)
inline_rating_div = entry_contents[current_index]
if isinstance(inline_rating_div, NavigableString):
break
claim_text = inline_rating_div.find("p", {"class": "claim-content"}).text
inline_rating = inline_rating_div.find("div", {"class", "indicator"}).find("span").text
previous_current_index = current_index
inline_body_text, inline_links, current_index = get_text_and_links_until_next_header(entry_contents,
current_index)
if previous_current_index == current_index:
current_index += 1
inline_claim = Claim()
inline_claim.set_source("africacheck")
inline_claim.set_claim(claim_text)
inline_claim.set_rating(inline_rating)
inline_claim.set_refered_links(",".join(inline_links))
inline_claim.set_body(inline_body_text)
inline_claim.set_tags(", ".join(tags))
inline_claim.set_date(global_date_str)
inline_claim.set_url(url)
if author:
inline_claim.set_author(author.get_text())
inline_claim.set_title(global_title_text)

local_claims.append(inline_claim)
elif "PROMISE:" in entry_section_full_text and "VERDICT:" in entry_section_full_text:
entry_contents = entry_section.contents # type : List[Tag]
current_index = 0

# First we extract the bit of text common to everything until we meed a sub-section
body_text, links, current_index = get_text_and_links_until_next_header(entry_contents, current_index)
claim.set_body(body_text)
claim.set_refered_links(links)

while current_index < len(entry_contents):
inline_rating_div = entry_contents[current_index]
if isinstance(inline_rating_div, NavigableString):
break
claim_text = entry_contents[current_index + 2].span.text
inline_rating = entry_contents[current_index + 4].span.text
current_index += 5
previous_current_index = current_index
inline_body_text, inline_links, current_index = get_text_and_links_until_next_header(entry_contents,
current_index)
if previous_current_index == current_index:
current_index += 1
inline_claim = Claim()
inline_claim.set_source("africacheck")
inline_claim.set_claim(claim_text)
inline_claim.set_rating(inline_rating)
inline_claim.set_refered_links(",".join(inline_links))
inline_claim.set_body(inline_body_text)
inline_claim.set_tags(", ".join(tags))
inline_claim.set_date(global_date_str)
inline_claim.set_url(url)
if author:
inline_claim.set_author(author.get_text())
inline_claim.set_title(global_title_text)

local_claims.append(inline_claim)
if verdict_box and len(verdict_box) > 0 and "Verdict" in verdict_box.text:
report_claim_div = parsed_claim_review_page.find("div", {"class": "field--name-field-claims"})
if report_claim_div:
claim.set_claim(report_claim_div.get_text())
else:
claim.set_claim(claim.title)

# rating
inline_ratings = parsed_claim_review_page.findAll("div", {"class", "rating"})

if inline_ratings:
if (hasattr( inline_ratings[0], 'class')):
try:
if ('class' in inline_ratings[0].attrs):
if (inline_ratings[0].attrs['class'][1]):
rating_tmp = inline_ratings[0].attrs['class'][1]
claim.rating = rating_tmp.replace('rating--','').replace("-","").capitalize()
except KeyError:
print("KeyError: Skip")
else:
# body
body = parsed_claim_review_page.find("div", {"id": "main"})
claim.set_body(body.get_text())

# related links
divTag = parsed_claim_review_page.find("div", {"id": "main"})
related_links = []
for link in divTag.findAll('a', href=True):
related_links.append(link['href'])
claim.set_refered_links(",".join(related_links))
# alternative rating (If there is no article--aside box with verdict)
global_truth_rating = ""
if parsed_claim_review_page.find("div", {"class": "verdict-stamp"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "verdict-stamp"}).get_text()
if parsed_claim_review_page.find("div", {"class": "verdict"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "verdict"}).get_text()
if parsed_claim_review_page.find("div", {"class": "indicator"}):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "indicator"}).get_text()
if parsed_claim_review_page.find("div", {"class": "indicator"}).find('span'):
global_truth_rating = parsed_claim_review_page.find("div", {"class": "indicator"}).find(
'span').get_text()

#if (url == "https://africacheck.org/fact-checks/fbchecks/beware-details-viral-message-about-discovery-health-and-oxygen-tanks-covid-19"):
# print("ws")

# If still no rathing value, try to extract from picture name
if (global_truth_rating == ""):
filename =""
if parsed_claim_review_page.select( 'div.hero__image > picture' ):
for child in parsed_claim_review_page.select( 'div.hero__image > picture' ):
# child.contents[1].attrs['srcset']
if (hasattr( child, 'contents' )):
try:
filename=child.contents[1].attrs['srcset']
continue
except KeyError:
print("KeyError: Skip")

if (filename != ""):
filename_split = filename.split("/")
filename_split = filename_split[len(filename_split)-1].split(".png")
filename_split = filename_split[0].split("_")
if len(filename_split) == 1:
global_truth_rating = filename_split[0]
else:
global_truth_rating = filename_split[len(filename_split)-1]

#global_truth_rating = filename_split[0]

claim.set_rating(str(re.sub('[^A-Za-z0-9 -]+', '', global_truth_rating)).lower().strip().replace("pfalse","false").replace("-","").capitalize())

if (not self.rating_value_is_valid(claim.rating)):
print ("\nURL: " + url)
print ("\n Rating:" + claim.rating)
claim.rating = ""

###########

# # There are several claims checked within the page. Common date, author, tags ,etc.
# if inline_ratings and len(inline_ratings) > 0:
# entry_contents = entry_section.contents # type : List[Tag]
# current_index = 0

# # First we extract the bit of text common to everything until we meed a sub-section
# body_text, links, current_index = get_text_and_links_until_next_header(entry_contents, current_index)
# claim.set_body(body_text)
# claim.set_refered_links(links)

# while current_index < len(entry_contents):
# current_index = forward_until_inline_rating(entry_contents, current_index)
# inline_rating_div = entry_contents[current_index]
# if isinstance(inline_rating_div, NavigableString):
# break
# claim_text = inline_rating_div.find("p", {"class": "claim-content"}).text
# inline_rating = inline_rating_div.find("div", {"class", "indicator"}).find("span").text
# previous_current_index = current_index
# inline_body_text, inline_links, current_index = get_text_and_links_until_next_header(entry_contents,
# current_index)
# if previous_current_index == current_index:
# current_index += 1
# inline_claim = Claim()
# inline_claim.set_source("africacheck")
# inline_claim.set_claim(claim_text)
# inline_claim.set_rating(inline_rating)
# inline_claim.set_refered_links(",".join(inline_links))
# inline_claim.set_body(inline_body_text)
# inline_claim.set_tags(", ".join(tags))
# inline_claim.set_date(global_date_str)
# inline_claim.set_url(url)
# if author:
# inline_claim.set_author(author.get_text())
# inline_claim.set_title(global_title_text)

# local_claims.append(inline_claim)


local_claims.append(claim)
# body
body = parsed_claim_review_page.find("div", {"class": "article--main"})
claim.set_body(body.get_text())

return local_claims
# related links
related_links = []
for link in body.findAll('a', href=True):
related_links.append(link['href'])
claim.related_links = related_links

# local_claims.append(claim)
# return local_claims
if claim.rating:
return [claim]
else:
return []

def rating_value_is_valid(self, rating_value: str) -> str:
dictionary = {
"Correct": "1",
"Mostlycorrect": "1",
"Unproven": "1",
"Misleading": "1",
"Exaggerated": "1",
"Understated": "1",
"Incorrect": "1",
"Checked": "1",
"True": "1",
"False": "1",
"Partlyfalse": "1",
"Partlytrue": "1",
"Fake": "1",
"Scam": "1",
"Satire": "1"
}

if rating_value in dictionary:
return dictionary[rating_value]
else:
return ""

def get_text_and_links_until_next_header(contents: List[Tag], current_index) -> (Tag, List[str], int):
links = [] # type : List[str]
Expand Down Expand Up @@ -407,3 +415,4 @@ def forward_until_inline_rating(contents: List[Tag], current_index) -> int:
div_rating = current_element.find("div", {"class", "inline-rating"})

return current_index

Loading

0 comments on commit 80c1a23

Please sign in to comment.