Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

first pass at finding captions within tables #111

Merged
merged 3 commits into from
Jun 13, 2017
Merged

first pass at finding captions within tables #111

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
019fe84
first pass at finding captions within tables
axfelix Jun 5, 2017
43820b9
reset caption_element to None for each table
axfelix Jun 5, 2017
7a91751
fixed errors on tiny tables
axfelix Jun 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions bin/captionclassifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from lxml import etree
import uuid
import re
import editdistance


class CaptionClassifier(Debuggable):
Expand Down Expand Up @@ -321,6 +322,7 @@ def run_tables(self):
separator = ':'

for table in tables:
caption_element = None
use_next = False
use_previous = False
used_title = False
Expand Down Expand Up @@ -467,6 +469,34 @@ def run_tables(self):

self.debug.print_debug(self, u'Moved table and siblings to parent section')

# If none of that worked, try to find caption in table rows
if caption_element is None:
table_rows = table.find("table").getchildren()

# Check if first row has fewer columns than others
# Therefore not likely to be data or a header
columns_count = {}
first_column = {}
row_number = 0

for row in table_rows:
row_number += 1
columns_count[row_number] = len(row.getchildren())
first_column[row_number] = row.getchildren()[0].text
fewest_columns = min(columns_count, key=columns_count.get)

if len(columns_count) > 2 and columns_count[1] == fewest_columns and columns_count[2] != fewest_columns:
# If it has fewest columns, also check Levenshtein distance
# To ensure this row is unlike the others
if editdistance.eval(first_column[1], first_column[2]) > editdistance.eval(first_column[2], first_column[3]):

# OK, we have something, move it
caption_element = etree.Element('caption')
caption_element.text = first_column[1]
NlmManipulate.append_safe(table, caption_element, self)
table.find("table").remove(table_rows[0])


paragraphs = tree.xpath('//p')

self.link(table_ids, table_titles, paragraphs, 'table')
Expand Down