Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add backoff, add login with password, and add cookie handling. #26

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Don't version user oauth credentials
# These are generated per user, and will be generated on your system
cookie.txt
storage.json

# Don't version cached or compiled python files
2 changes: 2 additions & 0 deletions Pipfile
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ pylint = "*"
twine = "*"
coverage = "*"
coveralls = "*"
rope = "*"

[packages]
google-api-python-client = "*"
@@ -16,3 +17,4 @@ lxml = "*"
python-dateutil = "*"
pytz = "*"
oauth2client = "*"
backoff = "*"
478 changes: 237 additions & 241 deletions Pipfile.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ So you want to work with the code? Awesome!
This project uses [Pipenv](https://pipenv.readthedocs.io/en/latest/), after cloning the repo, do the following:

1. Make sure you have python 3 installed.
2. Create a pipenv in your working directory with `pipenv --python 3`.
2. Create a pipenv in your working directory with `pipenv --three`.
3. Install both the default and development packages from the Pipfile with `pipenv install --dev`.

You should now be ready to work.
44 changes: 25 additions & 19 deletions lectocal/gcalendar.py
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import backoff
import datetime
from httplib2 import Http
import dateutil.parser
@@ -150,7 +151,7 @@ def _parse_event_to_lesson(event):
if "description" in event:
description = event["description"]
else:
description = None
description = ""
if "source" in event and "url" in event["source"]:
link = event["source"]["url"]
else:
@@ -173,28 +174,22 @@ def get_schedule(google_credentials, calendar_name, n_weeks):
events = _get_events_in_date_range(service, calendar_id, start, end)
return _parse_events_to_schedule(events)


@backoff.on_exception(backoff.expo, HttpError, max_tries=8)
def _delete_lesson(service, calendar_id, lesson_id):
service \
.events() \
.delete(calendarId=calendar_id, eventId=lesson_id) \
.execute()


@backoff.on_exception(backoff.expo, HttpError, max_tries=4)
def _add_lesson(service, calendar_id, lesson):
try:
service \
.events() \
.insert(calendarId=calendar_id, body=lesson.to_gcalendar_format()) \
.execute()
except HttpError as err:
#Status code 409 is conflict. In this case, it means the id already exists.
if err.resp.status == 409:
_update_lesson(service, calendar_id, lesson)
else:
raise err

# try:
service \
.events() \
.insert(calendarId=calendar_id, body=lesson.to_gcalendar_format()) \
.execute()

@backoff.on_exception(backoff.expo, HttpError, max_tries=8)
def _update_lesson(service, calendar_id, lesson):
service \
.events() \
@@ -204,18 +199,29 @@ def _update_lesson(service, calendar_id, lesson):
.execute()


def _add_lesson_or_update_lesson(service, calendar_id, new_lesson):
try:
_add_lesson(service, calendar_id, new_lesson)
except HttpError as err:
#Status code 409 is conflict. In this case, it means the id already exists.
if err.resp.status == 409:
_update_lesson(service, calendar_id, new_lesson)
else:
raise err


def _delete_removed_lessons(service, calendar_id, old_schedule, new_schedule):
for old_lesson in old_schedule:
if not any(new_lesson.id == old_lesson.id
for new_lesson in new_schedule):
_delete_lesson(service, calendar_id, old_lesson.id)
for new_lesson in new_schedule):
_delete_lesson(service, calendar_id, old_lesson.id)


def _add_new_lessons(service, calendar_id, old_schedule, new_schedule):
for new_lesson in new_schedule:
if not any(old_lesson.id == new_lesson.id
for old_lesson in old_schedule):
_add_lesson(service, calendar_id, new_lesson)
for old_lesson in old_schedule):
_add_lesson_or_update_lesson(service, calendar_id, new_lesson)


def _update_current_lessons(service, calendar_id, old_schedule, new_schedule):
94 changes: 67 additions & 27 deletions lectocal/lectio.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
# limitations under the License.

import datetime
import pickle
import re
import requests
from lxml import html
@@ -23,8 +24,8 @@
LESSON_STATUS = {None: "normal", "Ændret!": "changed", "Aflyst!": "cancelled"}


class UserDoesNotExistError(Exception):
""" Attempted to get a non-existing user from Lectio. """
class CannotLoginToLectioError(Exception):
""" Could not login to Lectio (using cookie or login provided). """


class IdNotFoundInLinkError(Exception):
@@ -43,15 +44,53 @@ class InvalidLocationError(Exception):
""" The line doesn't include any location. """


def _get_user_page(school_id, user_type, user_id, week=""):
def _get_user_page(school_id, user_type, user_id, week = "", login = "", password = ""):
URL_TEMPLATE = "https://www.lectio.dk/lectio/{0}/" \
"SkemaNy.aspx?type={1}&{1}id={2}&week={3}"

r = requests.get(URL_TEMPLATE.format(school_id,
USER_TYPE[user_type],
user_id,
week),

LOGIN_URL = "https://www.lectio.dk/lectio/{0}/login.aspx".format(school_id)

# Start requests session
s = requests.Session()

if(login != ""):
# Get eventvalidation key
result = s.get(LOGIN_URL)
tree = html.fromstring(result.text)
authenticity_token = list(set(tree.xpath("//input[@name='__EVENTVALIDATION']/@value")))[0]

# Create payload
payload = {
"m$Content$username2": login,
"m$Content$password2": password,
"m$Content$passwordHidden": password,
"__EVENTVALIDATION": authenticity_token,
"__EVENTTARGET": "m$Content$submitbtn2",
"__EVENTARGUMENT": "",
"LectioPostbackId": ""
}

# Perform login
result = s.post(LOGIN_URL, data = payload, headers = dict(referer = LOGIN_URL))

# Save cookies to file
with open('cookie.txt', 'wb') as f:
pickle.dump(s.cookies, f)

else:
# Load cookies from file
with open('cookie.txt', 'rb') as f:
s.cookies.update(pickle.load(f))

# Scrape url and save cookies to file
r = s.get(URL_TEMPLATE.format(school_id,
USER_TYPE[user_type],
user_id,
week),
allow_redirects=False)
with open('cookie.txt', 'wb') as f:
pickle.dump(s.cookies, f)

return r


@@ -65,10 +104,9 @@ def _get_lectio_weekformat_with_offset(offset):


def _get_id_from_link(link):
match = re.search("(?:absid|ProeveholdId|outboundCensorID)=(\d+)", link)
match = re.search(r"(?:absid|ProeveholdId|outboundCensorID|aftaleid)=(\d+)", link)
if match is None:
raise IdNotFoundInLinkError("Couldn't find id in link: {}".format(
link))
return None
return match.group(1)

def _get_complete_link(link):
@@ -92,8 +130,8 @@ def _is_time_line(line):
# 8/4-2016 17:30 til 9/4-2016 01:00
# 7/12-2015 10:00 til 11:30
# 17/12-2015 10:00 til 11:30
match = re.search("\d{1,2}/\d{1,2}-\d{4} (?:Hele dagen|\d{2}:\d{2} til "
"(?:\d{1,2}/\d{1,2}-\d{4} )?\d{2}:\d{2})", line)
match = re.search(r"\d{1,2}/\d{1,2}-\d{4} (?:Hele dagen|\d{2}:\d{2} til "
r"(?:\d{1,2}/\d{1,2}-\d{4} )?\d{2}:\d{2})", line)
return match is not None


@@ -132,8 +170,8 @@ def _get_time_from_line(line):
# 2 - start time
# 3 - end date
# 4 - end time
match = re.search("(\d{1,2}/\d{1,2}-\d{4})(?: (\d{2}:\d{2}) til "
"(\d{1,2}/\d{1,2}-\d{4})? ?(\d{2}:\d{2}))?", line)
match = re.search(r"(\d{1,2}/\d{1,2}-\d{4})(?: (\d{2}:\d{2}) til "
r"(\d{1,2}/\d{1,2}-\d{4})? ?(\d{2}:\d{2}))?", line)
if match is None:
raise InvalidTimeLineError("No time found in line: '{}'".format(line))

@@ -218,8 +256,8 @@ def _parse_page_to_lessons(page):
return lessons


def _retreive_week_schedule(school_id, user_type, user_id, week):
r = _get_user_page(school_id, user_type, user_id, week)
def _retreive_week_schedule(school_id, user_type, user_id, week, login = "", password = ""):
r = _get_user_page(school_id, user_type, user_id, week, login = "", password = "")
schedule = _parse_page_to_lessons(r.content)
return schedule

@@ -232,27 +270,29 @@ def _filter_for_duplicates(schedule):
return filtered_schedule


def _retreive_user_schedule(school_id, user_type, user_id, n_weeks):
def _retreive_user_schedule(school_id, user_type, user_id, n_weeks, login = "", password = ""):
schedule = []
for week_offset in range(n_weeks + 1):
week = _get_lectio_weekformat_with_offset(week_offset)
week_schedule = _retreive_week_schedule(school_id,
user_type,
user_id,
week)
week,
login = "",
password = "")
schedule += week_schedule
filtered_schedule = _filter_for_duplicates(schedule)
return filtered_schedule


def _user_exists(school_id, user_type, user_id):
r = _get_user_page(school_id, user_type, user_id)
def _can_login(school_id, user_type, user_id, login = "", password = ""):
r = _get_user_page(school_id, user_type, user_id, "", login, password)
return r.status_code == requests.codes.ok


def get_schedule(school_id, user_type, user_id, n_weeks):
if not _user_exists(school_id, user_type, user_id):
raise UserDoesNotExistError("Couldn't find user - school: {}, "
"type: {}, id: {} - in Lectio.".format(
school_id, user_type, user_id))
return _retreive_user_schedule(school_id, user_type, user_id, n_weeks)
def get_schedule(school_id, user_type, user_id, n_weeks, login = "", password = ""):
if not _can_login(school_id, user_type, user_id, login, password):
raise CannotLoginToLectioError(
"Couldn't login user - school: {}, type: {}, id: {}, login: {} "
"- in Lectio.".format(school_id, user_type, user_id, login))
return _retreive_user_schedule(school_id, user_type, user_id, n_weeks, login = "", password = "")
44 changes: 39 additions & 5 deletions lectocal/run.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
# limitations under the License.

import argparse
import getpass
from . import gauth
from . import lectio
from . import lesson
@@ -40,6 +41,14 @@ def _get_arguments():
default="Lectio",
help="Name to use for the calendar inside "
"Google Calendar. (default: Lectio)")
parser.add_argument("--login",
default="",
type=str,
help="The username from a Lectio login.")
parser.add_argument('--keepalive',
default=False,
dest='keepalive',
action='store_true')
parser.add_argument("--weeks",
type=int,
default=4,
@@ -51,21 +60,46 @@ def _get_arguments():

def main():
arguments = _get_arguments()

if arguments.keepalive:
_keepalive(arguments)
exit()

password = _get_password(arguments.login)

google_credentials = gauth.get_credentials(arguments.credentials)
if not gcalendar.has_calendar(google_credentials, arguments.calendar):
gcalendar.create_calendar(google_credentials, arguments.calendar)

lectio_schedule = lectio.get_schedule(arguments.school_id,
arguments.user_type,
arguments.user_id,
arguments.weeks)
arguments.user_type,
arguments.user_id,
arguments.weeks,
arguments.login,
password)
google_schedule = gcalendar.get_schedule(google_credentials,
arguments.calendar,
arguments.weeks)
arguments.calendar,
arguments.weeks)
if not lesson.schedules_are_identical(lectio_schedule, google_schedule):
gcalendar.update_calendar_with_schedule(google_credentials,
arguments.calendar,
google_schedule,
lectio_schedule)


def _keepalive(arguments):
lectio.get_schedule(arguments.school_id,
arguments.user_type,
arguments.user_id,
1)


def _get_password(login):
if(login != ""):
return getpass.getpass()
else:
return ""


if __name__ == "__main__":
main()
4 changes: 3 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -49,7 +49,9 @@ def readme():
"lxml",
"pytz",
"python-dateutil",
"oauth2client"
"oauth2client",
"backoff",
"pickle"
],
package_data={
"lectocal": [