diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..0e97f46 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,43 @@ +# Serves for configuration of CodeClimate (automated code review for test coverage, complexity, duplication, etc.) +engines: + radon: + enabled: true + config: + python_version: 2 + threshold: "C" + exclude_fingerprints: + # It is not needed to fight for cyclomatic complexity decreasing in tools/** scripts + - 986c7c1735f2a2e2a6500e7148ea4bd5 # cyclomatic complexity in course_setup.py + - ec3a2e4e4d53fb59016309bd007df078 # cyclomatic complexity in transform_to_adapt/transform_problems.py + fixme: + enabled: true + config: + strings: + - FIXME + - BUG + - CUSTOM + - TODO + duplication: + enabled: true + config: + languages: + - javascript: + - python: + python_version: 2 + exclude_fingerprints: + # Probabilities POST and PUT methods are marked as duplication + - 4e06d64888f1f615bfcf51416844dca5 + eslint: + enabled: true + markdownlint: + enabled: true + exclude_fingerprints: + # Long line in Code block marked as long, but length checking is not expected in code blocks + - 7e02053fa411ee2f67df597e15d10148 + +ratings: + paths: + - "**.js" + - "**.py" + - "**.md" + diff --git a/README.md b/README.md index 10cc466..c1ea493 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Adaptive problem selection interface for EdX courses -[![Travis](https://img.shields.io/travis/raccoongang/edx-adapt.svg)](https://travis-ci.org/raccoongang/edx-adapt) [![Code Climate](https://img.shields.io/codeclimate/github/raccoongang/edx-adapt.svg)](https://codeclimate.com/github/raccoongang/edx-adapt) +[![Travis](https://img.shields.io/travis/raccoongang/edx-adapt.svg)](https://travis-ci.org/raccoongang/edx-adapt) +[![Code Climate](https://img.shields.io/codeclimate/github/raccoongang/edx-adapt.svg)](https://codeclimate.com/github/raccoongang/edx-adapt) ## Installation @@ -29,13 +30,12 @@ Activate virtual environment and install all requirements: > pip install -r requirements ``` -## Run edx-adapt application in development mode: +## Run edx-adapt application in development mode ``` > python edx_adapt.py ``` - ## Run edx-adapt application in production mode There are few sample config files in `etc/folder` required to configure @@ -54,135 +54,120 @@ service manager `/api/v1/course` - GET: Show all courses registered in Edx-Adapt - - `response.data = {'course_ids': courses}` + - `response.data = {'course_ids': courses}` - POST: Create new course in Edx-Adapt - - Parameters: `course_id` (string) - + - Parameters: `course_id` (string) `/api/v1/course//skill` - GET: Show all skills which course `` contains. - - `response.data = {'skills': skills}` + - `response.data = {'skills': skills}` - POST: Add skill into course `` - - Parameters : `skill_name` (string) - + - Parameters : `skill_name` (string) `/api/v1/course//user` - GET: Show all users registered in the course `` - - `response.data = {'users': {'finished': finished_users, 'in_progress': -progress_users}}` + - `response.data = {'users': {'finished': finished_users, + 'in_progress': progress_users}}` - POST: Enroll new user in the course `` - - Parameters: `user_id` (string) - + - Parameters: `user_id` (string) `/api/v1/course/` - GET: Show all problems contained to the course `` - - `response.data = {'problems': problems}` + - `response.data = {'problems': problems}` - POST: Add new problem in the course `` - - Parameters: `problem_name` (string), `tutor_url` (string), `skills` -(list of strings), `pretest` (boolean), `posttest` (boolean) - + - Parameters: `problem_name` (string), `tutor_url` (string), `skills` + (list of strings), `pretest` (boolean), `posttest` (boolean) `/api/v1/course//skill/` - GET: Show all problems contained in the course `` related -to the skill `` - - `response.data = {'problems': problems}` - + to the skill `` + - `response.data = {'problems': problems}` `/api/v1/course//experiment` - GET: Show result of all experiments for the course `` - - `response.data = {'experiments': exps}` - + - `response.data = {'experiments': exps}` `/api/v1/course//probabilities` - GET: Show list with default BKT model parameters added to the course -`` - - `response.data = {'model_params': prob_list}` + `` + - `response.data = {'model_params': prob_list}` - POST: Add new or update existing model parameters in the course -`` - - Parameters: `prob_list` (list of dicts with models parameters) -`[{threshold: float, pg: float, ps: float, pi: float, pt: float}, ...]` - + `` + - Parameters: `prob_list` (list of dicts with models parameters) + `[{threshold: float, pg: float, ps: float, pi: float, pt: float}, + ...]` `/api/v1/course//user//interaction` - POST: Add user's interaction with Edx into Edx-Adapt - - Parameters: `problem` (string), `correct` (int), `attempt` (int), -`unix_seconds` (string) - + - Parameters: `problem` (string), `correct` (int), `attempt` (int), + `unix_seconds` (string) `/api/v1/course//user/` - GET: Show user's status: - - `reponse.data = { + - `reponse.data = { "next": next_problem, "current": current_problem, "done_with_current": done_with_current, "okay": okay, "done_with_course": done_with_course -}` - + }` `/api/v1/course//user//pageload` - POST: Add logging information about problem visited by user into -Edx-Adapt - - Parameters: `problem` (string), `unix_seconds` (string) - + Edx-Adapt + - Parameters: `problem` (string), `unix_seconds` (string) `/api/v1/parameters/bulk` - POST: Add BKT model parameters to each course's skill for the user - - Parameters: `course_id` (string), `user_id` (string), `skills_list` -(list of strings) - + - Parameters: `course_id` (string), `user_id` (string), `skills_list` + (list of strings) `/api/v1/data/logs/course//user//problem/` - GET: Show collected log data for users `` interaction with -problem `` on the course `` - - `response.data = {'log': problem log}` - + problem `` on the course `` + - `response.data = {'log': problem log}` `/api/v1/data/logs/course//user/` - GET: Show all collected log data for user `` interaction on -the course `` - - `response.data = {'log': log data}` - + the course `` + - `response.data = {'log': log data}` `/api/v1/data/logs/course/` - GET: Show all collected data for every user with status "in progress" -on the course `` - - `response.data = {'log': log data}` - + on the course `` + - `response.data = {'log': log data}` `/api/v1/data/logs/course//experiment/` - GET: Show all collected log data for users successfully passed -experiment `` on the course `` - - `response.data = {'log': log data}` - + experiment `` on the course `` + - `response.data = {'log': log data}` `/api/v1/data/trajectory/course//user/` - GET: Show trajectory of the user `` interaction with the -course `` - - `response.data = {'data': {}, 'trajectories': {}, 'pretest_length': + course `` + - `response.data = {'data': {}, 'trajectories': {}, 'pretest_length': {}, 'posttest_length': {}}` - `/api/v1/data/trajectory/course/` -- GET: Show trajectory for all users with status "in progress" in the -course `` - - `response.data = {: {'data': {}, 'trajectories': {}, - 'pretest_length': {}, 'posttest_length': {}}, : {}, ...}` +- GET: Show trajectory for all users with status "in progress" in the + course `` + - `response.data = {: {'data': {}, 'trajectories': {}, + 'pretest_length': {}, 'posttest_length': {}}, : {}, ...}` ## Data models stored in the database @@ -195,7 +180,7 @@ specifically for certain course. ### Generic collections are: `Courses` and `Generic` -#### `Courses` collection stores documents with base information about the course: +#### `Courses` collection stores documents with base information about the course ```js { @@ -234,13 +219,13 @@ specifically for certain course. } ``` -#### `Generic` collection stores all student - skills related data: - +#### `Generic` collection stores all student - skills related data ```js { _id: ObjectID(), - key: , // unique compound key, which contains , , and + key: , // unique compound key, which contains , + // , and val: { threshold: , pg: , @@ -258,7 +243,7 @@ specifically for certain course. Adaptive logging includes page load data and student's responses on problems questions. -##### Interaction log doc: +##### Interaction log doc ```js { @@ -278,7 +263,7 @@ problems questions. } ``` -##### Page load log doc: +##### Page load log doc ```js { @@ -313,10 +298,13 @@ student on every step through the adaptive course }, next: null, // Possible values: dict with problem description, null. student_id: , - perm: false, // (experimental attribute) boolean flag shows if student has permissions for fluent navigation. - // Permission is changed after student start to answer on post-assessment problems - nav: "adapt" // Save initial student permission. Possible value "adapt" and "free", "adapt" is default behavior - // (perm == false), "free" (perm == true) student had permissions for free navigation through - // adaptive problems + perm: false, // (experimental attribute) boolean flag shows if student + // has permissions for fluent navigation. + // Permission is changed after student start to answer on + // post-assessment problems + nav: "adapt" // Save initial student permission. Possible value "adapt" + // and "free", "adapt" is default behavior + // (perm == false), "free" (perm == true) student had + // permissions for free navigation throug adaptive problems } ``` diff --git a/data/BKT/problems.csv b/data/BKT/problems.csv index aea83cd..37f4646 100644 --- a/data/BKT/problems.csv +++ b/data/BKT/problems.csv @@ -61,7 +61,6 @@ Pre_assessment_8,center Pre_assessment_9,spread shape_0,shape shape_1,shape -skew_easy_0,None skew_easy_1,shape skew_easy_2,shape skew_easy_3,shape @@ -81,4 +80,3 @@ t4_2_1,histogram t5_center,center t5_shape,shape t5_spread,spread -labels_we,None diff --git a/edx_adapt/api/resources/base_resource.py b/edx_adapt/api/resources/base_resource.py new file mode 100644 index 0000000..028233d --- /dev/null +++ b/edx_adapt/api/resources/base_resource.py @@ -0,0 +1,10 @@ +""" +General Resources for all API resources modules +""" +from flask_restful import Resource + + +class BaseResource(Resource): + def __init__(self, **kwargs): + self.repo = kwargs['data'] # repo: DataInterface + self.selector = kwargs['selector'] # selector: SelectInterface diff --git a/edx_adapt/api/resources/course_resources.py b/edx_adapt/api/resources/course_resources.py index 21a735f..c85019c 100644 --- a/edx_adapt/api/resources/course_resources.py +++ b/edx_adapt/api/resources/course_resources.py @@ -2,8 +2,9 @@ For example, CRUDding courses, users, problems, skills... """ -from flask_restful import Resource, abort, reqparse +from flask_restful import abort, reqparse +from edx_adapt.api.resources.base_resource import BaseResource from edx_adapt.data.interface import DataException from edx_adapt import logger from edx_adapt.select.interface import SelectException @@ -12,78 +13,56 @@ course_parser.add_argument('course_id', type=str, required=True, location='json', help="Please supply a course ID") -class Courses(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - """@type repo: DataInterface""" - - def get(self): - courses = [] +class DefaultResource(BaseResource): + def _post_request(self, function_name, *args, **kwargs): try: - courses = self.repo.get_course_ids() + getattr(self.repo, function_name)(*args, **kwargs) except DataException as e: + logger.exception('DataException: ') abort(500, message=str(e)) - return {'course_ids': courses}, 200 + return {'success': True}, 201 - def post(self): - args = course_parser.parse_args() + def _get_request(self, function_name, *args): + output_list = [] try: - self.repo.post_course(args['course_id']) + output_list = getattr(self.repo, function_name)(*args) except DataException as e: - abort(500, message=str(e)) + logger.exception('DataException:') + abort(404, message=str(e)) + return output_list - return {'success': True}, 200 +class Courses(DefaultResource): + def get(self): + courses = self._get_request('get_course_ids') + return {'course_ids': courses}, 200 + + def post(self): + args = course_parser.parse_args() + return self._post_request('post_course', args['course_id']) skill_parser = reqparse.RequestParser() skill_parser.add_argument('skill_name', type=str, required=True, location='json', help="Please supply the name of a skill") -class Skills(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - """@type repo: DataInterface""" - +class Skills(DefaultResource): def get(self, course_id): - skills = [] - try: - skills = self.repo.get_skills(course_id) - except DataException as e: - abort(404, message=str(e)) - + skills = self._get_request('get_skills', course_id) return {'skills': skills}, 200 def post(self, course_id): args = skill_parser.parse_args() - try: - self.repo.post_skill(course_id, args['skill_name']) - except DataException as e: - abort(500, message=str(e)) - - return {'success': True}, 200 - + return self._post_request('post_skill', course_id, args['skill_name']) user_parser = reqparse.RequestParser() user_parser.add_argument('user_id', type=str, required=True, location='json', help="Please supply a user ID") -class Users(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - self.selector = kwargs['selector'] - """@type repo: DataInterface""" - """@type selector: SelectInterface""" - +class Users(DefaultResource): def get(self, course_id): - finished_users = [] - progress_users = [] - try: - finished_users = self.repo.get_finished_users(course_id) - progress_users = self.repo.get_in_progress_users(course_id) - except DataException as e: - abort(404, message=str(e)) - + finished_users = self._get_request('get_finished_users', course_id) + progress_users = self._get_request('get_in_progress_users', course_id) return {'users': {'finished': finished_users, 'in_progress': progress_users}}, 200 def post(self, course_id): @@ -120,35 +99,23 @@ def post(self, course_id): help="Set True if this is a posttest problem. Mutually exclusive with pretest") -class Problems(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - """@type repo: DataInterface""" - +class Problems(DefaultResource): def get(self, course_id, skill_name=None): - problems = [] - try: - problems = self.repo.get_problems(course_id, skill_name) - except DataException as e: - abort(500, message=str(e)) - + problems = self._get_request('get_problems', course_id, skill_name) return {'problems': problems}, 200 def post(self, course_id): args = problem_parser.parse_args() logger.debug("Post problem args: {}".format(args)) - try: - if args['pretest']: - self.repo.post_pretest_problem(course_id, args['skills'], args['problem_name'], args['tutor_url']) - elif args['posttest']: - self.repo.post_posttest_problem(course_id, args['skills'], args['problem_name'], args['tutor_url']) - else: - self.repo.post_problem(course_id, args['skills'], args['problem_name'], args['tutor_url']) - except DataException as e: - abort(500, message=str(e)) - - return {'success': True}, 200 - + return self._post_request( + 'post_problem', + course_id, + args['skills'], + args['problem_name'], + args['tutor_url'], + args['pretest'], + args['posttest'] + ) experiment_parser = reqparse.RequestParser() experiment_parser.add_argument('experiment_name', type=str, location='json', required=True, @@ -159,57 +126,30 @@ def post(self, course_id): help="Please supply the end date in unix seconds") -class Experiments(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - """@type repo: DataInterface""" - +class Experiments(DefaultResource): def get(self, course_id): - exps = [] - try: - exps = self.repo.get_experiments(course_id) - except DataException as e: - logger.exception("Data exception:") - abort(500, message=str(e)) + exps = self._get_request('get_experiments', course_id) return {'experiments': exps} def post(self, course_id): args = experiment_parser.parse_args() - try: - self.repo.post_experiment(course_id, args['experiment_name'], args['start_time'], args['end_time']) - except DataException as e: - logger.exception("Data exception:") - abort(500, message=str(e)) + return self._post_request( + 'post_experiment', course_id, args['experiment_name'], args['start_time'], args['end_time'] + ) prob_parser = reqparse.RequestParser() prob_parser.add_argument('prob_list', type=list, location='json', help="Please supply list with default model_params") -class Probabilities(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - +class Probabilities(DefaultResource): def get(self, course_id): - prob_list = [] - try: - prob_list = self.repo.get_model_params(course_id) - except DataException as e: - abort(404, message=str(e)) - + prob_list = self._get_request('get_model_params', course_id) return {'model_params': prob_list}, 200 def post(self, course_id): args = prob_parser.parse_args() - try: - self.repo.post_model_params(course_id, args['prob_list'], new=True) - except DataException as e: - abort(500, message=str(e)) - return {'success': True}, 201 + return self._post_request('post_model_params', course_id, args['prob_list'], new=True) def put(self, course_id): args = prob_parser.parse_args() - try: - self.repo.post_model_params(course_id, args['prob_list']) - except DataException as e: - abort(500, message=str(e)) - return {'success': True}, 201 + return self._post_request('post_model_params', course_id, args['prob_list']) diff --git a/edx_adapt/api/resources/data_serve_resources.py b/edx_adapt/api/resources/data_serve_resources.py index ed59d7d..ad77b08 100644 --- a/edx_adapt/api/resources/data_serve_resources.py +++ b/edx_adapt/api/resources/data_serve_resources.py @@ -1,21 +1,18 @@ -""" This file contains api resources for serving data from the course. +""" +This file contains api resources for serving data from the course. """ -from flask_restful import Resource, abort +from flask_restful import abort +from edx_adapt.api.resources.base_resource import BaseResource from edx_adapt.data.interface import DataException from edx_adapt import logger -""" First, all requests for logs """ - -class SingleProblemRequest(Resource): +class SingleProblemRequest(BaseResource): """ Handle request for a user's log on one problem """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] def get(self, course_id, user_id, problem_name): problog = [] @@ -28,14 +25,10 @@ def get(self, course_id, user_id, problem_name): return {'log': problog} -class UserLogRequest(Resource): +class UserLogRequest(BaseResource): """ Handle request for a user's log """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id, user_id): log = [] try: @@ -46,14 +39,10 @@ def get(self, course_id, user_id): return {'log': log} -class CourseLogRequest(Resource): +class CourseLogRequest(BaseResource): """ Handle request for logs from all users of a course """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id): data = {} try: @@ -69,14 +58,10 @@ def get(self, course_id): return {'log': data} -class ExperimentLogRequest(Resource): +class ExperimentLogRequest(BaseResource): """ Handle request for logs from all users from an experiment (only gives logs for finished users) """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id, experiment_name): data = {} try: @@ -90,7 +75,27 @@ def get(self, course_id, experiment_name): abort(500, message=str(e)) return {'log': data} -""" Now requests for trajectories """ + +def _fulfill_correct(repo, course_id, user_id): + correct = {} + correct['pretest'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) + if x['problem']['pretest']] + correct['posttest'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) + if x['problem']['posttest']] + correct['problems'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) + if not (x['problem']['posttest'] and x['problem']['pretest'])] + return correct + + +def _fulfill_skills(repo, course_id, user_id): + skills = {} + skills['pretest'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) + if x['problem']['pretest']] + skills['posttest'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) + if x['problem']['posttest']] + skills['problems'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) + if x['problem']['posttest'] is False and x['problem']['pretest'] is False] + return skills # helper function @@ -99,17 +104,11 @@ def fill_user_data(repo, course_id, user_id): correct = {} num_pre = {} num_post = {} - skillz = {} data['all'] = repo.get_all_interactions(course_id, user_id) correct['all'] = repo.get_whole_trajectory(course_id, user_id) - correct['pretest'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['pretest']] - correct['posttest'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['posttest']] - correct['problems'] = [x['correct'] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['posttest'] is False and x['problem']['pretest'] is False] + correct.update(_fulfill_correct(repo, course_id, user_id)) data['by_skill'] = {} correct['by_skill'] = {} @@ -120,45 +119,37 @@ def fill_user_data(repo, course_id, user_id): num_pre[skill] = repo.get_num_pretest(course_id, skill) num_post[skill] = repo.get_num_posttest(course_id, skill) - skillz['pretest'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['pretest']] - skillz['posttest'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['posttest']] - skillz['problems'] = [x['problem']['skills'][0] for x in repo.get_all_interactions(course_id, user_id) - if x['problem']['posttest'] is False and x['problem']['pretest'] is False] + skills = _fulfill_skills(repo, course_id, user_id) - blob = {'data': data, 'trajectories': correct, 'trajectory_skills': skillz, 'pretest_length': num_pre, - 'posttest_length': num_post} + blob = { + 'data': data, + 'trajectories': correct, + 'trajectory_skills': skills, + 'pretest_length': num_pre, + 'posttest_length': num_post + } return blob -class UserTrajectoryRequest(Resource): +class UserTrajectoryRequest(BaseResource): """ Handle request for a user's trajectories """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id, user_id): - blob = {'data':{}, 'trajectories':{}, 'pretest_length': {}, 'posttest_length': {}} + blob = {'data': {}, 'trajectories': {}, 'pretest_length': {}, 'posttest_length': {}} try: blob = fill_user_data(self.repo, course_id, user_id) except DataException as e: - logger.error("Data exception: {}".format(e)) + logger.exception("Data exception:") abort(500, message=str(e)) return blob -class CourseTrajectoryRequest(Resource): +class CourseTrajectoryRequest(BaseResource): """ Handle request for logs from all users of a course """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id): userblobs = {} try: @@ -175,14 +166,10 @@ def get(self, course_id): return userblobs -class ExperimentTrajectoryRequest(Resource): +class ExperimentTrajectoryRequest(BaseResource): """ Handle request for logs from all users from an experiment (only gives logs for finished users) """ - def __init__(self, **kwargs): - """@type repo: DataInterface""" - self.repo = kwargs['data'] - def get(self, course_id, experiment_name): userblobs = {} try: @@ -196,5 +183,3 @@ def get(self, course_id, experiment_name): logger.error("Data exception: {}".format(e)) abort(500, message=str(e)) return userblobs - -#TODO: serve by date maybe...? diff --git a/edx_adapt/api/resources/model_resources.py b/edx_adapt/api/resources/model_resources.py index 6e1b74f..7082615 100644 --- a/edx_adapt/api/resources/model_resources.py +++ b/edx_adapt/api/resources/model_resources.py @@ -2,8 +2,9 @@ """ import random -from flask_restful import Resource, abort, reqparse +from flask_restful import abort, reqparse +from edx_adapt.api.resources.base_resource import BaseResource from edx_adapt.data.interface import DataException from edx_adapt import logger from edx_adapt.select.interface import SelectException @@ -12,19 +13,16 @@ param_parser.add_argument('course_id', type=str, location='json', help="Optionally supply a course id") param_parser.add_argument('user_id', type=str, location='json', help="Optionally supply a user ID") param_parser.add_argument('skill_name', type=str, location='json', help="Optionally supply the name of a skill") -param_parser.add_argument('params', type=dict, location='json', required=True, - help="Please supply the desired model parameters as a dictionary") - +param_parser.add_argument( + 'params', + type=dict, + location='json', + required=True, + help="Please supply the desired model parameters as a dictionary", +) -class Parameters(Resource): - def __init__(self, **kwargs): - """ - @type repo: DataInterface - @type selector: SelectInterface - """ - self.repo = kwargs['data'] - self.selector = kwargs['selector'] +class Parameters(BaseResource): def get(self): param_list = [] try: @@ -48,7 +46,7 @@ def post(self): except SelectException as e: abort(500, message=str(e)) - return {'success': True}, 200 + return {'success': True}, 201 param_bulk_parser = reqparse.RequestParser() param_bulk_parser.add_argument('course_id', type=str, location='json', help="Optionally supply a course id") @@ -61,22 +59,11 @@ def post(self): type=dict, location='json', required=True, - help="Please supply the desired model parameters as a dictionary") - - -class ParametersBulk(Resource): - def __init__(self, **kwargs): - self.repo = kwargs['data'] - self.selector = kwargs['selector'] + help="Please supply the desired model parameters as a dictionary", +) - def get(self): - param_list = [] - try: - param_list = self.selector.get_all_parameters() - except SelectException as e: - abort(500, message=str(e)) - return {'parameters': param_list}, 200 +class ParametersBulk(Parameters): def post(self): args = param_parser.parse_args() course = args['course_id'] diff --git a/edx_adapt/api/resources/tutor_resources.py b/edx_adapt/api/resources/tutor_resources.py index 2b45617..4e66f0b 100644 --- a/edx_adapt/api/resources/tutor_resources.py +++ b/edx_adapt/api/resources/tutor_resources.py @@ -1,101 +1,92 @@ -""" This file contains api resources for interacting with the tutor -as users take the course. """ - -import threading +This file contains api resources for interacting with the tutor as users take the course. +""" import time -from flask_restful import Resource, abort, reqparse +from flask_restful import abort, reqparse +from edx_adapt.api.resources.base_resource import BaseResource from edx_adapt.data.interface import DataException from edx_adapt import logger from edx_adapt.select.interface import SelectException -class UserProblems(Resource): +class DefaultResource(BaseResource): + def run_selector(self, course_id, user_id): + """ + Run the problem selection sequence + """ + nex = self.repo.get_next_problem(course_id, user_id) + + # only run if no next problem has been selected yet, or there was an error previously + if nex is None or 'error' in nex: + logger.info("SELECTOR CHOOSING NEXT PROBLEM") + prob = self.selector.choose_next_problem(course_id, user_id) + logger.info("FINISHED CHOOSING NEXT PROBLEM: {}".format(str(prob))) + self.repo.set_next_problem(course_id, user_id, prob) + else: + logger.info("SELECTION NOT REQUIRED!") + + def _rotate_problem(self, next_problem, course_id, user_id, **args): + if next_problem and 'error' not in next_problem and args['problem'] == next_problem.get('problem_name'): + self.repo.advance_problem(course_id, user_id) + + +class UserProblems(DefaultResource): """ Handle request for user's current and next problem. """ - # Name of the question to test whether the user is paying attention on pretest. - ATTENTION_QUESTION_NAME = 'Pre_assessment_13' - - def __init__(self, **kwargs): - """ - @type repo: DataInterface - """ - self.repo = kwargs['data'] + def _check_current_done(self, course_id, user_id, current): + log = self.repo.get_raw_user_data(course_id, user_id) + done_with_current = any( + [x for x in log if x['type'] == 'response' and + x['problem']['problem_name'] == current['problem_name'] and x['correct'] == 1] + ) + + # account for test questions: user is "done" after they input any answer + if not done_with_current and (current["pretest"] or current["posttest"]): + done_with_current = any([x for x in log if x['type'] == 'response']) + return done_with_current + + def _check_course_done(self, course_id, user_id): + fin = self.repo.get_finished_users(course_id) + done_with_course = user_id in fin + if not done_with_course: + answers = self.repo.get_all_interactions(course_id, user_id) + done_with_course = ( + # Course set to be done if student answer correctly on more than a half of pre-assessment problems + sum([x['correct'] for x in answers if (x['problem']['pretest'])]) > ( + self.repo.get_num_pretest(course_id) // 2 + ) + ) + return done_with_course def get(self, course_id, user_id): - nex = {} - cur = {} try: nex = self.repo.get_next_problem(course_id, user_id) cur = self.repo.get_current_problem(course_id, user_id) except DataException as e: abort(404, message=e.message) - okay = True - if not nex or 'error' in nex: - okay = False + okay = bool(nex and 'error' not in nex) - done_with_current = False done_with_course = False - if not cur: done_with_current = True else: try: - log = self.repo.get_raw_user_data(course_id, user_id) - current_correct = [x for x in log if x['type'] == 'response' and - x['problem']['problem_name'] == cur['problem_name'] and x['correct'] == 1] - done_with_current = (len(current_correct) > 0) - - # account for test questions: user is "done" after they input any answer - if cur["pretest"] or cur["posttest"]: - if len([x for x in log if x['type'] == 'response']) > 0: - done_with_current = True - - fin = self.repo.get_finished_users(course_id) - if user_id in fin: - done_with_course = True - # reject high pretest scores - # if more than 7 out of the first 13 pretest questions are correct - - # then the user knows too much already (ATTENTION_QUESTION_NAME - # is only for checking that the user is paying attention, - # not knowledge assessment, so skip it) - # FIXME(idegtirov) threshold for pre_assessment problems should be changed from int to percentage - answers = self.repo.get_all_interactions(course_id, user_id) - if sum([ - x['correct'] for x in answers if ( - x['problem']['pretest'] and x['problem']['problem_name'] != UserProblems.ATTENTION_QUESTION_NAME - ) - ]) > 7: - done_with_course = True - nex = None - # if answer to ATTENTION_QUESTION_NAME is wrong - - # then filter out this user, because they are not paying - # attention and simply clicking buttons - pretest_done = len(self.repo.get_all_remaining_pretest_problems(course_id, user_id)) == 0 - # Disable cut off of students who gave no correct answers at pre-assessment for easier debugging - # if pretest_done and (sum( [x['correct'] for x in answers if (x['problem']['pretest'] and x['problem']['problem_name'] == UserProblems.ATTENTION_QUESTION_NAME) ] ) < 1): - # if pretest_done and ( - # sum([x['correct'] for x in answers if ( - # x['problem']['pretest'] and x['problem']['problem_name'] == UserProblems.ATTENTION_QUESTION_NAME - # )]) < 0 - # ): - # done_with_course = True - # nex = None + done_with_current = self._check_current_done(course_id, user_id, cur) + done_with_course = self._check_course_done(course_id, user_id) except DataException as e: - logger.exception("DATA EXCEPTION:") + logger.exception("DATA EXCEPTION: ") abort(500, message=str(e)) - return { "next": nex, "current": cur, "done_with_current": done_with_current, "okay": okay, "done_with_course": done_with_course } -""" Argument parser for posting a user response """ +# Argument parser for posting a user response result_parser = reqparse.RequestParser() result_parser.add_argument('problem', type=str, required=True, location='json', help="Must supply the name of the problem which the user answered") @@ -105,104 +96,33 @@ def get(self, course_id, user_id): help="Must supply the attempt number, starting from 1 for the first attempt") result_parser.add_argument('unix_seconds', type=int, location='json', help="Optionally supply timestamp in seconds since unix epoch") -"""result_parser.add_argument('done', type=bool, required=True, - help="Legacy support for multiple part problems. Supply false if" - " the user must answer more parts, otherwise leave this true")""" -""" Global lock for calling choose_next_problem """ -selector_lock = threading.Lock() - -def run_selector(course_id, user_id, selector, repo): - """ - Run the problem selection sequence (in separate thread) - - :param selector: SelectInterface - :param repo: DataInterface - """ - with selector_lock: - logger.info("SELECTOR LOCK ACQUIRED!") - nex = None - try: - nex = repo.get_next_problem(course_id, user_id) - except DataException as e: - # exception here probably means the user/course combo doesn't exist. Screw it, quit - return - - # only run if no next problem has been selected yet, or there was an error previously - if nex is None or 'error' in nex: - try: - logger.info("SELECTOR CHOOSING NEXT PROBLEM") - prob = selector.choose_next_problem(course_id, user_id) - logger.info("FINISHED CHOOSING NEXT PROBLEM: {}".format(str(prob))) - repo.set_next_problem(course_id, user_id, prob) - except SelectException as e: - # assume that the user/course exists. Set an error... - logger.exception("SELECTION EXCEPTION OCCURED:") - repo.set_next_problem(course_id, user_id, {'error': str(e)}) - - except DataException as e: - logger.exception("DATA EXCEPTION HAPPENED, OH NO!") - # TODO: after deciding if set_next_problem could throw an exception here - - else: - logger.info("SELECTION NOT REQUIRED!") - - -class UserInteraction(Resource): +class UserInteraction(DefaultResource): """ Post a user's response to their current problem. - - :param repo: DataInterface - :param selector: SelectInterface """ - def __init__(self, **kwargs): - self.repo = kwargs['data'] - self.selector = kwargs['selector'] - def post(self, course_id, user_id): args = result_parser.parse_args() - if args['unix_seconds'] is None: - args['unix_seconds'] = int(time.time()) + timestamp = args['unix_seconds'] or int(time.time()) try: # If this is a response to the "next" problem, advance to it first before storing # (shouldn't happen if PageLoad messages are posted correctly, but we won't require that) nex = self.repo.get_next_problem(course_id, user_id) - if nex and 'error' not in nex and args['problem'] == nex.get('problem_name'): - self.repo.advance_problem(course_id, user_id) - - #TODO: guard against answering other problems...? - #possibly outside the scope of this software - - self.repo.post_interaction(course_id, args['problem'], user_id, args['correct'], - args['attempt'], args['unix_seconds']) - - #is the user now done? if so hack in a call to psiturk+bo module TODO: do this only once - """ - if user_id in self.repo.get_finished_users(course_id): - print "USER IS DONE! ONTO BAYESIAN OPTIMIZATION!" - psiturk_with_bo.set_next_users_parameters(self.repo, self.selector, course_id) - """ + self._rotate_problem(nex, course_id, user_id, **args) + self.repo.post_interaction(course_id, args['problem'], user_id, args['correct'], args['attempt'], timestamp) # the user needs a new problem, start choosing one - try: - logger.info("STARTING SELECTOR!") - """t = threading.Thread(target=run_selector, args=(course_id, user_id, self.selector, self.repo)) - t.start() - t.join() - #TODO: actually run in other thread """ - run_selector(course_id, user_id, self.selector, self.repo) - except Exception as e: - logger.exception("EXCEPTION STARTING SELECTION THREAD:") - abort(500, message="Interaction successfully stored, but an error occurred starting " - "a problem selection thread: " + e.message) - + self.run_selector(course_id, user_id) + except SelectException as e: + abort(500, message="Interaction successfully stored, but an error occurred starting " + "a problem selection: " + e.message) except DataException as e: logger.exception("DATA EXCEPTION:") abort(500, message=e.message) - return {"success": True}, 200 + return {"success": True}, 201 load_parser = reqparse.RequestParser() load_parser.add_argument('problem', required=True, help="Must supply the name of the problem loaded", location='json') @@ -210,27 +130,20 @@ def post(self, course_id, user_id): location='json') -class UserPageLoad(Resource): +class UserPageLoad(DefaultResource): """ Post the time when a user loads a problem. Used to log time spent solving a problem. - - :param repo: DataInterface """ - def __init__(self, **kwargs): - self.repo = kwargs['data'] - def post(self, course_id, user_id): args = load_parser.parse_args() - if args['unix_seconds'] is None: - args['unix_seconds'] = int(time.time()) + timestamp = args['unix_seconds'] or int(time.time()) try: - self.repo.post_load(course_id, args['problem'], user_id, args['unix_seconds']) + self.repo.post_load(course_id, args['problem'], user_id, timestamp) nex = self.repo.get_next_problem(course_id, user_id) - if nex and 'error' not in nex and args['problem'] == nex.get('problem_name'): - self.repo.advance_problem(course_id, user_id) + self._rotate_problem(nex, course_id, user_id, **args) except DataException as e: logger.exception("DATA EXCEPTION:") abort(500, message=e.message) - return {"success": True}, 200 + return {"success": True}, 201 diff --git a/edx_adapt/data/course_repository.py b/edx_adapt/data/course_repository.py index 5f80ce7..284905f 100644 --- a/edx_adapt/data/course_repository.py +++ b/edx_adapt/data/course_repository.py @@ -1,5 +1,3 @@ -"""Repository that implements DataInterface using a tinydb backend """ - from datetime import datetime import interface @@ -82,14 +80,8 @@ def _add_problem(self, course_id, skill_names, problem_name, tutor_url, b_pretes } ) - def post_problem(self, course_id, skill_names, problem_name, tutor_url): - self._add_problem(course_id, skill_names, problem_name, tutor_url, False, False) - - def post_pretest_problem(self, course_id, skill_names, problem_name, tutor_url): - self._add_problem(course_id, skill_names, problem_name, tutor_url, True, False) - - def post_posttest_problem(self, course_id, skill_names, problem_name, tutor_url): - self._add_problem(course_id, skill_names, problem_name, tutor_url, False, True) + def post_problem(self, course_id, skill_names, problem_name, tutor_url, pretest=False, posttest=False): + self._add_problem(course_id, skill_names, problem_name, tutor_url, pretest, posttest) def enroll_user(self, course_id, user_id): coll = course_id + COLL_SUFFIX['user_problem'] @@ -133,26 +125,33 @@ def get_skills(self, course_id): def get_course_ids(self): return self.store.get_tables() - def get_problems(self, course_id, skill_name=None): + @staticmethod + def _compose_search_dict(**kwargs): + return {key: val for key, val in kwargs.iteritems() if val is not None} + + def get_problems(self, course_id, skill_name=None, pretest=None, posttest=None): """ Get all problems related to this course-skill pre-test, normal, and post-test :param course_id: string ID of the course :param skill_name: string (optional) name of the skill + :param pretest: boolean (optional) flag to return pretest or not pretest problems + :param posttest: boolean (optional) flag to return posttest or not posttest problems :return: list of problems """ - problems = self.store.course_get(course_id, 'problems') - if skill_name: - return [problem for problem in problems if skill_name in problem['skills']] - return problems + skills = [skill_name] if skill_name else None + search_dict = self._compose_search_dict(skills=skills, pretest=pretest, posttest=posttest) + if search_dict: + problems_doc = self.store.course_problems_search(course_id, search_dict) + return problems_doc if problems_doc else [] + else: + return self.store.course_get(course_id, 'problems') - def get_num_pretest(self, course_id, skill_name): - pretest = [x for x in self.get_problems(course_id, skill_name) if x['pretest'] is True] - return len(pretest) + def get_num_pretest(self, course_id, skill_name=None): + return len(self.get_problems(course_id, skill_name, pretest=True)) - def get_num_posttest(self, course_id, skill_name): - posttest = [x for x in self.get_problems(course_id, skill_name) if x['posttest'] is True] - return len(posttest) + def get_num_posttest(self, course_id, skill_name=None): + return len(self.get_problems(course_id, skill_name, posttest=True)) def get_in_progress_users(self, course_id): return self.store.course_get(course_id, 'users_in_progress') @@ -167,7 +166,7 @@ def _get_problem(self, course_id, problem_name): if problems: return problems.get('problems')[0] else: - raise interface.DataException("Problem not found: {}".format(problem_name)) + raise interface.DataException('Problem not found: {}'.format(problem_name)) def post_interaction(self, course_id, problem_name, user_id, correct, attempt, unix_seconds): """ @@ -215,8 +214,6 @@ def post_load(self, course_id, problem_name, user_id, unix_seconds): 'type': 'page_load', 'timestamp': datetime.fromtimestamp(unix_seconds).strftime('%Y-%m-%d %H:%M:%S') } - # NOTE(idegtiarov) checking that load was already stored for this problem doesn't need if we are interesting in - # correct statistic and logging additional/same data is ok self.store.record_data(coll, data) def set_next_problem(self, course_id, user_id, problem_dict): @@ -237,36 +234,35 @@ def advance_problem(self, course_id, user_id): self.store.update_doc(coll, {'student_id': user_id}, {'$set': {'current': current_problem, 'next': None}}) def get_all_remaining_problems(self, course_id, user_id): - return [x for x in self._get_remaining_by_user(course_id, user_id) - if x['pretest'] is False and x['posttest'] is False] + return self._get_remaining_by_user(course_id, user_id, pretest=False, posttest=False) def get_remaining_problems(self, course_id, skill_name, user_id): - remaining = self.get_all_remaining_problems(course_id, user_id) - return [x for x in remaining if skill_name in x['skills']] + return self._get_remaining_by_user(course_id, user_id, skill_name, pretest=False, posttest=False) def get_all_remaining_posttest_problems(self, course_id, user_id): - return [x for x in self._get_remaining_by_user(course_id, user_id) if x['posttest'] is True] + return self._get_remaining_by_user(course_id, user_id, posttest=True) def get_remaining_posttest_problems(self, course_id, skill_name, user_id): - remaining = self.get_all_remaining_posttest_problems(course_id, user_id) - return [x for x in remaining if skill_name in x['skills']] + return self._get_remaining_by_user(course_id, user_id, skill_name, posttest=True) def get_all_remaining_pretest_problems(self, course_id, user_id): - return [x for x in self._get_remaining_by_user(course_id, user_id) if x['pretest'] is True] + return self._get_remaining_by_user(course_id, user_id, pretest=True) def get_remaining_pretest_problems(self, course_id, skill_name, user_id): - remaining = self.get_all_remaining_pretest_problems(course_id, user_id) - return [x for x in remaining if skill_name in x['skills']] + return self._get_remaining_by_user(course_id, user_id, skill_name, pretest=True) - def _get_remaining_by_user(self, course_id, user_id): + def _get_remaining_by_user(self, course_id, user_id, skill_name=None, pretest=None, posttest=None): coll = course_id + COLL_SUFFIX['log'] - done = self.store. get_statistics( + done = self.store.get_statistics( coll, user_id, {'type': 'response'}, 'done', op='$addToSet', op_value='$problem') if done.alive: done = done.next()['done'] - return [problem for problem in self.get_problems(course_id) if problem not in done] + return [ + problem for problem in self.get_problems(course_id, skill_name, pretest, posttest) + if problem not in done + ] else: - return self.get_problems(course_id) + return self.get_problems(course_id, pretest=pretest, posttest=posttest) def post_experiment(self, course_id, experiment_name, start, end): experiment = {'experiment_name': experiment_name, 'start_time': start, 'end_time': end} @@ -328,13 +324,15 @@ def get_raw_user_skill_data(self, course_id, skill_name, user_id): coll = course_id + COLL_SUFFIX['log'] return self.store.get_user_logs(coll, user_id, add_filter={'problem.skills': skill_name}) - def get_next_problem(self, course_id, user_id): + def _get_user_problem(self, course_id, user_id, cur_or_next): coll = course_id + COLL_SUFFIX['user_problem'] - return self.store.get_one(coll, user_id, 'next') + return self.store.get_one(coll, user_id, cur_or_next) + + def get_next_problem(self, course_id, user_id): + return self._get_user_problem(course_id, user_id, 'next') def get_current_problem(self, course_id, user_id): - coll = course_id + COLL_SUFFIX['user_problem'] - return self.store.get_one(coll, user_id, 'current') + return self._get_user_problem(course_id, user_id, 'current') def get_all_interactions(self, course_id, user_id): coll = course_id + COLL_SUFFIX['log'] diff --git a/edx_adapt/data/interface.py b/edx_adapt/data/interface.py index c670757..48e9da6 100644 --- a/edx_adapt/data/interface.py +++ b/edx_adapt/data/interface.py @@ -45,19 +45,13 @@ def post_course(self, course_id): def post_skill(self, course_id, skill_name): raise NotImplementedError( "Data module must implement this" ) - def post_problem(self, course_id, skill_names, problem_name, tutor_url): - raise NotImplementedError( "Data module must implement this" ) - - def post_pretest_problem(self, course_id, skill_names, problem_name, tutor_url): - raise NotImplementedError( "Data module must implement this" ) - - def post_posttest_problem(self, course_id, skill_names, problem_name, tutor_url): + def post_problem(self, course_id, skill_names, problem_name, tutor_url, pretest, posttest): raise NotImplementedError( "Data module must implement this" ) def enroll_user(self, course_id, user_id): raise NotImplementedError( "Data module must implement this" ) - def post_model_params(self, course_id, prob_list, new=False): + def post_model_params(self, course_id, prob_list, new): raise NotImplementedError("Data module must implement this") def get_model_params(self, course_id): @@ -70,7 +64,7 @@ def get_course_ids(self): def get_skills(self, course_id): raise NotImplementedError( "Data module must implement this" ) - def get_problems(self, course_id, skill_name): + def get_problems(self, course_id, skill_name, pretest, posttest): raise NotImplementedError( "Data module must implement this" ) def get_num_pretest(self, course_id, skill_name): diff --git a/edx_adapt/data/mongodb_storage.py b/edx_adapt/data/mongodb_storage.py index 90a3346..6753fbe 100644 --- a/edx_adapt/data/mongodb_storage.py +++ b/edx_adapt/data/mongodb_storage.py @@ -108,6 +108,38 @@ def course_search( return self.db.Courses.find_one({'course_id': course_id, search_field: {'$elemMatch': search_condition}}, {'_id': 0, projection_field: {'$elemMatch': projection_condition}}) + def course_problems_search(self, course_id, search_dict): + """ + Make search for problems in the Course collection + + :param course_id: course id + :param search_dict: is a dict with filter conditions, e.g. {"skill_name"=name, "pretest"=True, "posttest"=True} + :return: list with found problems + """ + cond = [] + cond_key = "$$problem." + + for key, value in search_dict.items(): + cond.append({"$eq": [cond_key + key, value]}) + problems = self.db.Courses.aggregate( + [ + {"$match": {'course_id': course_id}}, + {"$project": { + "_id": 0, "problems": { + "$filter": { + "input": "$problems", "as": "problem", "cond": {"$and": cond} + } + } + }} + ] + ) + if not problems.alive: + logger.warning("Problems with search request: {}; are not found in the course".format( + search_dict, course_id + )) + return [] + return problems.next()["problems"] + def update_doc(self, collection, search_dict, update_dict, new=False): """ Update document in collection @@ -151,7 +183,6 @@ def append(self, coll_name, list_key, val): {'$push': {'val': val}}) def remove(self, table_name, list_key, val): - # TODO: do raise NotImplementedError("Storage module must implement this") def export(self, tables=None): diff --git a/edx_adapt/data/sqlite_storage.py b/edx_adapt/data/sqlite_storage.py deleted file mode 100644 index a9d1c01..0000000 --- a/edx_adapt/data/sqlite_storage.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Storage module that implements StorageInterface using an sqlitedict backend. (Should be) thread safe """ - - -import interface -from sqlitedict import SqliteDict - - -class SqliteStorage(interface.StorageInterface): - - def __init__(self, db_path): - super(SqliteStorage, self).__init__() - #self.db = TinyDB(db_path) - self.path = db_path - #make a "tables" table - self.table_names = SqliteDict(self.path, tablename='__TABLES__', autocommit=True) - self.tables = {} - if('tables' in self.table_names): - for name in self.table_names['tables']: - self.tables[name] = SqliteDict(self.path, tablename=name, autocommit=True) - else: - self.table_names['tables'] = [] - - def create_table(self, table_name): - self._assert_no_table(table_name) - l = self.table_names['tables'] - l.append(table_name) - self.table_names['tables'] = l - table = SqliteDict(self.path, tablename=table_name, autocommit=True) - self.tables[table_name] = table - - print self.table_names['tables'] - - def get_tables(self): - #just return table names, not actual tables - return self.table_names['tables'] - - def get(self, table_name, key): - self._assert_table(table_name) - table = self.tables[table_name] - if key not in table: - raise interface.DataException("Key {} not found in table".format(key)) - return table[key] - - def set(self, table_name, key, val): - self._assert_table(table_name) - table = self.tables[table_name] - table[key] = val - - def append(self, table_name, list_key, val): - self._assert_table(table_name) - table = self.tables[table_name] - if list_key not in table: - raise interface.DataException("List: {0} not in table: {1}".format(list_key, table_name)) - l = table[list_key] - #TODO: check if l is a list maybe - if val in l: - raise interface.DataException("Value: {0} already exists in list: {1}".format(val, list_key)) - l.append(val) - table[list_key] = l - - def remove(self, table_name, list_key, val): - #TODO: do - raise NotImplementedError("Storage module must implement this") - - def export(self, tables=None): - r = {} - for table_name, table in self.tables.items(): - r[table_name] = dict(table) - return r - - def _assert_no_table(self, table_name): - if table_name in self.table_names['tables']: - raise interface.DataException("Table already exists: {}".format(table_name)) - - def _assert_table(self, table_name): - if table_name not in self.table_names['tables']: - print self.table_names['tables'] - raise interface.DataException("Table does not exist: {}".format(table_name)) diff --git a/edx_adapt/data/tinydb_storage.py b/edx_adapt/data/tinydb_storage.py deleted file mode 100644 index 5cdf51c..0000000 --- a/edx_adapt/data/tinydb_storage.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Storage module that implements StorageInterface using a tinydb backend. Not entirely thread safe """ - - -import interface -from tinydb import TinyDB, Query - - -class TinydbStorage(interface.StorageInterface): - - def __init__(self, db_path): - super(TinydbStorage, self).__init__() - self.db = TinyDB(db_path) - - def create_table(self, table_name): - self._assert_no_table(table_name) - table = self.db.table(table_name) - table.insert({'table_exists': True}) - - def get_tables(self): - return self.db.tables() - - def get(self, table_name, key): - self._assert_table(table_name) - table = self.db.table(table_name) - element = Query() - result = table.search(element.key == key) - if len(result) == 0: - raise interface.DataException("Key {} not found in table".format(key)) - return result[0]['val'] - - def set(self, table_name, key, val): - self._assert_table(table_name) - table = self.db.table(table_name) - element = Query() - table.remove(element.key == key) - table.insert({'key': key, 'val': val}) - - def append(self, table_name, list_key, val): - self._assert_table(table_name) - table = self.db.table(table_name) - l = self.get(table_name, list_key) - #TODO: check if l is a list maybe - if val in l: - raise interface.DataException("Value: {0} already exists in list: {1}".format(val, list_key)) - l.append(val) - element = Query() - table.update({'val': l}, element.key == list_key) - - def remove(self, table_name, list_key, val): - #TODO: do - raise NotImplementedError("Storage module must implement this") - - def _assert_no_table(self, table_name): - table = self.db.table(table_name) - if len(table) > 0: - raise interface.DataException("Table already exists: {}".format(table_name)) - - def _assert_table(self, table_name): - table = self.db.table(table_name) - if len(table) == 0: - raise interface.DataException("Table does not exist: {}".format(table_name)) \ No newline at end of file diff --git a/edx_adapt/model/pfm.py b/edx_adapt/model/pfm.py index 8155f88..5b946fc 100644 --- a/edx_adapt/model/pfm.py +++ b/edx_adapt/model/pfm.py @@ -36,8 +36,8 @@ def get_current_probability_correct(self, params): """ return ( 1.0 / (1 + math.exp(-( - params['beta_intercept'] + - params['beta_incorrect'] * self.counts[0] + - params['beta_correct'] * self.counts[1] - ))) + params['beta_intercept'] + + params['beta_incorrect'] * self.counts[0] + + params['beta_correct'] * self.counts[1] + ))) ) diff --git a/edx_adapt/select/skill_separate_random_selector.py b/edx_adapt/select/skill_separate_random_selector.py index 326a828..c348fa7 100644 --- a/edx_adapt/select/skill_separate_random_selector.py +++ b/edx_adapt/select/skill_separate_random_selector.py @@ -2,7 +2,6 @@ from interface import SelectInterface, SelectException from edx_adapt.data.interface import DataException -from edx_adapt.model.interface import ModelException from edx_adapt import logger @@ -47,30 +46,25 @@ def __init__(self, data_interface, model_interface, parameter_access_mode=""): if mode not in self.valid_mode_list: raise SelectException("Parameter access mode is invalid") - def get_p_list(self, namelist, course, user): - probs = self.data_interface.get_problems(course) - prob_list = [] - for name in namelist: - prob_list.extend([x for x in probs if x['problem_name'] == name]) - - done = self.data_interface.get_all_interactions(course, user) - for p in done: - if p['problem'] in prob_list: - prob_list.remove(p['problem']) - - new_prob_list = [] - for prob in prob_list: - skill = prob['skills'][0] - skill_parameter = self.data_interface.get(self._get_key(course, user, skill)) - prob_mastery = self.model_interface.get_probability_mastered( - self.data_interface.get_skill_trajectory(course, skill, user), # trajectory of correctness - skill_parameter # parameters for the skill + def _prepare_problems_list(self, course_id, user_id): + candidate_problem_list = [] # List of problems to choose from + for skill_name in self.data_interface.get_skills(course_id): # For each skill + if skill_name == 'None': + continue + # Gets the parameters corresponding to the course, user, skill - parameter set must include "threshold" + skill_parameter = self.data_interface.get(self._get_key(course_id, user_id, skill_name)) + prob_correct = self.model_interface.get_probability_mastered( + # trajectory of correctness + self.data_interface.get_skill_trajectory(course_id, skill_name, user_id), + skill_parameter # parameters for the skill ) - logger.info("threshold {}, prob mastery {}".format(skill_parameter['threshold'], prob_mastery)) # If the probability is less than threshold, add the problems to candidate list - if prob_mastery < skill_parameter['threshold']: - new_prob_list.append(prob) - return new_prob_list + if prob_correct < skill_parameter['threshold']: + problems_to_add = self.data_interface.get_remaining_problems(course_id, skill_name, user_id) + logger.debug("Skill name: {} UNDER THRESHOLD!".format(skill_name)) + logger.debug("Adding candidates: {}".format(str(problems_to_add))) + candidate_problem_list.extend(problems_to_add) + return candidate_problem_list def choose_next_problem(self, course_id, user_id): """ @@ -81,81 +75,25 @@ def choose_next_problem(self, course_id, user_id): :return: the next problem to give to the user """ try: - #if pretest problems are left, give the next one + # if pretest problems are left, give the next one pretest_problems = self.data_interface.get_all_remaining_pretest_problems(course_id, user_id) - if len(pretest_problems) > 0: - for id in range(14): - prob = 'Pre_assessment_{}'.format(id) - for pre_prob in pretest_problems: - if pre_prob['problem_name'] == prob: - return pre_prob - logger.warning("Something goes wrong while choosing Pretest problem") - #return sorted(pretest_problems, key=lambda k: k['problem_name'])[0] - - #Do the first 3 baseline problems (if model says to) - for prob in self.get_p_list(['b3', 'b4', 'b3_2_0'], course_id, user_id): - return prob - #Do the next 2 problems always - p_done = self.data_interface.get_all_interactions(course_id, user_id) - p_all = self.data_interface.get_problems(course_id) - for prob in ['labels_we', 'skew_easy_0']: - give = True - for p in p_done: - if p['problem']['problem_name'] == prob: - give = False - if give: - ret = [x for x in p_all if x['problem_name'] == prob] - if len(ret) > 0: - return ret[0] + if pretest_problems: + return random.choice(pretest_problems) # if the user has started the post-test, finish it - if len( - [x for x in self.data_interface.get_all_interactions(course_id, user_id) if x['problem']['posttest']] - ) > 0: + if [x for x in self.data_interface.get_all_interactions(course_id, user_id) if x['problem']['posttest']]: post = self.data_interface.get_all_remaining_posttest_problems(course_id, user_id) - if len(post) > 0: - for id in range(14): - prob = 'Post_assessment_' + str(id) - for post_prob in post: - if post_prob['problem_name'] == prob: - return post_prob - logger.warning("Something goes wrong while choosing Post test problem") - - if len(post) == 0: - return {'congratulations': True, 'done': True} - return post[0] - - candidate_problem_list = [] # List of problems to choose from - for skill_name in self.data_interface.get_skills(course_id): # For each skill - if skill_name == 'None': - continue - # Gets the parameters corresponding to the course, user, skill - parameter set must include "threshold" - skill_parameter = self.data_interface.get(self._get_key(course_id, user_id, skill_name)) - prob_correct = self.model_interface.get_probability_mastered( - self.data_interface.get_skill_trajectory(course_id, skill_name, user_id), # trajectory of correctness - skill_parameter # parameters for the skill - ) - # If the probability is less than threshold, add the problems to candidate list - if prob_correct < skill_parameter['threshold']: - logger.info("Skill name: {} UNDER THRESHOLD!".format(skill_name)) - logger.info("Adding candidates: {}".format( - str(self.data_interface.get_remaining_problems(course_id, skill_name, user_id)) - )) - candidate_problem_list.extend( - self.data_interface.get_remaining_problems(course_id, skill_name, user_id) - ) - - if candidate_problem_list: # If candidate list is not empty, randomly choose one from it - return random.choice(candidate_problem_list) - else: # If candidate list is empty, return post-test - return self.data_interface.get_all_remaining_posttest_problems(course_id, user_id)[0] + return random.choice(post) if post else {'congratulations': True, 'done': True} + + # List of problems to choose from + candidate_problem_list = self._prepare_problems_list(course_id, user_id) + return random.choice( + candidate_problem_list if candidate_problem_list + else self.data_interface.get_all_remaining_posttest_problems(course_id, user_id) + ) except DataException as e: raise SelectException("DataException: " + e.message) - except ModelException as e: - raise SelectException("ModelException: " + e.message) - except SelectException as e: - raise e def choose_first_problem(self, course_id, user_id): """ @@ -169,7 +107,6 @@ def choose_first_problem(self, course_id, user_id): for prob in pretest: if prob['problem_name'] == 'Pre_assessment_0': return prob - #return sorted(pretest, key=lambda k: k['problem_name'])[0] def _get_key(self, course_id, user_id, skill_name): """ @@ -190,7 +127,7 @@ def _get_key(self, course_id, user_id, skill_name): raise SelectException("Parameter access mode is invalid") return key.strip() - def get_parameter(self, course_id, user_id=None, skill_name=None): + def _compose_key(self, course_id, user_id, skill_name): mode_id_map = {"course": course_id, "user": user_id, "skill": skill_name} key = "" @@ -199,9 +136,12 @@ def get_parameter(self, course_id, user_id=None, skill_name=None): key += mode_id_map[mode] else: raise SelectException("Mode and the arguments do not match") - return self.data_interface.get(key) + return key - def set_parameter(self, parameter, course_id = None, user_id = None, skill_name = None): + def get_parameter(self, course_id, user_id=None, skill_name=None): + return self.data_interface.get(self._compose_key(course_id, user_id, skill_name)) + + def set_parameter(self, parameter, course_id=None, user_id=None, skill_name=None): """ Set the parameter for the specified course, user, skill (all optional) @@ -210,12 +150,4 @@ def set_parameter(self, parameter, course_id = None, user_id = None, skill_name :param user_id :param skill_name """ - mode_id_map = {"course": course_id, "user": user_id, "skill": skill_name} - - key = "" - for mode in self.parameter_access_mode_list: - if mode_id_map[mode]: - key += mode_id_map[mode] - else: - raise SelectException("Mode and the arguments do not match") - self.data_interface.set(key, parameter) + self.data_interface.set(self._compose_key(course_id, user_id, skill_name), parameter) diff --git a/edx_adapt/tests/logic_tests.py b/edx_adapt/tests/logic_tests.py index 0908bf9..6724411 100644 --- a/edx_adapt/tests/logic_tests.py +++ b/edx_adapt/tests/logic_tests.py @@ -57,7 +57,6 @@ def setUpClass(cls): } _setup_course_in_edxadapt(cls.app, **cls.params) - @classmethod def tearDownClass(cls): # NOTE(idegtiarov) sqlite is too slow for using on server we will support only MongoDB @@ -65,19 +64,25 @@ def tearDownClass(cls): # TODO(idegtiarov) improve application start-up to use another database name for tests mclient.drop_database('edx-adapt') - def _answer_pre_assessment_problems(self, correct_answers=0, attention_question=True): + def setUp(self): + # set up new student + self.student_name = 'test_student_' + id_generator() + + self.app.post( + base_api_path + '/{}/user'.format(self.course_id), + data=json.dumps({'user_id': self.student_name}), + headers=self.headers + ) + + def _answer_pre_assessment_problems(self, correct_answers=0): """ Automation fulfilling Pre-Assessment problems :param correct_answers: int, Number of correct answers - :param attention_question: bool, by default one of correct answers id assigned to AttentionQuestion """ pre_assessments = ['Pre_assessment_{}'.format(i) for i in range(0, 14)] for answered_problem, problem in enumerate(pre_assessments): - is_correct = ( - answered_problem < correct_answers - attention_question or - (correct_answers and attention_question and answered_problem == len(pre_assessments) - 1) - ) + is_correct = answered_problem < correct_answers data = {'problem': problem, 'correct': is_correct, 'attempt': 1} self.app.post( base_api_path + '/{}/user/{}/interaction'.format(self.course_id, self.student_name), @@ -114,6 +119,10 @@ def _add_probabilities_to_user_skill(self, probabilities): }) self.app.post('/api/v1/parameters', data=payload, headers=self.headers) + def _get_problems_num(self, pretest=None, posttest=None): + """Returns number of the problems registered in the course""" + return len(adapt_api.database.get_problems(self.course_id, pretest=pretest, posttest=posttest)) + class CourseTestCase(BaseTestCase): def test_course_created(self): @@ -128,9 +137,8 @@ def test_course_has_skills(self): def test_course_problem_fulfilled(self): problems = json.loads(self.app.get(base_api_path + '/{}'.format(self.course_id)).data) self.assertTrue(problems, msg='Not any problem is found in course') - # NOTE(idegtiarov) Assert correct value is based on amount of problems defined in problems.csv file, if this - # file is changed test will fail. - self.assertEqual(84, len(problems['problems'])) + expecting_problems_number = self._get_problems_num() + self.assertEqual(expecting_problems_number, len(problems['problems'])) def test_course_experiment_fulfilled(self): experiments = json.loads(self.app.get(base_api_path + '/{}/experiment'.format(self.course_id)).data) @@ -148,14 +156,7 @@ def test_course_experiment_fulfilled(self): class PreAssessmentTestCase(BaseTestCase): def setUp(self): - # set up new student - self.student_name = 'test_student_' + id_generator() - - self.app.post( - base_api_path + '/{}/user'.format(self.course_id), - data=json.dumps({'user_id': self.student_name}), - headers=self.headers - ) + super(PreAssessmentTestCase, self).setUp() # Default student params probabilities = {'pg': 0.25, 'ps': 0.25, 'pi': 0.1, 'pt': 0.5, 'threshold': 0.99} @@ -172,23 +173,17 @@ def test_student_enrolled(self): def test_student_cut_off_after_all_correct_answers(self): """ - Test student got status "done_with_course" after answering correctly on all pre-assessment problems. + Test student got status "done_with_course" after answering correctly on more than half pre-assessment problems. """ - self._answer_pre_assessment_problems(correct_answers=8, attention_question=False) + # Course is done if more than half + correct_pretest_to_done_course = self._get_problems_num(pretest=True) // 2 + 1 + self._answer_pre_assessment_problems(correct_answers=correct_pretest_to_done_course) status = json.loads(self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data) self.assertEqual(True, status['done_with_course']) class MainLogicTestCase(BaseTestCase): - def setUp(self): - self.student_name = 'test_student_' + id_generator() - self.app.post( - base_api_path + '/{}/user'.format(self.course_id), - data=json.dumps({'user_id': self.student_name}), - headers=self.headers - ) - def test_alternative_parameters_set_one(self): """ Test student with alternative parameter set one (no need to go through course if pre-assessment not fault). @@ -210,18 +205,23 @@ def test_alternative_parameters_set_two(self): probabilities = {'pg': 0.5, 'ps': 0.5, 'pi': 0.01, 'pt': 0.01, 'threshold': 0.95} self._add_probabilities_to_user_skill(probabilities) self._answer_pre_assessment_problems(correct_answers=5) - pre_and_post_assessment = 28 # Sum of pre-assessment and post-assessment problems - # NOTE(idegtiarov) to ensure that all problems in main part was touched we check that before answered - # all 'main' part's problems edx-adapt doesn't propose problem from Post_assessment part - given_answer = ( - len(adapt_api.database.get_problems(self.course_id)) - pre_and_post_assessment - 1) - for _ in range(given_answer): + + given_answer = self._get_problems_num(pretest=False, posttest=False) + # Checking there is no Post_assessment problem set to "next" before all generic problems are answered + for _ in range(given_answer - 1): self._answer_problem() next_problem = json.loads( self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data )['next'] self.assertFalse(next_problem['posttest']) self.assertFalse(next_problem['problem_name'].startswith('Post_assessment')) + # Checking finally get Post assessment problem in "next" after all generic problems are answered + self._answer_problem() + next_problem = json.loads( + self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data + )['next'] + self.assertTrue(next_problem['posttest']) + self.assertTrue(next_problem['problem_name'].startswith('Post_assessment')) def test_default_parameter_set(self): """ @@ -234,40 +234,8 @@ def test_default_parameter_set(self): probabilities = {'pg': 0.25, 'ps': 0.25, 'pi': 0.1, 'pt': 0.5, 'threshold': 0.99} self._add_probabilities_to_user_skill(probabilities) self._answer_pre_assessment_problems(correct_answers=5) - next_problem = json.loads( - self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data - )['next'] - # NOTE(idegtiarov) ['b3', 'b4', 'b3_2_0'] is list of baseline problems which are mandatory to student after - # pre_assessment part is successfully passed - self.assertEqual('b3', next_problem['problem_name']) - - self._answer_problem() - next_problem = json.loads( - self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data - )['next'] - self.assertEqual('b4', next_problem['problem_name']) - self._answer_problem() - next_problem = json.loads( - self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data - )['next'] - self.assertEqual('b3_2_0', next_problem['problem_name']) - # NOTE(idegtiarov) problems 'labels_we' and 'skew_easy_0' are defined as always to do and they are proposed - # after baseline problems - self._answer_problem() - next_problem = json.loads( - self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data - )['next'] - self.assertEqual('labels_we', next_problem['problem_name']) - - self._answer_problem() - next_problem = json.loads( - self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data - )['next'] - self.assertEqual('skew_easy_0', next_problem['problem_name']) - # NOTE(idegtiarov) next problems come in random order on three of them student answers correctly from second - # attempt - self._answer_problem() + self._answer_problem(repeat=5) self._answer_problem(correct=False) status = json.loads( self.app.get(base_api_path + '/{}/user/{}'.format(self.course_id, self.student_name)).data diff --git a/tools/transform_to_adapt/README.md b/tools/transform_to_adapt/README.md index 2987649..59a3653 100644 --- a/tools/transform_to_adapt/README.md +++ b/tools/transform_to_adapt/README.md @@ -1,6 +1,7 @@ -# Script to transform problem into adaptive capa problem. +# Script to transform problem into adaptive capa problem -Currently script supports transformation for two main problems: `Multiple Choice` and `Checkbox`. +Currently script supports transformation for two main problems: +`Multiple Choice` and `Checkbox`. For example Checkbox problem: ```xml @@ -43,11 +44,14 @@ feedback = { # Answer checking function called when the user hits "Check" def vglcfn(e, ans): - # Here we load the dictionary that EdX creates from the return values of GetState() and GetGrade() + # Here we load the dictionary that EdX creates from the return + # values of GetState() and GetGrade() par = json.loads(ans); - # Then we pull out and parse the return value from GetGrade(). The value from GetState() is in par["state"] + # Then we pull out and parse the return value from GetGrade(). The + # value from GetState() is in par["state"] answer = json.loads(par["answer"]) - # In our case the boolean value named correct_answer is true if the student got the question right + # In our case the boolean value named correct_answer is true if + # the student got the question right return { 'input_list': [ { 'ok': answer['correct_answer'], 'msg': feedback[tuple(answer['inputs'])}, @@ -61,7 +65,8 @@ def vglcfn(e, ans):