diff --git a/clockwork_frontend_test/test_dashboard.py b/clockwork_frontend_test/test_dashboard.py index 581fbaf5..a6df7fcb 100644 --- a/clockwork_frontend_test/test_dashboard.py +++ b/clockwork_frontend_test/test_dashboard.py @@ -20,6 +20,10 @@ DASHBOARD_TABLE_CONTENT = [] for job in fake_data["jobs"]: if job["cw"]["mila_email_username"] == "student00@mila.quebec": + # This element could be an array of states, or a simple string. + # For now, each array we encountered contained only one element. + job_states = job["slurm"]["job_state"] + DASHBOARD_TABLE_CONTENT.append( [ job["slurm"]["cluster_name"], @@ -27,7 +31,9 @@ job["slurm"]["job_id"] ), # job ID is currently handled as a numeric value job["slurm"]["name"], - job["slurm"]["job_state"].lower(), + job_states[0].lower() + if isinstance(job_states, list) + else job_states.lower(), get_default_display_date(job["slurm"]["submit_time"]), get_default_display_date(job["slurm"]["start_time"]), get_default_display_date(job["slurm"]["end_time"]), @@ -180,10 +186,10 @@ def _check_dashboard_table_sorting( header = headers.nth(column_id) expect(header).to_contain_text(column_text) header.click() - _check_dashboard_table(page, content) + _check_dashboard_table(page, content, column_id=column_id) -def _check_dashboard_table(page: Page, table_content: list): +def _check_dashboard_table(page: Page, table_content: list, column_id: int = None): """Check dashboard table contains expected table content. table_content is a list or rows, each row is a list of texts expected in related columns. @@ -195,5 +201,8 @@ def _check_dashboard_table(page: Page, table_content: list): for index_row, content_row in enumerate(table_content): cols = rows.nth(index_row).locator("td") expect(cols).to_have_count(8) - for index_col, content_col in enumerate(content_row): - expect(cols.nth(index_col)).to_contain_text(str(content_col)) + if column_id is None: + for index_col, content_col in enumerate(content_row): + expect(cols.nth(index_col)).to_contain_text(str(content_col)) + else: + expect(cols.nth(column_id)).to_contain_text(str(content_row[column_id])) diff --git a/clockwork_frontend_test/test_jobs_search.py b/clockwork_frontend_test/test_jobs_search.py index 8bc6a4ff..786b6a34 100644 --- a/clockwork_frontend_test/test_jobs_search.py +++ b/clockwork_frontend_test/test_jobs_search.py @@ -3,7 +3,7 @@ from random import choice from clockwork_frontend_test.utils import BASE_URL, get_fake_data -from clockwork_web.core.jobs_helper import get_inferred_job_state +from clockwork_web.core.jobs_helper import get_inferred_job_state, get_str_job_state # Retrieve data we are interested in from the fake data fake_data = get_fake_data() @@ -388,7 +388,8 @@ def test_filter_by_status_except_one(page: Page): job["slurm"]["job_id"], ] for job in sorted_jobs - if get_inferred_job_state(job["slurm"]["job_state"]) != "RUNNING" + if get_inferred_job_state(get_str_job_state(job["slurm"]["job_state"])) + != "RUNNING" ][:40] _check_jobs_table( diff --git a/clockwork_frontend_test/test_jobs_search_for_student06.py b/clockwork_frontend_test/test_jobs_search_for_student06.py index 03b974e5..9c85e634 100644 --- a/clockwork_frontend_test/test_jobs_search_for_student06.py +++ b/clockwork_frontend_test/test_jobs_search_for_student06.py @@ -1,7 +1,7 @@ from playwright.sync_api import Page, expect from clockwork_frontend_test.utils import BASE_URL, get_fake_data -from clockwork_web.core.jobs_helper import get_inferred_job_state +from clockwork_web.core.jobs_helper import get_inferred_job_state, get_str_job_state current_username = "student06@mila.quebec" @@ -283,7 +283,8 @@ def test_filter_by_status_except_one(page: Page): job["slurm"]["job_id"], ] for job in sorted_mila_jobs - if "RUNNING" != get_inferred_job_state(job["slurm"]["job_state"]) + if "RUNNING" + != get_inferred_job_state(get_str_job_state(job["slurm"]["job_state"])) ][:40] _check_jobs_table( page, @@ -419,7 +420,7 @@ def test_multiple_filters(page: Page): ] for job in sorted_jobs if job["cw"]["mila_email_username"] == "student06@mila.quebec" - and get_inferred_job_state(job["slurm"]["job_state"]) + and get_inferred_job_state(get_str_job_state(job["slurm"]["job_state"])) in ["COMPLETED", "PENDING", "FAILED"] ][:40] _check_jobs_table( diff --git a/clockwork_frontend_test/test_settings.py b/clockwork_frontend_test/test_settings.py index 356bdc37..4740b62f 100644 --- a/clockwork_frontend_test/test_settings.py +++ b/clockwork_frontend_test/test_settings.py @@ -1,6 +1,7 @@ from playwright.sync_api import Page, expect +import math -from clockwork_frontend_test.utils import BASE_URL +from clockwork_frontend_test.utils import BASE_URL, get_fake_data def test_languages(page: Page): @@ -42,17 +43,41 @@ def test_nb_items_per_page(page: Page): page.goto(f"{BASE_URL}/jobs/search") # Check we have 40 rows by default in table. rows = page.locator("table#search_table tbody tr") - expect(rows).to_have_count(40) - # Check we have 3 pages in table nav. - nav_elements = page.locator("nav.table_nav ul.pagination li.page-item") - expect(nav_elements).to_have_count(6) - expect(nav_elements.nth(0)).to_have_class("page-item first") - expect(nav_elements.nth(1)).to_have_class("page-item current") - expect(nav_elements.nth(1)).to_have_text("1") - expect(nav_elements.nth(2)).to_have_text("2") - expect(nav_elements.nth(3)).to_have_text("3") - expect(nav_elements.nth(4)).to_have_class("page-item last") - expect(nav_elements.nth(5)).to_have_class("page-item last") + nb_jobs_per_page = 40 + expect(rows).to_have_count(nb_jobs_per_page) + + # Get the number of jobs to display + fake_data = get_fake_data() + nb_jobs = len(fake_data["jobs"]) + # Check how should the table nav look + nb_pages = math.floor(nb_jobs / nb_jobs_per_page) + 1 + + # Check we have X pages in table nav. + def _check_nav_table(nb_pages): + if nb_pages < 4: + nav_elements = page.locator("nav.table_nav ul.pagination li.page-item") + expect(nav_elements).to_have_count(6) + expect(nav_elements.nth(0)).to_have_class("page-item first") + expect(nav_elements.nth(1)).to_have_class("page-item current") + expect(nav_elements.nth(1)).to_have_text("1") + expect(nav_elements.nth(2)).to_have_text("2") + expect(nav_elements.nth(3)).to_have_text("3") + expect(nav_elements.nth(4)).to_have_class("page-item last") + expect(nav_elements.nth(5)).to_have_class("page-item last") + else: + nav_elements = page.locator("nav.table_nav ul.pagination li") + expect(nav_elements).to_have_count(8) + expect(nav_elements.nth(0)).to_have_class("page-item first") + expect(nav_elements.nth(1)).to_have_class("page-item current") + expect(nav_elements.nth(1)).to_have_text("1") + expect(nav_elements.nth(2)).to_have_text("2") + expect(nav_elements.nth(3)).to_have_text("3") + expect(nav_elements.nth(4)).to_have_text("4") + expect(nav_elements.nth(5)).to_have_text("...") + expect(nav_elements.nth(6)).to_have_class("page-item last") + expect(nav_elements.nth(7)).to_have_class("page-item last") + + _check_nav_table(nb_pages) # Go to settings. page.goto(f"{BASE_URL}/settings/") @@ -67,17 +92,10 @@ def test_nb_items_per_page(page: Page): page.goto(f"{BASE_URL}/jobs/search") rows = page.locator("table#search_table tbody tr") expect(rows).to_have_count(25) - nav_elements = page.locator("nav.table_nav ul.pagination li") - expect(nav_elements).to_have_count(8) - expect(nav_elements.nth(0)).to_have_class("page-item first") - expect(nav_elements.nth(1)).to_have_class("page-item current") - expect(nav_elements.nth(1)).to_have_text("1") - expect(nav_elements.nth(2)).to_have_text("2") - expect(nav_elements.nth(3)).to_have_text("3") - expect(nav_elements.nth(4)).to_have_text("4") - expect(nav_elements.nth(5)).to_have_text("...") - expect(nav_elements.nth(6)).to_have_class("page-item last") - expect(nav_elements.nth(7)).to_have_class("page-item last") + + nb_jobs_per_page = 25 + nb_pages = math.floor(nb_jobs / nb_jobs_per_page) + 1 + _check_nav_table(nb_pages) # Move back to 40 jobs per page. page.goto(f"{BASE_URL}/settings/") diff --git a/clockwork_tools_test/conftest.py b/clockwork_tools_test/conftest.py index 7ab22c3c..dfa0ab35 100644 --- a/clockwork_tools_test/conftest.py +++ b/clockwork_tools_test/conftest.py @@ -40,6 +40,29 @@ def invalid_config_00(): return config +@pytest.fixture +def admin_config(fake_data): + """ + The configuration for an admin, who can access all jobs + """ + for user in fake_data["users"]: + if ( + "admin_access" in user and user["admin_access"] + ): # Fake data should always contain an admin + email = user["mila_email_username"] + clockwork_api_key = user["clockwork_api_key"] + break + + config = { + "host": os.environ["clockwork_tools_test_HOST"], + "port": os.environ["clockwork_tools_test_PORT"], + "email": email, + "clockwork_api_key": clockwork_api_key, + } + + return config + + @pytest.fixture def invalid_config_01(): """ @@ -81,6 +104,11 @@ def mtclient(config, db_with_fake_data): return clockwork_tools.client.ClockworkToolsClient(**config) +@pytest.fixture +def mtclient_admin(admin_config, db_with_fake_data): + return clockwork_tools.client.ClockworkToolsClient(**admin_config) + + @pytest.fixture def unauthorized_mtclient_00(invalid_config_00, db_with_fake_data): return clockwork_tools.client.ClockworkToolsClient(**invalid_config_00) diff --git a/clockwork_tools_test/test_mt_jobs.py b/clockwork_tools_test/test_mt_jobs.py index cc334e67..c3b187b0 100644 --- a/clockwork_tools_test/test_mt_jobs.py +++ b/clockwork_tools_test/test_mt_jobs.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize( "cluster_name", ("mila", "cedar", "graham", "beluga", "sephiroth") ) -def test_jobs_list_with_filter(mtclient, fake_data, cluster_name): +def test_jobs_list_with_filter(mtclient_admin, fake_data, cluster_name): """ Test the `jobs_list` command. This is just to make sure that the filtering works and the `cluster_name` argument is functional. @@ -19,41 +19,41 @@ def test_jobs_list_with_filter(mtclient, fake_data, cluster_name): Note that "sephiroth" is not a valid cluster, so we will expect empty lists as results. """ validator = helper_jobs_list_with_filter(fake_data, cluster_name=cluster_name) - LD_jobs = mtclient.jobs_list(cluster_name=cluster_name) + LD_jobs = mtclient_admin.jobs_list(cluster_name=cluster_name) validator(LD_jobs) @pytest.mark.parametrize("username", ("yoshi", "koopatroopa")) -def test_api_list_invalid_username(mtclient, username): +def test_api_list_invalid_username(mtclient_admin, username): """ """ - LD_retrieved_jobs = mtclient.jobs_list(username=username) + LD_retrieved_jobs = mtclient_admin.jobs_list(username=username) # we expect no matches for those made-up names assert len(LD_retrieved_jobs) == 0 @pytest.mark.parametrize("cluster_name", ("mila", "beluga", "cedar", "graham")) -def test_single_job_at_random(mtclient, fake_data, cluster_name): +def test_single_job_at_random(mtclient_admin, fake_data, cluster_name): """ This job entry should be present in the database. """ validator, job_id = helper_single_job_at_random(fake_data, cluster_name) - D_job = mtclient.jobs_one(job_id=job_id) + D_job = mtclient_admin.jobs_one(job_id=job_id) validator(D_job) -def test_single_job_missing(mtclient, fake_data): +def test_single_job_missing(mtclient_admin, fake_data): """ This job entry should be missing from the database. """ validator, job_id = helper_single_job_missing(fake_data) - D_job = mtclient.jobs_one(job_id=job_id) + D_job = mtclient_admin.jobs_one(job_id=job_id) validator(D_job) -def test_list_jobs_for_a_given_random_user(mtclient, fake_data): +def test_list_jobs_for_a_given_random_user(mtclient_admin, fake_data): """ Verify that we list the jobs properly for a given random user. """ validator, username = helper_list_jobs_for_a_given_random_user(fake_data) - LD_jobs = mtclient.jobs_list(username=username) + LD_jobs = mtclient_admin.jobs_list(username=username) validator(LD_jobs) diff --git a/clockwork_web/core/jobs_helper.py b/clockwork_web/core/jobs_helper.py index 37dc7155..10a7fc4d 100644 --- a/clockwork_web/core/jobs_helper.py +++ b/clockwork_web/core/jobs_helper.py @@ -53,6 +53,16 @@ def get_filter_after_end_time(end_time): } +def get_str_job_state(job_state): + """ + Handle the different job state formats we retrieve accross the different Slurm versions + """ + if isinstance(job_state, list) and len(job_state) > 0: + return job_state[0] + + return job_state + + def combine_all_mongodb_filters(*mongodb_filters): """ Creates a big AND clause if more than one argument is given. @@ -442,7 +452,7 @@ def get_inferred_job_state(job_state): Returns the associated Clockwork job state """ - return job_state_to_aggregated[job_state.upper()] + return job_state_to_aggregated[get_str_job_state(job_state).upper()] def get_jobs_properties_list_per_page(): diff --git a/clockwork_web/core/search_helper.py b/clockwork_web/core/search_helper.py index 2650c201..9ecf39db 100644 --- a/clockwork_web/core/search_helper.py +++ b/clockwork_web/core/search_helper.py @@ -106,9 +106,13 @@ def parse_search_request(user, args, force_pagination=True): def search_request(user, args, force_pagination=True): query = parse_search_request(user, args, force_pagination=force_pagination) + username = None + if not user.is_admin() and query.username != user.get_id(): + return (query, [], 0) # A non-admin user can not see other user's jobs + # Call a helper to retrieve the jobs (jobs, nbr_total_jobs) = get_jobs( - username=query.username, + username=query.username if user.is_admin() else user.mila_email_username, cluster_names=query.cluster_name, job_states=query.job_state, job_ids=query.job_ids, diff --git a/clockwork_web/rest_routes/jobs.py b/clockwork_web/rest_routes/jobs.py index 408069e5..e8199e85 100644 --- a/clockwork_web/rest_routes/jobs.py +++ b/clockwork_web/rest_routes/jobs.py @@ -112,8 +112,15 @@ def route_api_v1_jobs_one(): if len(cluster_names) < 1: return jsonify({}), 200 + # If the current user is not an admin, we only retrieve his/her jobs + username = None + if not current_user.is_admin(): + username = current_user.mila_email_username + # Set up the filters and retrieve the expected job - (LD_jobs, _) = get_jobs(job_ids=[job_id], cluster_names=cluster_names) + (LD_jobs, _) = get_jobs( + username=username, job_ids=[job_id], cluster_names=cluster_names + ) if len(LD_jobs) == 0: # Not a great when missing the value we want, but it's an acceptable answer. diff --git a/clockwork_web/templates/dashboard.html b/clockwork_web/templates/dashboard.html index 26a3663f..dca94bdd 100644 --- a/clockwork_web/templates/dashboard.html +++ b/clockwork_web/templates/dashboard.html @@ -78,7 +78,7 @@ }, "job_id": { "label": "{{ gettext('Job ID') }}", - "sortable": "alphabetically" + "sortable": "numeric" }, "job_name": { "label": "{{ gettext('Job name [:20]') }}", diff --git a/clockwork_web/user.py b/clockwork_web/user.py index 0ff1dbd6..be1bfa7a 100644 --- a/clockwork_web/user.py +++ b/clockwork_web/user.py @@ -101,6 +101,9 @@ def boolean(value): def get_id(self): return self.mila_email_username + def is_admin(self): + return self.admin_access + @staticmethod def get(mila_email_username: str): """ @@ -349,3 +352,6 @@ def get_web_settings(self): A dictionary presenting the default web settings. """ return self.web_settings + + def is_admin(self): + return False diff --git a/clockwork_web_test/conftest.py b/clockwork_web_test/conftest.py index be99da9c..61d995c4 100644 --- a/clockwork_web_test/conftest.py +++ b/clockwork_web_test/conftest.py @@ -99,6 +99,19 @@ def valid_rest_auth_headers(): return {"Authorization": f"Basic {encoded_s}"} +@pytest.fixture +def valid_admin_rest_auth_headers(fake_data): + for user in fake_data["users"]: + if "admin_access" in user and user["admin_access"]: + username = user["mila_email_username"] + api_key = user["clockwork_api_key"] + + s = f"{username}:{api_key}" + encoded_bytes = base64.b64encode(s.encode("utf-8")) + encoded_s = str(encoded_bytes, "utf-8") + return {"Authorization": f"Basic {encoded_s}"} + + @pytest.fixture def known_user(app, fake_data): # Assert that the users of the fake data exist and are not empty diff --git a/clockwork_web_test/test_browser_jobs.py b/clockwork_web_test/test_browser_jobs.py index ede58caa..4ae79d3c 100644 --- a/clockwork_web_test/test_browser_jobs.py +++ b/clockwork_web_test/test_browser_jobs.py @@ -31,17 +31,35 @@ get_default_setting_value, get_available_clusters_from_db, ) +from clockwork_web.user import User ########## # Global # ########## +ADMIN_USER = None +NON_ADMIN_USER = None + +# Define the admin and non-admin users (there must be at least one of each in the fake data) +def find_admin_and_non_admin_users(): + global ADMIN_USER + global NON_ADMIN_USER + + # Retrieve the fake data content + with open("test_common/fake_data.json", "r") as infile: + fake_data = json.load(infile) + + for user in fake_data["users"]: + if "admin_access" in user and user["admin_access"]: + ADMIN_USER = user["mila_email_username"] + elif ( + user["mila_email_username"] != "student06@mila.quebec" + ): # It has specific rights + NON_ADMIN_USER = user["mila_email_username"] + if ADMIN_USER and NON_ADMIN_USER: + break -def test_redirect_index(client): - response = client.get("/jobs/") - assert response.status_code == 302 - assert response.headers["Location"] == "dashboard" - +find_admin_and_non_admin_users() #################### # Single job route # @@ -54,8 +72,9 @@ def test_single_job(client, fake_data): found in "fake_data.json" if we want to compare the results that we get when requesting "/jobs/one/". """ - # Log in to Clockwork as student00 (who can access all clusters) - login_response = client.get("/login/testing?user_id=student00@mila.quebec") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + print(f"Login URL: /login/testing?user_id={ADMIN_USER}") + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect D_job = random.choice(fake_data["jobs"]) @@ -77,8 +96,8 @@ def test_single_job_missing_423908482367(client): """ This job entry should be missing from the database. """ - # Log in to Clockwork as student00 (who can access all clusters) - login_response = client.get("/login/testing?user_id=student00@mila.quebec") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect job_id = "423908482367" @@ -93,8 +112,8 @@ def test_single_job_missing_423908482367(client): def test_single_job_no_id(client): - # Log in to Clockwork as student00 (who can access all clusters) - login_response = client.get("/login/testing?user_id=student00@mila.quebec") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect response = client.get("/jobs/one") @@ -118,8 +137,8 @@ def test_search_jobs_for_a_given_random_user(client, fake_data): When we have the html contents better nailed down, we should add some tests to make sure we are returning good values there. """ - # Log in to Clockwork as student00 (who can access all clusters) - login_response = client.get("/login/testing?user_id=student00@mila.quebec") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect # the usual validator doesn't work on the html contents @@ -161,8 +180,8 @@ def test_jobs(client, fake_data: dict[list[dict]]): Note that the `client` fixture depends on other fixtures that are going to put the fake data in the database for us. """ - # Log in to Clockwork as student00 (who can access all clusters) - login_response = client.get("/login/testing?user_id=student00@mila.quebec") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect # Sort the jobs contained in the fake data by submit time, then by job id @@ -204,21 +223,18 @@ def test_jobs_with_both_pagination_options( page_num The number of the page displaying the jobs nbr_items_per_page The number of jobs we want to display per page """ - # Define the user we want to use for this test - current_user_id = "student00@mila.quebec" # Can access all clusters - - # Log in to Clockwork - login_response = client.get(f"/login/testing?user_id={current_user_id}") + # Log in to Clockwork as an admin (who can access all jobs on all clusters) + login_response = client.get(f"/login/testing?user_id={ADMIN_USER}") assert login_response.status_code == 302 # Redirect # Sort the jobs contained in the fake data by submit time, then by job id sorted_all_jobs = sorted( fake_data["jobs"], key=lambda d: ( - d["slurm"]["submit_time"], - -int(d["slurm"]["job_id"]), + -d["slurm"]["submit_time"], + d["slurm"]["job_id"], ), # These are the default values for the sorting - reverse=True, + reverse=False, ) # Get the response @@ -229,7 +245,7 @@ def test_jobs_with_both_pagination_options( # Retrieve the bounds of the interval of index in which the expected jobs # are contained (number_of_skipped_items, nbr_items_per_page) = get_pagination_values( - current_user_id, page_num, nbr_items_per_page + ADMIN_USER, page_num, nbr_items_per_page ) body_text = response.get_data(as_text=True) @@ -260,7 +276,7 @@ def test_jobs_with_page_num_pagination_option( page_num The number of the page displaying the jobs """ # Define the user we want to use for this test - current_user_id = "student00@mila.quebec" # Can access all clusters + current_user_id = ADMIN_USER # Can access all clusters # Log in to Clockwork login_response = client.get(f"/login/testing?user_id={current_user_id}") @@ -270,10 +286,10 @@ def test_jobs_with_page_num_pagination_option( sorted_all_jobs = sorted( fake_data["jobs"], key=lambda d: ( - d["slurm"]["submit_time"], - -int(d["slurm"]["job_id"]), + -d["slurm"]["submit_time"], + d["slurm"]["job_id"], ), # These is the default sorting - reverse=True, + reverse=False, ) # Get the response @@ -313,7 +329,11 @@ def test_jobs_with_nbr_items_per_page_pagination_option( nbr_items_per_page The number of jobs we want to display per page """ # Define the user we want to use for this test - current_user_id = "student00@mila.quebec" # Can access all clusters + # An admin can access all clusters + for user in fake_data["users"]: + if "admin_access" in user and user["admin_access"]: + current_user_id = user["mila_email_username"] + assert current_user_id # Log in to Clockwork login_response = client.get(f"/login/testing?user_id={current_user_id}") @@ -323,10 +343,10 @@ def test_jobs_with_nbr_items_per_page_pagination_option( sorted_all_jobs = sorted( fake_data["jobs"], key=lambda d: ( - d["slurm"]["submit_time"], - -int(d["slurm"]["job_id"]), + -d["slurm"]["submit_time"], + d["slurm"]["job_id"], ), # This is the default sorting option - reverse=True, + reverse=False, ) # Get the response @@ -354,7 +374,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( "current_user_id,username,cluster_names,job_states,page_num,nbr_items_per_page,sort_by,sort_asc", [ ( - "student00@mila.quebec", + ADMIN_USER, "student05@mila.quebec", ["mila", "graham"], ["RUNNING", "PENDING"], @@ -364,7 +384,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( -1, ), ( - "student01@mila.quebec", + ADMIN_USER, "student10@mila.quebec", ["graham"], ["RUNNING", "PENDING"], @@ -384,7 +404,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( 1, ), ( - "student02@mila.quebec", + ADMIN_USER, "student13@mila.quebec", [], ["RUNNING", "PENDING"], @@ -394,7 +414,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( -1, ), ( - "student03@mila.quebec", + ADMIN_USER, "student13@mila.quebec", [], [], @@ -403,7 +423,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( "end_time", 1, ), - ("student04@mila.quebec", "student13@mila.quebec", [], [], -1, -10, "user", 1), + (ADMIN_USER, "student13@mila.quebec", [], [], -1, -10, "user", 1), ( "student05@mila.quebec", "student03@mila.quebec", @@ -414,31 +434,7 @@ def test_jobs_with_nbr_items_per_page_pagination_option( "job_state", -1, ), - ("student00@mila.quebec", None, ["cedar"], [], 2, 10, "name", 1), - ( - "student06@mila.quebec", - "student00@mila.quebec", - ["mila", "cedar"], - [], - 1, - 10, - "end_time", - -1, - ), - # Note: student06 has only access to the Mila cluster and student00 has jobs on Mila cluster and DRAC clusters, - # so these arguments test that student06 should NOT see the jobs on DRAC clusters - ( - "student06@mila.quebec", - "student00@mila.quebec", - ["cedar"], - [], - 1, - 10, - "start_time", - -1, - ), - # Note: student06 has only access to the Mila cluster and student00 has jobs on Mila cluster and DRAC clusters, - # so these arguments test that student06 should NOT see the jobs on DRAC clusters + (ADMIN_USER, None, ["cedar"], [], 2, 10, "name", 1), ], ) def test_route_search( @@ -538,7 +534,12 @@ def test_route_search( retrieved_job_state = D_job["slurm"]["job_state"] # Define the tests which will determine if the job is taken into account or not - test_username = (retrieved_username == username) or ignore_username_filter + if User.get(current_user_id).is_admin(): + test_username = (retrieved_username == username) or ignore_username_filter + else: + test_username = ( + retrieved_username == current_user_id and retrieved_username == username + ) test_cluster_names = retrieved_cluster_name in requested_clusters test_job_states = ( retrieved_job_state in job_states @@ -631,14 +632,10 @@ def test_cc_portal(client, fake_data): fake_data The data our tests are based on. It's a fixture. """ # Choose a user who have access to all the clusters - user_dict = fake_data["users"][0] - assert user_dict["mila_cluster_username"] is not None - assert user_dict["cc_account_username"] is not None + username = ADMIN_USER # Log in to Clockwork as this user - login_response = client.get( - f"/login/testing?user_id={user_dict['mila_email_username']}" - ) + login_response = client.get(f"/login/testing?user_id={username}") assert login_response.status_code == 302 # Redirect # Hidden assumption that the /jobs/search will indeed give us @@ -663,7 +660,7 @@ def test_cc_portal(client, fake_data): # the link is not displayed for a job owned by a user other than the authenticated user if ( D_job_slurm["cluster_name"] not in ["beluga", "narval"] - or D_job["cw"]["mila_email_username"] != user_dict["mila_email_username"] + or D_job["cw"]["mila_email_username"] != username ): continue else: diff --git a/clockwork_web_test/test_rest_jobs.py b/clockwork_web_test/test_rest_jobs.py index b14256a4..60bffa90 100644 --- a/clockwork_web_test/test_rest_jobs.py +++ b/clockwork_web_test/test_rest_jobs.py @@ -30,7 +30,9 @@ @pytest.mark.parametrize("cluster_name", ("mila", "beluga", "cedar", "graham")) -def test_single_job_at_random(client, fake_data, valid_rest_auth_headers, cluster_name): +def test_single_job_at_random( + client, fake_data, valid_admin_rest_auth_headers, cluster_name +): """ Make a request to the REST API endpoint /api/v1/clusters/jobs/one. @@ -44,7 +46,7 @@ def test_single_job_at_random(client, fake_data, valid_rest_auth_headers, cluste response = client.get( f"/api/v1/clusters/jobs/one?job_id={job_id}", - headers=valid_rest_auth_headers, + headers=valid_admin_rest_auth_headers, ) assert "application/json" in response.content_type D_job = response.json @@ -68,20 +70,19 @@ def test_single_job_missing(client, fake_data, valid_rest_auth_headers): validator(D_job) -def test_single_job_no_id(client, valid_rest_auth_headers): - response = client.get("/api/v1/clusters/jobs/one", headers=valid_rest_auth_headers) +def test_single_job_no_id(client, valid_admin_rest_auth_headers): + response = client.get( + "/api/v1/clusters/jobs/one", headers=valid_admin_rest_auth_headers + ) assert response.status_code == 400 -def test_list_jobs_for_a_given_random_user(client, fake_data, valid_rest_auth_headers): +def test_list_jobs_for_a_given_random_user( + client, fake_data, valid_admin_rest_auth_headers +): """ Make a request to the REST API endpoint /api/v1/clusters/jobs/list. - Note that the users for whom we list the jobs is probably not - going to be the user encoded in the auth_headers, which we get - from the configuration variables "clockwork.test.email" - and "clockwork.test.api_key". - We pick a user at random from our fake_data, and we see if we can list the jobs for that user. """ @@ -90,7 +91,7 @@ def test_list_jobs_for_a_given_random_user(client, fake_data, valid_rest_auth_he response = client.get( f"/api/v1/clusters/jobs/list?username={username}", - headers=valid_rest_auth_headers, + headers=valid_admin_rest_auth_headers, ) assert response.content_type == "application/json" LD_jobs = response.get_json() @@ -117,7 +118,7 @@ def test_api_list_invalid_username(client, valid_rest_auth_headers, username): "cluster_name", ("mila", "cedar", "graham", "beluga", "sephiroth") ) def test_jobs_list_with_filter( - client, fake_data, valid_rest_auth_headers, cluster_name + client, fake_data, valid_admin_rest_auth_headers, cluster_name ): """ Test the `jobs_list` command. This is just to make sure that the filtering works @@ -128,7 +129,7 @@ def test_jobs_list_with_filter( validator = helper_jobs_list_with_filter(fake_data, cluster_name=cluster_name) response = client.get( f"/api/v1/clusters/jobs/list?cluster_name={cluster_name}", - headers=valid_rest_auth_headers, + headers=valid_admin_rest_auth_headers, ) assert response.content_type == "application/json" assert response.status_code == 200 diff --git a/clockwork_web_test/test_rest_nodes.py b/clockwork_web_test/test_rest_nodes.py index 427e7c31..8af834b6 100644 --- a/clockwork_web_test/test_rest_nodes.py +++ b/clockwork_web_test/test_rest_nodes.py @@ -8,7 +8,7 @@ import pytest -@pytest.mark.parametrize("cluster_name", ("mila", "beluga", "cedar", "graham")) +@pytest.mark.parametrize("cluster_name", ("mila", "beluga", "cedar")) def test_single_node_at_random( client, fake_data, valid_rest_auth_headers, cluster_name ): diff --git a/docker-compose.yml b/docker-compose.yml index a2d0ed1e..e639467d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,7 @@ services: - ./clockwork_tools_test:/clockwork/clockwork_tools_test - ./clockwork_web:/clockwork/clockwork_web - ./clockwork_web_test:/clockwork/clockwork_web_test + - ./slurm_report:/clockwork/tmp/slurm_report - ./slurm_state:/clockwork/slurm_state - ./slurm_state_test:/clockwork/slurm_state_test - ./scripts:/clockwork/scripts diff --git a/scripts/concat_json_lists.py b/scripts/concat_json_lists.py index 49d349b6..29544db0 100644 --- a/scripts/concat_json_lists.py +++ b/scripts/concat_json_lists.py @@ -28,7 +28,10 @@ def main(argv): # regardless of the repartition of the jobs and nodes by clusters, # we divide the number we want to keep by the number of input # sources we use, and keep this number of elements from each source. - kept_elements_nb = round(args.keep / len(args.inputs)) + 1 + if args.keep is not None: + kept_elements_nb = round(args.keep / len(args.inputs)) + 1 + else: + kept_elements_nb = None # Nota bene: as this number has to be an integer, we are not sure to # have a perfect fraction of the requested number. Thus, we round up the # result then add one: the surplus would then be withdrawn from the @@ -44,6 +47,8 @@ def main(argv): # Get a part of the data np.random.shuffle(E) E = E[0:kept_elements_nb] + if kept_elements_nb is not None: + E = E[0:kept_elements_nb] L.extend(E) if args.keep is not None: diff --git a/scripts/produce_fake_data.sh b/scripts/produce_fake_data.sh index 8d3143b0..9271f06d 100644 --- a/scripts/produce_fake_data.sh +++ b/scripts/produce_fake_data.sh @@ -53,13 +53,17 @@ for SUBFOLDER in ${CLOCKWORK_ROOT}/tmp/slurm_report/*; do # "job anonymized" to "jobs anonymized dump file" # "node anonymized" to "nodes anonymized dump file" ANONYMIZED_JOBS_FILE=${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/job_anonymized_dump_file.json +: <