From bf111d560829b3568368eb25736c91abf176de8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Wed, 20 Mar 2024 07:52:00 +0100 Subject: [PATCH] WIP new validators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédric Foellmi --- oort/cli/cli.py | 72 ++++++--- oort/cli/folders.py | 217 ---------------------------- oort/cli/helpers.py | 74 +++++++--- oort/cli/validators.py | 170 ++++++++++++++++++++++ oort/monitor/errors.py | 20 +++ oort/shared/identity.py | 104 ++++++------- oort/uploader/engine/preparator.py | 16 +- oort/uploader/engine/walker.py | 4 - setup.py | 3 +- tests/cli/test_cli_folders_parse.py | 2 - tests/uploader/test_packer.py | 2 +- tests/uploader/test_preparator.py | 2 +- 12 files changed, 349 insertions(+), 337 deletions(-) delete mode 100644 oort/cli/folders.py create mode 100644 oort/cli/validators.py diff --git a/oort/cli/cli.py b/oort/cli/cli.py index 7979ffac..4df4bd3c 100644 --- a/oort/cli/cli.py +++ b/oort/cli/cli.py @@ -6,11 +6,11 @@ from arcsecond import ArcsecondAPI from oort import __version__ -from oort.cli.folders import (parse_upload_watch_options, save_upload_folders) -from oort.cli.helpers import display_command_summary +from oort.cli.helpers import display_command_summary, save_upload_folders, build_endpoint_kwargs from oort.cli.options import State, basic_options from oort.cli.supervisor import (get_supervisor_processes_status, get_supervisor_config) -from oort.monitor.errors import InvalidWatchOptionsOortCloudError +from oort.cli.validators import parse_upload_watch_options +from oort.monitor.errors import InvalidWatchOptionsOortCloudError, InvalidUploadOptionsOortCloudError from oort.shared.config import (get_oort_config_upload_folder_sections, remove_oort_config_folder_section, update_oort_config_upload_folder_sections_key) @@ -160,8 +160,12 @@ def folders(state): click.echo(" organisation = (no organisation)") if section.get('telescope'): click.echo(f" telescope = {section.get('telescope')}") + elif section.get('telescope_uuid'): + click.echo(f" telescope = {section.get('telescope_name')} ({section.get('telescope_uuid')})") else: click.echo(" telescope = (no telescope)") + if section.get('dataset_name'): + click.echo(f" dataset = {section.get('dataset_name')} ({section.get('dataset_uuid')})") click.echo(f" path = {section.get('path')}") click.echo(f" zip = {section.get('zip', 'False')}") click.echo() @@ -170,24 +174,24 @@ def folders(state): @main.command(help="Display the list of (organisation) telescopes.") @click.option('-o', '--organisation', required=False, nargs=1, - help="The Organisation subdomain, if uploading to an organisation.") + help="The subdomain, if uploading to an organisation (Observatory Portal).") @basic_options @pass_state def telescopes(state, organisation=None): - test = os.environ.get('OORT_TESTS') == '1' - kwargs = {'api': state.api_name, 'test': test, 'upload_key': ArcsecondAPI.upload_key(api=state.api_name)} - org_subdomain = organisation or '' if org_subdomain: - kwargs.update(organisation=org_subdomain) click.echo(f" • Fetching telescopes for organisation {org_subdomain}...") else: click.echo(" • Fetching telescopes...") + kwargs = build_endpoint_kwargs(state.api_name, subdomain=organisation) telescope_list, error = ArcsecondAPI.telescopes(**kwargs).list() if error is not None: raise OortCloudError(str(error)) + if isinstance(telescope_list, dict) and 'results' in telescope_list.keys(): + telescope_list = telescope_list['results'] + click.echo(f" • Found {len(telescope_list)} telescope{'s' if len(telescope_list) > 1 else ''}.") for telescope_dict in telescope_list: s = f" 🔭 \"{telescope_dict['name']}\" " @@ -198,28 +202,55 @@ def telescopes(state, organisation=None): click.echo(s) +@main.command(help="Display the list of (organisation) datasets.") +@click.option('-o', '--organisation', + required=False, nargs=1, + help="The subdomain, if uploading to an organisation (Observatory Portal).") +@basic_options +@pass_state +def datasets(state, organisation=None): + org_subdomain = organisation or '' + if org_subdomain: + click.echo(f" • Fetching datasets for organisation '{org_subdomain}'...") + else: + click.echo(" • Fetching datasets...") + + kwargs = build_endpoint_kwargs(state.api_name, subdomain=organisation) + dataset_list, error = ArcsecondAPI.datasets(**kwargs).list() + if error is not None: + raise OortCloudError(str(error)) + + if isinstance(dataset_list, dict) and 'results' in dataset_list.keys(): + dataset_list = dataset_list['results'] + + click.echo(f" • Found {len(dataset_list)} dataset{'s' if len(dataset_list) > 1 else ''}.") + for dataset_dict in dataset_list: + s = f" 💾 \"{dataset_dict['name']}\" " + s += f"(uuid: {dataset_dict['uuid']}) " + # s += f"[ObservingSite UUID: {telescope_dict['observing_site']}]" + click.echo(s) + + @main.command(help='Directly upload a folder\'s content.') @click.argument('folder', required=True, nargs=1) @click.option('-o', '--organisation', required=False, nargs=1, - help="The Organisation subdomain, if uploading to an organisation.") + help="The subdomain, if uploading for an Observatory Portal.") @click.option('-t', '--telescope', required=False, nargs=1, type=click.STRING, - help="The UUID or the alias of the telescope acquiring the data (mandatory only for organisation " - "uploads).") + help="The UUID or alias of the telescope acquiring the data (mandatory only for Portal uploads).") +@click.option('-d', '--dataset', + required=False, nargs=1, type=click.STRING, + help="The UUID or name of the dataset to put data in.") @click.option('-f', '--force', required=False, nargs=1, type=click.BOOL, is_flag=True, - help="Force the re-uploading of folder's content, resetting the local Uploads information. Default is " - "False.") + help="Force the re-uploading of folder's data, resetting the local Uploads information. Default is False.") @click.option('-z', '--zip', required=False, nargs=1, type=click.BOOL, is_flag=True, help="Zip the data files (FITS and XISF) before sending to the cloud. Default is False.") -@click.option('-d', '--dataset', - required=False, nargs=1, type=click.STRING, - help="Put all data contained in the folder into a single dataset designated by its name or UUID.") @basic_options @pass_state -def upload(state, folder, organisation=None, telescope=None, force=False, zip=False, dataset=None): +def upload(state, folder, organisation=None, telescope=None, dataset=None, force=False, zip=False): """ Upload the content of a folder. @@ -236,8 +267,9 @@ def upload(state, folder, organisation=None, telescope=None, force=False, zip=Fa click.echo(f"{80 * '*'}\n") try: - identity = parse_upload_watch_options(organisation, telescope, zip, dataset, state.api_name) - except InvalidWatchOptionsOortCloudError: + identity = parse_upload_watch_options(organisation, telescope, dataset, zip, state.api_name) + except InvalidUploadOptionsOortCloudError as e: + click.echo(f"\n • ERROR {str(e)} \n") return display_command_summary([folder, ], identity) @@ -280,7 +312,7 @@ def watch(state, folders, organisation=None, telescope=None, zip=False): click.echo(f"{80 * '*'}\n") try: - identity = parse_upload_watch_options(organisation, telescope, zip, state.api_name) + identity = parse_upload_watch_options(organisation, telescope, '', zip, state.api_name) except InvalidWatchOptionsOortCloudError: return diff --git a/oort/cli/folders.py b/oort/cli/folders.py deleted file mode 100644 index 45a389b0..00000000 --- a/oort/cli/folders.py +++ /dev/null @@ -1,217 +0,0 @@ -import os -import pathlib -from typing import Optional, Union - -import click -from arcsecond import ArcsecondAPI -from click import UUID -from peewee import DoesNotExist - -from oort.monitor.errors import ( - InvalidAstronomerOortCloudError, - InvalidOrgMembershipOortCloudError, - InvalidOrganisationTelescopeOortCloudError, - InvalidOrganisationUploadKeyOortCloudError, - InvalidTelescopeOortCloudError, - InvalidWatchOptionsOortCloudError, - UnknownOrganisationOortCloudError -) -from oort.shared.config import get_oort_logger -from oort.shared.identity import Identity -from oort.shared.models import Organisation - - -def check_local_astronomer(api: str): - test = os.environ.get('OORT_TESTS') == '1' - - username = ArcsecondAPI.username(test=test, api=api) - if username is None: - raise InvalidAstronomerOortCloudError('') - - upload_key = ArcsecondAPI.upload_key(test=test, api=api) - return username, upload_key - - -def check_remote_organisation(org_subdomain: str, api: str = 'main'): - try: - return Organisation.get(subdomain=org_subdomain) - except DoesNotExist: - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - org_resource, error = ArcsecondAPI.organisations(test=test, api=api, upload_key=upload_key).read(org_subdomain) - if error is not None: - raise UnknownOrganisationOortCloudError(org_subdomain, str(error)) - else: - return Organisation.create(subdomain=org_subdomain) - - -def check_local_astronomer_remote_organisation_membership(org_subdomain: str, api: str) -> str: - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - - # Reading local memberships if an org_subdomain is provided. - memberships = ArcsecondAPI.memberships(test=test, api=api, upload_key=upload_key) if org_subdomain else None - - # An org_subdomain is provided, but memberships are empty. - if org_subdomain and memberships in [None, {}]: - raise InvalidOrgMembershipOortCloudError(org_subdomain) - - # Ok, an org_subdomain is provided and memberships are not empty. Checking for membership - # of that org_subdomain AND checking it is at least a member (not a visitor). - role = memberships.get(org_subdomain, None) - if not role or role not in ['member', 'admin', 'superadmin']: - raise InvalidOrgMembershipOortCloudError(org_subdomain) - - return role - - -def list_organisation_telescopes(org_subdomain: str, api: str): - click.echo("Error: if an organisation is provided, you must specify a telescope UUID.") - click.echo(f"Here is a list of existing telescopes for organisation {org_subdomain}:") - - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - telescope_list, error = ArcsecondAPI.telescopes(organisation=org_subdomain, - api=api, - test=test, - upload_key=upload_key).list() - for telescope in telescope_list: - click.echo(f" • {telescope['name']} ({telescope['uuid']})") - - -# The organisation is actually optional. It allows to check for a telescope -# also in the case of a custom astronomer. -def check_telescope(telescope_uuid: Union[str, UUID], - org_subdomain: Optional[str], - api: Optional[str]) -> \ - Optional[dict]: - click.echo(" • Fetching telescope details...") - - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - - kwargs = {'test': test, 'api': api, 'upload_key': upload_key} - if org_subdomain: - kwargs.update(organisation=org_subdomain) - - telescope_detail, error = ArcsecondAPI.telescopes(**kwargs).read(str(telescope_uuid)) - if error is not None: - if org_subdomain: - raise InvalidOrganisationTelescopeOortCloudError(str(telescope_uuid), org_subdomain, str(error)) - else: - raise InvalidTelescopeOortCloudError(str(telescope_uuid), str(error)) - - if telescope_detail is not None and telescope_detail.get('coordinates', None) is None: - site_uuid = telescope_detail.get('observing_site', None) - site_detail, error = ArcsecondAPI.observingsites(api=api, upload_key=upload_key).read(site_uuid) - if site_detail: - telescope_detail['coordinates'] = site_detail.get('coordinates') - - return telescope_detail - - -def check_remote_astronomer(username: str, upload_key: str, api: str): - click.echo("Checking astronomer credentials...") - - if username is None or upload_key is None: - raise InvalidWatchOptionsOortCloudError() - - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - - result, error = ArcsecondAPI.me(test=test, api=api, upload_key=upload_key).read(username) - if error is not None: - raise InvalidAstronomerOortCloudError(username, upload_key, error_string=str(error)) - - -def check_organisation_shared_keys(org_subdomain: str, username: str, upload_key: str, api: str): - test = os.environ.get('OORT_TESTS') == '1' - upload_key = ArcsecondAPI.upload_key(api=api) - - kwargs = {'api': api, 'test': test, 'organisation': org_subdomain, upload_key: upload_key} - upload_keys_details, error = ArcsecondAPI.uploadkeys(**kwargs).list() - if error is not None: - InvalidWatchOptionsOortCloudError(str(error)) - - upload_keys_mapping = {uk['username']: uk for uk in upload_keys_details} - upload_key_details = upload_keys_mapping.get(username, None) - if upload_key_details is None: - raise InvalidOrganisationUploadKeyOortCloudError(org_subdomain, username, upload_key) - - upload_key = upload_key_details.get('key', None) - if upload_key is None or upload_key != upload_key: - raise InvalidOrganisationUploadKeyOortCloudError(org_subdomain, username, upload_key) - - -def parse_upload_watch_options(organisation: str = '', - telescope: str = '', - zip: bool = True, - dataset: str = '', - api: str = 'main') -> Identity: - assert api != '' - telescope_uuid = telescope or '' - org_subdomain = organisation or '' - - # Validation of the organisation # - - org = None - org_role = '' - if org_subdomain: - # Check that the provided subdomain corresponds to an existing remote organisation. - org = check_remote_organisation(org_subdomain, api) - # Check that the provided subdomain is an organisation of which the current logged in astronomer is a member. - org_role = check_local_astronomer_remote_organisation_membership(org_subdomain, api) - - # Validation of the telescope # - - # In every case, check for telescope details if a UUID is provided. - telescope_details = None - if telescope_uuid: - telescope_details = check_telescope(telescope_uuid, org_subdomain, api) - - # Validation of the uploader # - - # Fetch the username of the currently logged in astronomer. - username, upload_key = check_local_astronomer(api) - if not username or not upload_key: - raise InvalidWatchOptionsOortCloudError('Missing username or upload_key.') - - # If we have an organisation and no telescope UUID, we list the one available - # and then raise an error - if org is not None and telescope_details is None: - list_organisation_telescopes(org_subdomain, api) - raise InvalidWatchOptionsOortCloudError('For an organisation, a telescope UUID must be provided.') - - identity = Identity(username, - upload_key, - org_subdomain, - org_role, - telescope_uuid, - zip, - dataset, - api) - - if telescope_details is not None: - identity.attach_telescope_details(**telescope_details) - - return identity - - -def save_upload_folders(folders: list, identity: Identity) -> list: - logger = get_oort_logger('cli', debug=identity.api == 'dev') - - prepared_folders = [] - for raw_folder in folders: - upload_path = pathlib.Path(raw_folder).resolve() - - if not upload_path.exists() and os.environ.get('OORT_TESTS') != '1': - logger.warn(f'Upload folder "{upload_path}" does not exists. Skipping.') - continue - - if upload_path.is_file(): - upload_path = upload_path.parent - - identity.save_with_folder(upload_folder_path=str(upload_path)) - prepared_folders.append((str(upload_path), identity)) - - return prepared_folders diff --git a/oort/cli/helpers.py b/oort/cli/helpers.py index b50ee6d2..4f7878f6 100644 --- a/oort/cli/helpers.py +++ b/oort/cli/helpers.py @@ -1,37 +1,44 @@ +import os import pathlib +from typing import Optional import click +from arcsecond import ArcsecondAPI -from oort.shared.config import (get_oort_config_upload_folder_sections) -from oort.shared.utils import get_formatted_bytes_size, get_formatted_size_times, is_hidden +from oort.shared.config import (get_oort_config_upload_folder_sections, get_oort_logger) from oort.shared.identity import Identity +from oort.shared.utils import get_formatted_bytes_size, get_formatted_size_times, is_hidden def display_command_summary(folders: list, identity: Identity): - click.echo(f"\n --- Folder{'s' if len(folders) > 1 else ''} summary --- ") + click.echo(f"\n --- Upload/watch summary --- ") click.echo(f" • Arcsecond username: @{identity.username} (Upload key: {identity.upload_key[:4]}••••)") - if not identity.subdomain: - click.echo(" • Uploading to your *personal* account.") + if identity.subdomain: + msg = f" • Uploading to Observatory Portal '{identity.subdomain}' (as {identity.role})." + else: + msg = " • Uploading to your *personal* account." + click.echo(msg) + + if identity.dataset_uuid and identity.dataset_name: + msg = f" • Data will be appended to existing dataset '{identity.dataset_name}' ({identity.dataset_uuid})." + elif not identity.dataset_uuid and identity.dataset_name: + msg = f" • Data will be inserted into a new dataset named '{identity.dataset_name}'." else: - click.echo(f" • Uploading to organisation account '{identity.subdomain}' (as {identity.role}).") - - if identity.telescope_details: - msg = f" • Datasets will be attached to telescope '{identity.telescope_details.get('name')}' " - if identity.telescope_details.get('alias', ''): - msg += f"alias \"{identity.telescope_details.get('alias')}\" " - msg += f"({identity.telescope_details.get('uuid')}))" - click.echo(msg) + msg = " • Using folder names for dataset names (one folder = one dataset)." + click.echo(msg) + + if identity.telescope_uuid: + msg = f" • Dataset(s) will be attached to telescope '{identity.telescope_name}' " + if identity.telescope_alias: + msg += f"a.k.a '{identity.telescope_alias}' " + msg += f"({identity.telescope_uuid}))" else: - click.echo(" • No designated telescope.") + msg = " • No designated telescope." + click.echo(msg) click.echo(f" • Using API server: {identity.api}") click.echo(f" • Zip before upload: {'True' if zip else 'False'}") - if identity.dataset: - click.echo(f" • Ignoring folder names. Using a single dataset with name|uuid {identity.dataset}.") - else: - click.echo(" • Using folder names for dataset names (one folder = one dataset).") - home_path = pathlib.Path.home() existing_folders = [section.get('path') for section in get_oort_config_upload_folder_sections()] @@ -46,3 +53,32 @@ def display_command_summary(folders: list, identity: Identity): size = sum(f.stat().st_size for f in folder_path.glob('**/*') if f.is_file() and not is_hidden(f)) click.echo(f" > Volume: {get_formatted_bytes_size(size)} in total in this folder.") click.echo(f" > Estimated upload time: {get_formatted_size_times(size)}") + + +def save_upload_folders(folders: list, identity: Identity) -> list: + logger = get_oort_logger('cli', debug=identity.api == 'dev') + + prepared_folders = [] + for raw_folder in folders: + upload_path = pathlib.Path(raw_folder).resolve() + + if not upload_path.exists() and os.environ.get('OORT_TESTS') != '1': + logger.warning(f'Upload folder "{upload_path}" does not exists. Skipping.') + continue + + if upload_path.is_file(): + upload_path = upload_path.parent + + identity.save_with_folder(upload_folder_path=str(upload_path)) + prepared_folders.append((str(upload_path), identity)) + + return prepared_folders + + +def build_endpoint_kwargs(api: str = 'main', subdomain: Optional[str] = None): + test = os.environ.get('OORT_TESTS') == '1' + upload_key = ArcsecondAPI.upload_key(api=api) + kwargs = {'test': test, 'api': api, 'upload_key': upload_key} + if subdomain is not None: + kwargs.update(organisation=subdomain) + return kwargs diff --git a/oort/cli/validators.py b/oort/cli/validators.py new file mode 100644 index 00000000..0d375802 --- /dev/null +++ b/oort/cli/validators.py @@ -0,0 +1,170 @@ +import os +import uuid +from typing import Optional, Union + +import click +from arcsecond import ArcsecondAPI +from click import UUID + +from oort.cli.helpers import build_endpoint_kwargs +from oort.monitor.errors import ( + InvalidAstronomerOortCloudError, + InvalidOrgMembershipOortCloudError, + InvalidOrganisationTelescopeOortCloudError, + InvalidTelescopeOortCloudError, + InvalidWatchOptionsOortCloudError, + UnknownOrganisationOortCloudError, + InvalidOrganisationDatasetOortCloudError, + InvalidDatasetOortCloudError, + InvalidUploadOptionsOortCloudError +) +from oort.shared.identity import Identity + + +def __validate_remote_organisation(org_subdomain: str, api: str = 'main') -> dict: + click.echo(f" • Fetching details of organisation {org_subdomain}...") + + # Do NOT include subdomain since we want to access the public endpoint of the list of organisations here. + # Including the subdomain kwargs would filter the result for that organisation. + # Which would make no sense, since there is no list of organisations for a given organisation... + kwargs = build_endpoint_kwargs(api) + org_details, error = ArcsecondAPI.organisations(**kwargs).read(org_subdomain) + if error is not None: + raise UnknownOrganisationOortCloudError(org_subdomain, str(error)) + + return org_details + + +def __validate_astronomer_role_in_remote_organisation(org_subdomain: str, api: str = 'main') -> str: + # Do NOT include subdomain since we want the global list of memberships of the profile here. + # An alternative would be to fetch the list of members of a given organisation and see if the profile + # is inside. But there is actually no need to fetch anything since the memberships are included in + # the profile of the user logged in. + kwargs = build_endpoint_kwargs(api) + # Reading local memberships if an org_subdomain is provided. + memberships = ArcsecondAPI.memberships(**kwargs) if org_subdomain else None + + # An org_subdomain is provided, but memberships are empty. + if org_subdomain and memberships in [None, {}]: + raise InvalidOrgMembershipOortCloudError(org_subdomain) + + # Ok, an org_subdomain is provided and memberships are not empty. Checking for membership + # of that org_subdomain AND checking it is at least a member (not a visitor). + role = memberships.get(org_subdomain, None) + if not role or role not in ['member', 'admin', 'superadmin']: + raise InvalidOrgMembershipOortCloudError(org_subdomain) + + return role + + +def __print_organisation_telescopes(org_subdomain: str, api: str = 'main') -> None: + click.echo(f" • Here is a list of existing telescopes for organisation '{org_subdomain}':") + kwargs = build_endpoint_kwargs(api, org_subdomain) + telescope_list, error = ArcsecondAPI.telescopes(**kwargs).list() + for telescope in telescope_list: + s = f" 🔭 {telescope['name']}" + if telescope.get('alias', ''): + s += f" a.k.a. {telescope['alias']}" + s += f" ({telescope['uuid']})" + click.echo(s) + + +# The organisation is actually optional. It allows to check for a telescope +# also in the case of an individual astronomer. +def __validate_telescope_uuid(telescope_uuid_or_alias: Union[str, UUID], + org_subdomain: Optional[str], + api: str = 'main') -> Optional[dict]: + click.echo(f" • Fetching details of telescope {telescope_uuid_or_alias}...") + + # To the contrary of datasets, telescope models have a field 'alias', and it also works for + # reading telescope details in /telescopes//, even for organisations. + # Hence, a single "read" is enough to find out if the telescope exists. + kwargs = build_endpoint_kwargs(api, org_subdomain) + telescope_detail, error = ArcsecondAPI.telescopes(**kwargs).read(str(telescope_uuid_or_alias)) + if error is not None: + if org_subdomain: + raise InvalidOrganisationTelescopeOortCloudError(str(telescope_uuid_or_alias), org_subdomain, str(error)) + else: + raise InvalidTelescopeOortCloudError(str(telescope_uuid_or_alias), str(error)) + + # Will contain UUID, name and alias. + return telescope_detail + + +# The organisation is actually optional. It allows to check for a dataset +# also in the case of an individual astronomer. +def __validate_dataset_uuid(dataset_uuid_or_name: Union[str, UUID], + org_subdomain: Optional[str], + api: str = 'main') -> Optional[dict]: + kwargs = build_endpoint_kwargs(api, org_subdomain) + dataset = None + + try: + uuid.UUID(dataset_uuid_or_name) + except ValueError: + click.echo(f" • Parameter {dataset_uuid_or_name} is not an UUID. Looking for a dataset with that name...") + datasets_list, error = ArcsecondAPI.datasets(**kwargs).list({'name': dataset_uuid_or_name}) + if len(datasets_list) == 0: + click.echo(f" • No dataset with name {dataset_uuid_or_name} found. It will be created.") + dataset = {'name': dataset_uuid_or_name} + elif len(datasets_list) == 1: + click.echo(f" • One dataset known with name {dataset_uuid_or_name}. Data will be appended to it.") + dataset = datasets_list[0] + else: + error = f"Multiple datasets with name containing {dataset_uuid_or_name} found. Be more specific." + else: + click.echo(f" • Fetching details of dataset {dataset_uuid_or_name}...") + dataset, error = ArcsecondAPI.datasets(**kwargs).read(str(dataset_uuid_or_name)) + + if error is not None: + if org_subdomain: + raise InvalidOrganisationDatasetOortCloudError(str(dataset_uuid_or_name), org_subdomain, str(error)) + else: + raise InvalidDatasetOortCloudError(str(dataset_uuid_or_name), str(error)) + + return dataset + + +def ___read_local_astronomer_credentials(api: str): + test = os.environ.get('OORT_TESTS') == '1' + + username = ArcsecondAPI.username(test=test, api=api) + if username is None: + raise InvalidAstronomerOortCloudError('') + + upload_key = ArcsecondAPI.upload_key(test=test, api=api) + return username, upload_key + + +def parse_upload_watch_options(subdomain: str = '', + telescope_uuid_or_name: str = '', + dataset_uuid_or_name: str = '', + zip: bool = True, + api: str = 'main') -> Identity: + assert api != '' and api is not None + + username, upload_key = ___read_local_astronomer_credentials(api) + if not username or not upload_key: + raise InvalidWatchOptionsOortCloudError('Missing username or upload_key.') + + organisation = None + if subdomain: + __validate_remote_organisation(subdomain, api) + role = __validate_astronomer_role_in_remote_organisation(subdomain, api) + organisation = {'subdomain': subdomain, 'role': role} + + telescope = None + if telescope_uuid_or_name: + telescope = __validate_telescope_uuid(telescope_uuid_or_name, subdomain, api) + + if subdomain and not telescope_uuid_or_name: + __print_organisation_telescopes(subdomain, api) + raise InvalidUploadOptionsOortCloudError('For an organisation, a telescope UUID (or alias) must be provided.') + + dataset = None + if dataset_uuid_or_name: + dataset = __validate_dataset_uuid(dataset_uuid_or_name, subdomain, api) + + identity = Identity(username, upload_key, organisation, telescope, dataset, zip, api) + + return identity diff --git a/oort/monitor/errors.py b/oort/monitor/errors.py index 3f30035c..2b2be4f4 100644 --- a/oort/monitor/errors.py +++ b/oort/monitor/errors.py @@ -10,6 +10,10 @@ class InvalidWatchOptionsOortCloudError(OortCloudError): def __init__(self, msg=''): super().__init__(f'Invalid or incomplete Watch options: {msg}') +class InvalidUploadOptionsOortCloudError(OortCloudError): + def __init__(self, msg=''): + super().__init__(f'Invalid or incomplete Upload options: {msg}') + class UnknownOrganisationOortCloudError(OortCloudError): def __init__(self, subdomain, error_string=''): @@ -67,3 +71,19 @@ def __init__(self, subdomain, username, upload_key, error_string=''): if error_string: msg += f'\n{error_string}' super().__init__(msg) + + +class InvalidDatasetOortCloudError(OortCloudError): + def __init__(self, dataset_uuid, error_string=''): + msg = f'Invalid / unknown dataset with UUID {dataset_uuid}.' + if error_string: + msg += f'\n{error_string}' + super().__init__(msg) + + +class InvalidOrganisationDatasetOortCloudError(OortCloudError): + def __init__(self, dataset_uuid, org_subdomain, error_string=''): + msg = f'Dataset with UUID {dataset_uuid} unknown within organisation {org_subdomain}.' + if error_string: + msg += f'\n{error_string}' + super().__init__(msg) diff --git a/oort/shared/identity.py b/oort/shared/identity.py index 6e6cd3e4..a7982698 100644 --- a/oort/shared/identity.py +++ b/oort/shared/identity.py @@ -1,48 +1,30 @@ import hashlib import os -import uuid from typing import Optional from oort.shared.config import write_oort_config_section_values class Identity(object): - @classmethod - def from_folder_section(cls, folder_section): - return cls(folder_section.get('username'), - folder_section.get('upload_key'), - folder_section.get('subdomain', ''), - folder_section.get('role', ''), - folder_section.get('telescope', ''), - folder_section.get('zip', 'False').lower() == 'true', - folder_section.get('dataset', ''), - folder_section.get('api', '')) - def __init__(self, username: str, upload_key: str, - subdomain: str = '', - role: str = '', - telescope_uuid: str = '', + organisation: Optional[dict] = None, + telescope: Optional[dict] = None, + dataset: Optional[dict] = None, zip: bool = False, - dataset: str = '', api: str = 'main'): assert username is not None assert upload_key is not None - assert subdomain is not None - assert role is not None assert upload_key is not None - assert telescope_uuid is not None - assert dataset is not None assert api is not None self._username = username self._upload_key = upload_key - self._subdomain = subdomain or '' - self._role = role or '' - self._telescope_uuid = telescope_uuid or '' + self._organisation = organisation or {} + self._telescope = telescope or {} + self._dataset = dataset or {} self._telescope_details = None self._zip = zip - self._dataset = dataset or '' self._api = api # In python3, this will do the __ne__ by inverting the value @@ -52,7 +34,7 @@ def __eq__(self, other: object) -> bool: return self.username == other.username and self.upload_key == other.upload_key and \ self.subdomain == other.subdomain and self.role == other.role and \ self.telescope_uuid == other.telescope_uuid and self.zip == other.zip \ - and self.dataset == other.dataset and self.api == other.api + and self.dataset_uuid == other.dataset_uuid and self.api == other.api @property def username(self) -> str: @@ -64,59 +46,40 @@ def upload_key(self) -> str: @property def subdomain(self) -> str: - return self._subdomain + return self._organisation.get('subdomain', '') @property def role(self) -> str: - return self._role + return self._organisation.get('role', '') @property def telescope_uuid(self) -> str: - return self._telescope_uuid + return self._telescope.get('uuid', '') @property - def telescope_details(self) -> Optional[dict]: - return self._telescope_details + def telescope_name(self) -> str: + return self._telescope.get('name', '') @property - def zip(self) -> bool: - return self._zip + def telescope_alias(self) -> str: + return self._telescope.get('alias', '') @property - def dataset(self) -> str: - return self._dataset + def dataset_uuid(self) -> str: + return self._dataset.get('uuid', '') @property - def has_dataset(self) -> bool: - return len(self._dataset) > 0 + def dataset_name(self) -> str: + return self._dataset.get('name', '') @property - def is_dataset_uuid(self) -> bool: - if self.has_dataset: - try: - uuid.UUID(str(self._dataset)) - return True - except ValueError: - return False - return False + def zip(self) -> bool: + return self._zip @property def api(self) -> str: return self._api - def attach_telescope_details(self, **details): - if 'uuid' not in details.keys() or \ - (self._telescope_uuid != '' and details.get('uuid') != self._telescope_uuid): - raise ValueError('Wrong telescope UUID') - if self._telescope_uuid == '': - self._telescope_uuid = details.get('uuid') - self._telescope_details = details - - def get_args_string(self): - s = f"{self.username},{self.upload_key},{self.subdomain},{self.role},{self.telescope_uuid}," - s += f"{str(self.zip)},{self.dataset},{str(self.api)}" - return s - def save_with_folder(self, upload_folder_path: str): # If data are on disk that are attached, then detached and re-attached to a different volume # the full upload_folder_path will change, thus the folder_hash, and a new folder will be watched... @@ -128,7 +91,30 @@ def save_with_folder(self, upload_folder_path: str): subdomain=self.subdomain, role=self.role, path=upload_folder_path, - telescope=self.telescope_uuid, + telescope_uuid=self.telescope_uuid, + telescope_name=self.telescope_name, + telescope_alias=self.telescope_alias, + dataset_uuid=self.dataset_uuid, + dataset_name=self.dataset_name, zip=str(self.zip), - dataset=self.dataset, api=self.api) + + @classmethod + def from_folder_section(cls, folder_section): + return cls(folder_section.get('username'), + folder_section.get('upload_key'), + { + 'subdomain': folder_section.get('subdomain', ''), + 'role': folder_section.get('role', '') + }, + { + 'uuid': folder_section.get('telescope_uuid', ''), + 'name': folder_section.get('telescope_name', ''), + 'alias': folder_section.get('telescope_alias', '') + }, + { + 'uuid': folder_section.get('dataset_uuid', ''), + 'name': folder_section.get('dataset_name', ''), + }, + folder_section.get('zip', 'False').lower() == 'true', + folder_section.get('api', '')) diff --git a/oort/uploader/engine/preparator.py b/oort/uploader/engine/preparator.py index 1a9cd7fa..84b3a5f8 100644 --- a/oort/uploader/engine/preparator.py +++ b/oort/uploader/engine/preparator.py @@ -1,4 +1,3 @@ -import os import socket from typing import Optional @@ -6,10 +5,11 @@ from arcsecond.api.error import ArcsecondRequestTimeoutError from peewee import DoesNotExist +from cli.helpers import build_endpoint_kwargs from oort import __version__ from oort.shared.config import get_oort_logger -from oort.shared.models import (Dataset, Organisation, Status, Substatus, Telescope) from oort.shared.identity import Identity +from oort.shared.models import (Dataset, Organisation, Status, Substatus, Telescope) from . import errors @@ -41,15 +41,7 @@ def __init__(self, pack, debug=False): @property def _api_kwargs(self) -> dict: - test = os.environ.get('OORT_TESTS') == '1' - kwargs = {'api': self._identity.api, 'upload_key': self._identity.upload_key, 'test': test} - if self._identity.subdomain is not None and len(self._identity.subdomain) > 0: - # We have an organisation subdomain. - # We are uploading for an organisation, using ORGANISATION APIs, - # If no upload_key or api_key is provided, it will be using the current - # logged-in astronomer credentials. - kwargs.update(organisation=self._identity.subdomain) - return kwargs + return build_endpoint_kwargs(self._identity.api, self._identity.subdomain) @property def log_prefix(self) -> str: @@ -164,7 +156,7 @@ def _sync_dataset(self): self._update_remote_resource(datasets_api, dataset_dict['uuid'], **create_kwargs) # Create local resource. But avoids pointing to (possibly) non-existing ForeignKeys for which - # we have only the uuid for now, not the local Database ID. + # we have only the uuid for now, and not the local Database ID. if 'observation' in dataset_dict.keys(): dataset_dict.pop('observation') if 'calibration' in dataset_dict.keys(): diff --git a/oort/uploader/engine/walker.py b/oort/uploader/engine/walker.py index 271553c5..1046bccf 100644 --- a/oort/uploader/engine/walker.py +++ b/oort/uploader/engine/walker.py @@ -3,7 +3,6 @@ import click -from oort.cli.folders import check_remote_organisation from oort.shared.config import get_oort_logger from oort.shared.identity import Identity from oort.shared.models import Status @@ -71,9 +70,6 @@ def walk_second_pass(root_path: Path, identity: Identity, unfinished_paths: list def walk(folder_string: str, identity: Identity, force: bool): - if identity.subdomain: - check_remote_organisation(identity.subdomain, identity.api) - log_prefix = '[Walker]' root_path = Path(folder_string).resolve() if root_path.is_file(): # Just in case we pass a file... diff --git a/setup.py b/setup.py index 45e84e85..e3393e94 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,9 @@ install_requires=[ 'arcsecond>=2.0.4', 'astropy>=5', - 'flask>=2.2', + 'flask>=2.3', 'peewee>=3', 'watchdog>=2.2', - 'supervisor>4.2.2', 'dateparser', 'python-dotenv' ], diff --git a/tests/cli/test_cli_folders_parse.py b/tests/cli/test_cli_folders_parse.py index 9f98d9d2..d6a3b82f 100644 --- a/tests/cli/test_cli_folders_parse.py +++ b/tests/cli/test_cli_folders_parse.py @@ -44,7 +44,6 @@ def test_cli_folders_no_options(): assert identity.upload_key == TEST_LOGIN_UPLOAD_KEY assert identity.subdomain == '' assert identity.role == '' - assert identity.telescope_details is None @use_test_database @@ -61,7 +60,6 @@ def test_cli_folders_no_org_but_with_valid_telescope(): assert identity.upload_key == TEST_LOGIN_UPLOAD_KEY assert identity.subdomain == '' assert identity.role == '' - assert identity.telescope_details == TEL_DETAILS @use_test_database diff --git a/tests/uploader/test_packer.py b/tests/uploader/test_packer.py index 44b85b17..11353fca 100644 --- a/tests/uploader/test_packer.py +++ b/tests/uploader/test_packer.py @@ -17,7 +17,7 @@ fixture_path = root_path / 'tests' / 'fixtures' telescope_uuid = '44f5bee9-a557-4264-86d6-c877d5013788' -identity = Identity('cedric', str(uuid.uuid4()), 'saao', 'admin', telescope_uuid, True) +identity = Identity('cedric', str(uuid.uuid4()), 'saao', 'admin', telescope_uuid, zip=True) @use_test_database diff --git a/tests/uploader/test_preparator.py b/tests/uploader/test_preparator.py index 569e21d7..71954f14 100644 --- a/tests/uploader/test_preparator.py +++ b/tests/uploader/test_preparator.py @@ -163,7 +163,7 @@ def test_preparator_prepare_with_org_telescope_and_new_dataset_name(): TEST_LOGIN_ORG_SUBDOMAIN, TEST_LOGIN_ORG_ROLE, TEL_UUID, - dataset=dataset_name, + dataset_uuid=dataset_name, api='test') pack = UploadPack(str(folder_path), str(fits_file_path), identity)