diff --git a/.gitignore b/.gitignore index 080d1e18958..8cdd39ffe09 100644 --- a/.gitignore +++ b/.gitignore @@ -153,9 +153,12 @@ prof/ # outputs from make .stack-*.yml - -# Copies +# copies services/**/.codeclimate.yml + # WSL .fake_hostname_file .bash_history + +# pytest-fixture-tools output +artifacts diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 2e9d98d15b7..43095d2dcaf 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -75,15 +75,13 @@ def set_logging_handler( formatting = DEFAULT_FORMATTING if not formatter_base: formatter_base = CustomFormatter - for handler in logger.handlers: - # handler = logging.StreamHandler() + for handler in logger.handlers: handler.setFormatter( formatter_base( "%(levelname)s: %(name)s:%(funcName)s(%(lineno)s) - %(message)s" ) ) - # logger.addHandler(handler) def _log_arguments( diff --git a/services/web/server/src/simcore_service_webserver/__init__.py b/services/web/server/src/simcore_service_webserver/__init__.py index 9226fe7e39e..0ea9612c61c 100644 --- a/services/web/server/src/simcore_service_webserver/__init__.py +++ b/services/web/server/src/simcore_service_webserver/__init__.py @@ -1 +1,12 @@ -from .__version__ import __version__ +import warnings + +from ._meta import __version__ + +# +# NOTE: Some BaseSettings are using aliases (e.g. version for vtag) to facility construct +# pydantic settings from names defined in trafaret schemas for the config files +# +warnings.filterwarnings( + "ignore", + message='aliases are no longer used by BaseSettings to define which environment variables to read. Instead use the "env" field setting. See https://pydantic-docs.helpmanual.io/usage/settings/#environment-variable-names', +) diff --git a/services/web/server/src/simcore_service_webserver/__version__.py b/services/web/server/src/simcore_service_webserver/_meta.py similarity index 56% rename from services/web/server/src/simcore_service_webserver/__version__.py rename to services/web/server/src/simcore_service_webserver/_meta.py index 1d6e7152e70..6161010540b 100644 --- a/services/web/server/src/simcore_service_webserver/__version__.py +++ b/services/web/server/src/simcore_service_webserver/_meta.py @@ -14,3 +14,15 @@ # legacy api_version_prefix: str = api_vtag + + +WELCOME_MSG = r""" + _ _ _ +| | | | | | +| | | | ___ | |__ ___ ___ _ __ __ __ ___ _ __ +| |/\| | / _ \| '_ \ / __| / _ \| '__|\ \ / // _ \| '__| +\ /\ /| __/| |_) |\__ \| __/| | \ V /| __/| | + \/ \/ \___||_.__/ |___/ \___||_| \_/ \___||_| {0} +""".format( + f"v{__version__}" +) diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 54296de098b..b29a7842465 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -9,6 +9,7 @@ from servicelib.application import create_safe_application +from ._meta import WELCOME_MSG from .activity import setup_activity from .catalog import setup_catalog from .computation import setup_computation @@ -67,14 +68,14 @@ def create_application(config: Dict) -> web.Application: setup_storage(app) setup_users(app) setup_groups(app) - setup_projects(app) # needs storage - setup_studies_access(app) + setup_projects(app) setup_activity(app) setup_resource_manager(app) setup_tags(app) setup_catalog(app) setup_publications(app) setup_products(app) + setup_studies_access(app) return app @@ -85,6 +86,11 @@ def run_service(config: dict): app = create_application(config) + async def welcome_banner(_app: web.Application): + print(WELCOME_MSG, flush=True) + + app.on_startup.append(welcome_banner) + web.run_app(app, host=config["main"]["host"], port=config["main"]["port"]) diff --git a/services/web/server/src/simcore_service_webserver/catalog.py b/services/web/server/src/simcore_service_webserver/catalog.py index 878a2a848ab..0b2acbff827 100644 --- a/services/web/server/src/simcore_service_webserver/catalog.py +++ b/services/web/server/src/simcore_service_webserver/catalog.py @@ -13,7 +13,7 @@ from servicelib.rest_responses import wrap_as_envelope from servicelib.rest_routing import iter_path_operations -from .__version__ import api_version_prefix +from ._meta import api_version_prefix from .catalog_config import assert_valid_config, get_client_session from .constants import RQ_PRODUCT_KEY, X_PRODUCT_NAME_HEADER from .login.decorators import RQT_USERID_KEY, login_required diff --git a/services/web/server/src/simcore_service_webserver/cli.py b/services/web/server/src/simcore_service_webserver/cli.py index 4c59ef5e513..18096480986 100644 --- a/services/web/server/src/simcore_service_webserver/cli.py +++ b/services/web/server/src/simcore_service_webserver/cli.py @@ -18,17 +18,12 @@ from argparse import ArgumentParser from typing import Dict, List, Optional -from aiodebug import log_slow_callbacks -from aiohttp.log import access_logger -from servicelib.logging_utils import set_logging_handler - from .application import run_service from .application_config import CLI_DEFAULT_CONFIGFILE, app_schema from .cli_config import add_cli_options, config_from_options +from .log import setup_logging from .utils import search_osparc_repo_dir -LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR - log = logging.getLogger(__name__) @@ -104,26 +99,7 @@ def main(args: Optional[List] = None): config = parse(args, parser) # service log level - log_level = getattr(logging, config["main"]["log_level"]) - logging.basicConfig(level=log_level) - logging.root.setLevel(log_level) - set_logging_handler(logging.root) - - # aiohttp access log-levels - access_logger.setLevel(log_level) - - # keep mostly quiet noisy loggers - quiet_level = max( - min(log_level + LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING - ) - logging.getLogger("engineio").setLevel(quiet_level) - logging.getLogger("openapi_spec_validator").setLevel(quiet_level) - logging.getLogger("sqlalchemy").setLevel(quiet_level) - logging.getLogger("sqlalchemy.engine").setLevel(quiet_level) - - # NOTE: Every task blocking > AIODEBUG_SLOW_DURATION_SECS secs is considered slow and logged as warning - slow_duration = float(os.environ.get("AIODEBUG_SLOW_DURATION_SECS", 0.1)) - log_slow_callbacks.enable(slow_duration) + setup_logging(level=config["main"]["log_level"]) # run run_service(config) diff --git a/services/web/server/src/simcore_service_webserver/log.py b/services/web/server/src/simcore_service_webserver/log.py new file mode 100644 index 00000000000..84b03b50ed2 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/log.py @@ -0,0 +1,45 @@ +""" Configuration and utilities for service logging + +""" +import logging +import os +from typing import Union + +from aiodebug import log_slow_callbacks +from aiohttp.log import access_logger + +from servicelib.logging_utils import set_logging_handler + +LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR + + +def setup_logging(*, level: Union[str, int]): + # service log level + logging.basicConfig(level=level) + + # root + logging.root.setLevel(level) + set_logging_handler(logging.root) + + # aiohttp access log-levels + access_logger.setLevel(level) + + # keep mostly quiet noisy loggers + quiet_level: int = max(min(logging.root.level + LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING) + logging.getLogger("engineio").setLevel(quiet_level) + logging.getLogger("openapi_spec_validator").setLevel(quiet_level) + logging.getLogger("sqlalchemy").setLevel(quiet_level) + logging.getLogger("sqlalchemy.engine").setLevel(quiet_level) + + # NOTE: Every task blocking > AIODEBUG_SLOW_DURATION_SECS secs is considered slow and logged as warning + slow_duration = float(os.environ.get("AIODEBUG_SLOW_DURATION_SECS", 0.1)) + log_slow_callbacks.enable(slow_duration) + + +def test_logger_propagation(logger: logging.Logger): + msg = f"TESTING %s log with {logger}" + logger.critical(msg, "critical") + logger.error(msg, "error") + logger.info(msg, "info") + logger.warning(msg, "warning") + logger.debug(msg, "debug") diff --git a/services/web/server/src/simcore_service_webserver/products.py b/services/web/server/src/simcore_service_webserver/products.py index 75c75027e39..8e0da2c48e9 100644 --- a/services/web/server/src/simcore_service_webserver/products.py +++ b/services/web/server/src/simcore_service_webserver/products.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, ValidationError, validator from servicelib.application_setup import ModuleCategory, app_module_setup -from .__version__ import api_vtag +from ._meta import api_vtag from .constants import ( APP_DB_ENGINE_KEY, APP_PRODUCTS_KEY, diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/config.py b/services/web/server/src/simcore_service_webserver/resource_manager/config.py index d20fc811fed..eb83b61ffa9 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/config.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/config.py @@ -7,7 +7,7 @@ import trafaret as T from aiohttp.web import Application -from pydantic import BaseSettings, PositiveInt +from pydantic import BaseSettings, PositiveInt, Field from models_library.settings.redis import RedisConfig from servicelib.application_keys import APP_CONFIG_KEY @@ -46,8 +46,12 @@ class RedisSection(RedisConfig): class ResourceManagerSettings(BaseSettings): enabled: bool = True - resource_deletion_timeout_seconds: Optional[PositiveInt] = 900 - garbage_collection_interval_seconds: Optional[PositiveInt] = 30 + resource_deletion_timeout_seconds: Optional[PositiveInt] = Field( + 900, description="Expiration time (or Time to live (TTL) in redis jargon) for a registered resource" + ) + garbage_collection_interval_seconds: Optional[PositiveInt] = Field( + 30, description="Waiting time between consecutive runs of the garbage-colector" + ) redis: RedisSection diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py index 8caf4d0a45a..e0000b54eb2 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/garbage_collector.py @@ -63,10 +63,9 @@ async def garbage_collector_task(app: web.Application): while keep_alive: logger.info("Starting garbage collector...") try: - registry = get_registry(app) interval = get_garbage_collector_interval(app) while True: - await collect_garbage(registry, app) + await collect_garbage(app) await asyncio.sleep(interval) except asyncio.CancelledError: @@ -81,29 +80,30 @@ async def garbage_collector_task(app: web.Application): await asyncio.sleep(5) -async def collect_garbage(registry: RedisResourceRegistry, app: web.Application): +async def collect_garbage(app: web.Application): """ - Garbage collection has the task of removing trash from the system. The trash + Garbage collection has the task of removing trash (i.e. unused resources) from the system. The trash can be divided in: - Websockets & Redis (used to keep track of current active connections) - GUEST users (used for temporary access to the system which are created on the fly) - - deletion of users. If a user needs to be deleted it is manually marked as GUEST - in the database + - Deletion of users. If a user needs to be deleted it can be set as GUEST in the database The resources are Redis entries where all information regarding all the - websocket identifiers for all opened tabs accross all broser for each user + websocket identifiers for all opened tabs accross all browser for each user are stored. - The alive/dead keys are normal Redis keys. To each key and ALIVE key is associated, - which has an assigned TTL. The browser will call the `client_heartbeat` websocket + The alive/dead keys are normal Redis keys. To each key an ALIVE key is associated, + which has an assigned TTL (Time To Live). The browser will call the `client_heartbeat` websocket endpoint to refresh the TTL, thus declaring that the user (websocket connection) is - still active. The `resource_deletion_timeout_seconds` is theTTL of the key. + still active. The `resource_deletion_timeout_seconds` is the TTL of the key. The field `garbage_collection_interval_seconds` defines the interval at which this function will be called. """ - logger.info("collecting garbage...") + logger.info("Collecting garbage...") + + registry: RedisResourceRegistry = get_registry(app) # Removes disconnected user resources # Triggers signal to close possible pending opened projects diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/redis.py b/services/web/server/src/simcore_service_webserver/resource_manager/redis.py index bbc915ebe02..3e22859d77d 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/redis.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/redis.py @@ -33,7 +33,7 @@ async def redis_client(app: web.Application): client = None for attempt in Retrying(**retry_upon_init_policy): with attempt: - client = await aioredis.create_redis_pool(url, encoding="utf-8") + client: aioredis.Redis = await aioredis.create_redis_pool(url, encoding="utf-8") # create lock manager lock_manager = Aioredlock([url]) diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py index 2dbbb9b567a..8edc13d7e7c 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/registry.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/registry.py @@ -16,6 +16,7 @@ import logging from typing import Dict, List, Tuple +import aioredis import attr from aiohttp import web @@ -53,60 +54,60 @@ def _decode_hash_key(cls, hash_key: str) -> Dict[str, str]: key = dict(x.split("=") for x in tmp_key.split(":")) return key + @property + def client(self) -> aioredis.Redis: + return get_redis_client(self.app) + async def set_resource( self, key: Dict[str, str], resource: Tuple[str, str] ) -> None: - client = get_redis_client(self.app) hash_key = f"{self._hash_key(key)}:{RESOURCE_SUFFIX}" - await client.hmset_dict(hash_key, **{resource[0]: resource[1]}) + field, value = resource + await self.client.hmset_dict(hash_key, **{field: value}) async def get_resources(self, key: Dict[str, str]) -> Dict[str, str]: - client = get_redis_client(self.app) hash_key = f"{self._hash_key(key)}:{RESOURCE_SUFFIX}" - return await client.hgetall(hash_key) + return await self.client.hgetall(hash_key) async def remove_resource(self, key: Dict[str, str], resource_name: str) -> None: - client = get_redis_client(self.app) hash_key = f"{self._hash_key(key)}:{RESOURCE_SUFFIX}" - await client.hdel(hash_key, resource_name) + await self.client.hdel(hash_key, resource_name) async def find_resources( self, key: Dict[str, str], resource_name: str ) -> List[str]: - client = get_redis_client(self.app) resources = [] # the key might only be partialy complete partial_hash_key = f"{self._hash_key(key)}:{RESOURCE_SUFFIX}" - async for key in client.iscan(match=partial_hash_key): - if await client.hexists(key, resource_name): - resources.append(await client.hget(key, resource_name)) + async for key in self.client.iscan(match=partial_hash_key): + if await self.client.hexists(key, resource_name): + resources.append(await self.client.hget(key, resource_name)) return resources async def find_keys(self, resource: Tuple[str, str]) -> List[Dict[str, str]]: keys = [] if not resource: return keys - client = get_redis_client(self.app) - async for hash_key in client.iscan(match=f"*:{RESOURCE_SUFFIX}"): - if resource[1] == await client.hget(hash_key, resource[0]): + + field, value = resource + + async for hash_key in self.client.iscan(match=f"*:{RESOURCE_SUFFIX}"): + if value == await self.client.hget(hash_key, field): keys.append(self._decode_hash_key(hash_key)) return keys async def set_key_alive(self, key: Dict[str, str], timeout: int) -> None: # setting the timeout to always expire, timeout > 0 timeout = int(max(1, timeout)) - client = get_redis_client(self.app) hash_key = f"{self._hash_key(key)}:{ALIVE_SUFFIX}" - await client.set(hash_key, 1, expire=timeout) + await self.client.set(hash_key, 1, expire=timeout) async def is_key_alive(self, key: Dict[str, str]) -> bool: - client = get_redis_client(self.app) hash_key = f"{self._hash_key(key)}:{ALIVE_SUFFIX}" - return await client.exists(hash_key) > 0 + return await self.client.exists(hash_key) > 0 async def remove_key(self, key: Dict[str, str]) -> None: - client = get_redis_client(self.app) - await client.delete( + await self.client.delete( f"{self._hash_key(key)}:{RESOURCE_SUFFIX}", f"{self._hash_key(key)}:{ALIVE_SUFFIX}", ) @@ -114,14 +115,13 @@ async def remove_key(self, key: Dict[str, str]) -> None: async def get_all_resource_keys( self, ) -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]: - client = get_redis_client(self.app) alive_keys = [ self._decode_hash_key(hash_key) - async for hash_key in client.iscan(match=f"*:{ALIVE_SUFFIX}") + async for hash_key in self.client.iscan(match=f"*:{ALIVE_SUFFIX}") ] dead_keys = [ self._decode_hash_key(hash_key) - async for hash_key in client.iscan(match=f"*:{RESOURCE_SUFFIX}") + async for hash_key in self.client.iscan(match=f"*:{RESOURCE_SUFFIX}") if self._decode_hash_key(hash_key) not in alive_keys ] diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/websocket_manager.py b/services/web/server/src/simcore_service_webserver/resource_manager/websocket_manager.py index a0c84478c58..7b588d0258e 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/websocket_manager.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/websocket_manager.py @@ -45,7 +45,7 @@ def _resource_key(self) -> Dict[str, str]: else "*", } - async def set_socket_id(self, socket_id: str) -> None: + async def set_socket_id(self, socket_id: str, *, extra_tll: int = 0) -> None: log.debug( "user %s/tab %s adding socket %s in registry...", self.user_id, @@ -54,9 +54,9 @@ async def set_socket_id(self, socket_id: str) -> None: ) registry = get_registry(self.app) await registry.set_resource(self._resource_key(), (SOCKET_ID_KEY, socket_id)) - # hearthbeat is not emulated in tests, make sure that with very small GC intervals - # the resources do not result as timeout; this value is usually in the order of minutes - timeout = max(3, get_service_deletion_timeout(self.app)) + # NOTE: hearthbeat is not emulated in tests, make sure that with very small GC intervals + # the resources do not expire; this value is usually in the order of minutes + timeout = max(3, get_service_deletion_timeout(self.app)) + abs(extra_tll) await registry.set_key_alive(self._resource_key(), timeout) async def get_socket_id(self) -> Optional[str]: diff --git a/services/web/server/src/simcore_service_webserver/rest.py b/services/web/server/src/simcore_service_webserver/rest.py index 93ea29015a6..c6859d49ab2 100644 --- a/services/web/server/src/simcore_service_webserver/rest.py +++ b/services/web/server/src/simcore_service_webserver/rest.py @@ -24,7 +24,7 @@ from simcore_service_webserver.resources import resources from . import rest_routes -from .__version__ import api_version_prefix +from ._meta import api_version_prefix from .rest_config import APP_OPENAPI_SPECS_KEY, assert_valid_config from .diagnostics_config import get_diagnostics_config diff --git a/services/web/server/src/simcore_service_webserver/rest_config.py b/services/web/server/src/simcore_service_webserver/rest_config.py index 46e35b2b616..dbd29df6b9d 100644 --- a/services/web/server/src/simcore_service_webserver/rest_config.py +++ b/services/web/server/src/simcore_service_webserver/rest_config.py @@ -5,7 +5,7 @@ """ from typing import Dict, Optional -from .__version__ import api_vtag +from ._meta import api_vtag import trafaret as T from aiohttp import web diff --git a/services/web/server/src/simcore_service_webserver/settings.py b/services/web/server/src/simcore_service_webserver/settings.py index 0893cada82c..a2621967462 100644 --- a/services/web/server/src/simcore_service_webserver/settings.py +++ b/services/web/server/src/simcore_service_webserver/settings.py @@ -8,7 +8,7 @@ from aiohttp import web from pydantic import BaseSettings, Field -from .__version__ import api_version, app_name +from ._meta import api_version, app_name from .constants import APP_SETTINGS_KEY from .utils import snake_to_camel diff --git a/services/web/server/src/simcore_service_webserver/studies_access.py b/services/web/server/src/simcore_service_webserver/studies_access.py index 3b1083239b4..8ea7111f973 100644 --- a/services/web/server/src/simcore_service_webserver/studies_access.py +++ b/services/web/server/src/simcore_service_webserver/studies_access.py @@ -32,10 +32,10 @@ @lru_cache() def compose_uuid(template_uuid, user_id, query="") -> str: - """ Creates a new uuid composing a project's and user ids such that - any template pre-assigned to a user + """Creates a new uuid composing a project's and user ids such that + any template pre-assigned to a user - Enforces a constraint: a user CANNOT have multiple copies of the same template + Enforces a constraint: a user CANNOT have multiple copies of the same template """ new_uuid = str( uuid.uuid5(BASE_UUID, str(template_uuid) + str(user_id) + str(query)) @@ -46,7 +46,7 @@ def compose_uuid(template_uuid, user_id, query="") -> str: # TODO: from .projects import get_public_project async def get_public_project(app: web.Application, project_uuid: str): """ - Returns project if project_uuid is a template and is marked as published, otherwise None + Returns project if project_uuid is a template and is marked as published, otherwise None """ from .projects.projects_db import APP_PROJECT_DBAPI @@ -55,34 +55,30 @@ async def get_public_project(app: web.Application, project_uuid: str): return prj -# TODO: from .users import create_temporary_user async def create_temporary_user(request: web.Request): """ - TODO: user should have an expiration date and limited persmissions! + TODO: user should have an expiration date and limited persmissions! """ from .login.cfg import get_storage - from .login.handlers import ACTIVE, GUEST + from .login.handlers import ACTIVE, ANONYMOUS from .login.utils import get_client_ip, get_random_string from .security_api import encrypt_password - # from .utils import generate_passphrase - # from .utils import generate_password - db = get_storage(request.app) # TODO: avatar is an icon of the hero! - # FIXME: # username = generate_passphrase(number_of_words=2).replace(" ", "_").replace("'", "") username = get_random_string(min_len=5) email = username + "@guest-at-osparc.io" password = get_random_string(min_len=12) + # creates a user that is marked as ANONYMOUS. see update_user_as_guest user = await db.create_user( { "name": username, "email": email, "password_hash": encrypt_password(password), "status": ACTIVE, - "role": GUEST, + "role": ANONYMOUS, "created_ip": get_client_ip(request), } ) @@ -90,6 +86,33 @@ async def create_temporary_user(request: web.Request): return user +async def activate_as_guest_user(user: Dict, app: web.Application): + from .login.cfg import get_storage + from .login.handlers import GUEST, ANONYMOUS + from .resource_manager.websocket_manager import WebsocketRegistry + + # Save time to allow user to be redirected back and not being deleted by GC + # SEE https://github.com/ITISFoundation/osparc-simcore/pull/1928#discussion_r517176479 + EXTRA_TIME_TO_COMPLETE_REDIRECT = 3 + + if user.get("role") == ANONYMOUS: + db = get_storage(app) + username = user["name"] + + # creates an entry in the socket's registry to avoid garbage collector (GC) deleting it + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1853 + # + registry = WebsocketRegistry(user["id"], f"{username}-guest-session", app) + await registry.set_socket_id( + f"{username}-guest-socket-id", extra_tll=EXTRA_TIME_TO_COMPLETE_REDIRECT + ) + + # Now that we know the ID, we set the user as GUEST + # This extra step is to prevent the possibility that the GC is cleaning while + # the function above is creating the entry in the socket + await db.update_user(user, updates={"role": GUEST}) + + # TODO: from .users import get_user? async def get_authorized_user(request: web.Request) -> Dict: from .login.cfg import get_storage @@ -106,10 +129,10 @@ async def copy_study_to_account( request: web.Request, template_project: Dict, user: Dict ): """ - Creates a copy of the study to a given project in user's account + Creates a copy of the study to a given project in user's account - - Replaces template parameters by values passed in query - - Avoids multiple copies of the same template on each account + - Replaces template parameters by values passed in query + - Avoids multiple copies of the same template on each account """ from .projects.projects_db import APP_PROJECT_DBAPI from .projects.projects_exceptions import ProjectNotFoundError @@ -156,11 +179,11 @@ async def copy_study_to_account( # HANDLERS -------------------------------------------------------- async def access_study(request: web.Request) -> web.Response: """ - Handles requests to get and open a public study + Handles requests to get and open a public study - - public studies are templates that are marked as published in the database - - if user is not registered, it creates a temporary guest account with limited resources and expiration - - this handler is NOT part of the API and therefore does NOT respond with json + - public studies are templates that are marked as published in the database + - if user is not registered, it creates a temporary guest account with limited resources and expiration + - this handler is NOT part of the API and therefore does NOT respond with json """ # TODO: implement nice error-page.html project_id = request.match_info["id"] @@ -224,6 +247,7 @@ async def access_study(request: web.Request) -> web.Response: log.debug("Auto login for anonymous user %s", user["name"]) identity = user["email"] await remember(request, response, identity) + await activate_as_guest_user(user, request.app) raise response @@ -243,7 +267,9 @@ def setup(app: web.Application): # TODO: make sure that these routes are filtered properly in active middlewares app.router.add_routes( - [web.get(r"/study/{id}", study_handler, name="study"),] + [ + web.get(r"/study/{id}", study_handler, name="study"), + ] ) return True diff --git a/services/web/server/tests/sandbox/app.py b/services/web/server/tests/sandbox/app.py index 7d3468265e1..99903356c5d 100644 --- a/services/web/server/tests/sandbox/app.py +++ b/services/web/server/tests/sandbox/app.py @@ -7,7 +7,7 @@ from aiohttp.web import middleware from multidict import MultiDict -from simcore_service_webserver.__version__ import api_vtag +from simcore_service_webserver._meta import api_vtag # FRONT_END #################################### frontend_folder = Path.home() / "devp/osparc-simcore/services/web/client" diff --git a/services/web/server/tests/unit/isolated/test_catalog_setup.py b/services/web/server/tests/unit/isolated/test_catalog_setup.py index 3e219c93c0d..3e9eec63a8e 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_setup.py +++ b/services/web/server/tests/unit/isolated/test_catalog_setup.py @@ -9,7 +9,7 @@ from servicelib.application import create_safe_application from servicelib.client_session import APP_CLIENT_SESSION_KEY -from simcore_service_webserver.__version__ import api_version_prefix +from simcore_service_webserver._meta import api_version_prefix from simcore_service_webserver.application_config import load_default_config from simcore_service_webserver.catalog import ( is_service_responsive, diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 384fe9e30cd..6d085985ae7 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -117,19 +117,23 @@ class _BaseSettingEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, BaseSettings): return o.json() + elif isinstance(o, Path): + return str(o) + # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, o) @pytest.fixture def web_server(loop, aiohttp_server, app_cfg, monkeypatch, postgres_db): - # original APP - app = create_application(app_cfg) - print( "Inits webserver with config", - json.dumps(app[APP_CONFIG_KEY], indent=2, cls=_BaseSettingEncoder), + json.dumps(app_cfg, indent=2, cls=_BaseSettingEncoder), ) + # original APP + app = create_application(app_cfg) + + assert app[APP_CONFIG_KEY] == app_cfg # with patched email _path_mail(monkeypatch) @@ -297,9 +301,7 @@ def postgres_service(docker_services, postgres_dsn): @pytest.fixture -def postgres_db( - app_cfg: Dict, postgres_dsn: Dict, postgres_service: str -) -> sa.engine.Engine: +def postgres_db( postgres_dsn: Dict, postgres_service: str ) -> sa.engine.Engine: url = postgres_service # Configures db and initializes tables diff --git a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py index c260c658e48..5c682fe62d0 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py @@ -4,6 +4,9 @@ # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name + + +import logging import re import textwrap from copy import deepcopy @@ -13,6 +16,7 @@ import pytest from aiohttp import ClientResponse, ClientSession, web + from models_library.projects import ( Owner, ProjectLocked, @@ -24,21 +28,12 @@ from pytest_simcore.helpers.utils_login import LoggedUser, UserRole from pytest_simcore.helpers.utils_mock import future_with_result from pytest_simcore.helpers.utils_projects import NewProject, delete_all_projects -from servicelib.application import create_safe_application from servicelib.rest_responses import unwrap_envelope from simcore_service_webserver import catalog -from simcore_service_webserver.db import setup_db -from simcore_service_webserver.login import setup_login -from simcore_service_webserver.products import setup_products -from simcore_service_webserver.projects import setup_projects +from simcore_service_webserver.log import setup_logging from simcore_service_webserver.projects.projects_api import delete_project_from_db -from simcore_service_webserver.rest import setup_rest -from simcore_service_webserver.security import setup_security -from simcore_service_webserver.session import setup_session -from simcore_service_webserver.settings import setup_settings -from simcore_service_webserver.statics import STATIC_DIRNAMES, setup_statics -from simcore_service_webserver.studies_access import setup_studies_access -from simcore_service_webserver.users import setup_users +from simcore_service_webserver.resource_manager.garbage_collector import collect_garbage +from simcore_service_webserver.statics import STATIC_DIRNAMES from simcore_service_webserver.users_api import delete_user, is_user_guest SHARED_STUDY_UUID = "e2e38eee-c569-4e55-b104-70d159e49c87" @@ -74,54 +69,56 @@ def qx_client_outdir(tmpdir): @pytest.fixture -def mocks_on_projects_api(mocker) -> Dict: - """ - All projects in this module are UNLOCKED - """ - state = ProjectState( - locked=ProjectLocked( - value=False, owner=Owner(first_name="Speedy", last_name="Gonzalez") - ), - state=ProjectRunningState(value=RunningState.NOT_STARTED), - ).dict(by_alias=True, exclude_unset=True) - mocker.patch( - "simcore_service_webserver.projects.projects_api.get_project_state_for_user", - return_value=future_with_result(state), - ) +def app_cfg(default_app_cfg, aiohttp_unused_port, qx_client_outdir, redis_service): + """App's configuration used for every test in this module - -@pytest.fixture -def client( - loop, aiohttp_client, app_cfg, postgres_db, qx_client_outdir, mocks_on_projects_api -): - cfg = deepcopy(app_cfg) - - cfg["projects"]["enabled"] = True - cfg["storage"]["enabled"] = False - cfg["computation"]["enabled"] = False - cfg["main"]["client_outdir"] = qx_client_outdir - - app = create_safe_application(cfg) - - setup_settings(app) - setup_statics(app) - setup_db(app) - setup_session(app) - setup_security(app) - setup_rest(app) # TODO: why should we need this?? - setup_login(app) - setup_users(app) - setup_products(app) - assert setup_projects(app), "Shall not skip this setup" - assert setup_studies_access(app), "Shall not skip this setup" - - # server and client - yield loop.run_until_complete( - aiohttp_client( - app, - server_kwargs={"port": cfg["main"]["port"], "host": cfg["main"]["host"]}, - ) - ) + NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup + """ + cfg = deepcopy(default_app_cfg) + + cfg["main"]["port"] = aiohttp_unused_port() + cfg["main"]["client_outdir"] = str(qx_client_outdir) + cfg["main"]["studies_access_enabled"] = True + + exclude = { + "tracing", + "director", + "smtp", + "storage", + "activity", + "diagnostics", + "groups", + "tags", + "publications", + "catalog", + "computation", + } + include = { + "db", + "rest", + "projects", + "login", + "socketio", + "resource_manager", + "users", + "studies_access", + "products", + } + + assert include.intersection(exclude) == set() + + for section in include: + cfg[section]["enabled"] = True + for section in exclude: + cfg[section]["enabled"] = False + + # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG + setup_logging(level=logging.DEBUG) + + # Enforces smallest GC in the background task + cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 + + return cfg @pytest.fixture @@ -222,7 +219,41 @@ async def assert_redirected_to_study( return redirected_project_id -# TESTS -------------------------------------- +@pytest.fixture +async def catalog_subsystem_mock(monkeypatch, published_project): + services_in_project = [ + {"key": s["key"], "version": s["version"]} + for _, s in published_project["workbench"].items() + ] + + async def mocked_get_services_for_user(*args, **kwargs): + return services_in_project + + monkeypatch.setattr( + catalog, "get_services_for_user_in_product", mocked_get_services_for_user + ) + + +@pytest.fixture +def mocks_on_projects_api(mocker) -> Dict: + """ + All projects in this module are UNLOCKED + """ + state = ProjectState( + locked=ProjectLocked( + value=False, owner=Owner(first_name="Speedy", last_name="Gonzalez") + ), + state=ProjectRunningState(value=RunningState.NOT_STARTED), + ).dict(by_alias=True, exclude_unset=True) + mocker.patch( + "simcore_service_webserver.projects.projects_api.get_project_state_for_user", + return_value=future_with_result(state), + ) + + +# TESTS ---------------------------------------------------------------------------------------------- + + async def test_access_to_invalid_study(client, published_project): resp = await client.get("/study/SOME_INVALID_UUID") content = await resp.text() @@ -243,27 +274,12 @@ async def test_access_to_forbidden_study(client, unpublished_project): ), f"STANDARD studies are NOT sharable: {content}" -@pytest.fixture -async def catalog_subsystem_mock(monkeypatch, published_project): - services_in_project = [ - {"key": s["key"], "version": s["version"]} - for _, s in published_project["workbench"].items() - ] - - async def mocked_get_services_for_user(*args, **kwargs): - return services_in_project - - monkeypatch.setattr( - catalog, "get_services_for_user_in_product", mocked_get_services_for_user - ) - - async def test_access_study_anonymously( client, - qx_client_outdir, published_project, storage_subsystem_mock, catalog_subsystem_mock, + mocks_on_projects_api, ): study_url = client.app.router["study"].url_for(id=published_project["uuid"]) @@ -295,10 +311,10 @@ async def test_access_study_anonymously( async def test_access_study_by_logged_user( client, logged_user, - qx_client_outdir, published_project, storage_subsystem_mock, catalog_subsystem_mock, + mocks_on_projects_api, ): study_url = client.app.router["study"].url_for(id=published_project["uuid"]) resp = await client.get(study_url) @@ -318,10 +334,10 @@ async def test_access_study_by_logged_user( async def test_access_cookie_of_expired_user( client, - qx_client_outdir, published_project, storage_subsystem_mock, catalog_subsystem_mock, + mocks_on_projects_api, ): # emulates issue #1570 app: web.Application = client.app @@ -340,7 +356,7 @@ async def test_access_cookie_of_expired_user( async def garbage_collect_guest(uid): # Emulates garbage collector: - # - anonymous user expired, cleaning it up + # - GUEST user expired, cleaning it up # - client still holds cookie with its identifier nonetheless # assert await is_user_guest(app, uid) @@ -371,3 +387,43 @@ async def garbage_collect_guest(uid): # But I am another user assert data["id"] != user_id assert data["login"] != user_email + + +async def test_guest_user_is_not_garbage_collected( + client, + published_project, + storage_subsystem_mock, + catalog_subsystem_mock, + mocks_on_projects_api, +): + ## NOTE: use pytest -s --log-cli-level=DEBUG to see GC logs + # + + study_url = client.app.router["study"].url_for(id=published_project["uuid"]) + + # clicks link to study + await collect_garbage(client.app) # <<-- + resp = await client.get(study_url) + + expected_prj_id = await assert_redirected_to_study(resp, client.session) + + # has auto logged in as guest? + await collect_garbage(client.app) # <<-- + me_url = client.app.router["get_my_profile"].url_for() + resp = await client.get(me_url) + + data, _ = await assert_status(resp, web.HTTPOk) + assert data["login"].endswith("guest-at-osparc.io") + assert data["gravatar_id"] + assert data["role"].upper() == UserRole.GUEST.name + + # guest user only a copy of the template project + await collect_garbage(client.app) # <<-- + projects = await _get_user_projects(client) + assert len(projects) == 1 + guest_project = projects[0] + + assert expected_prj_id == guest_project["uuid"] + _assert_same_projects(guest_project, published_project) + + assert guest_project["prjOwner"] == data["login"]