From b727ea55838d71575db96e987b536a0bac9f6a7a Mon Sep 17 00:00:00 2001 From: Wouter Claeys Date: Fri, 11 Jan 2019 10:15:13 +0100 Subject: [PATCH] Dev 0.8.0 (#99) * Update Etag tween to only ETag the crabpy_pyramid routes by default. (#98) * Allow setting a custom wsdl for CRABpy. * Include capakey by default (#93) Closes-Issue: #46 * Update Etag tween to only ETag the crabpy_pyramid routes by default. Etag tween can be disabled entirely by setting `etag_tween_disabled` to True in the settings .ini - Default: False. When the tween is enabled, extra routes can be set to get etagged by this tween by adding the route names to the `generate_etag_for.list` setting in the .ini Programmatically, route names could also by added by adding them to the `crabpy_pyramid.GENERATE_ETAG_ROUTE_NAMES` list. Closes-Issue: #97 * Update setup.py * Update CHANGES.rst --- CHANGES.rst | 5 + crabpy_pyramid/__init__.py | 57 ++++++++---- crabpy_pyramid/routes/capakey.py | 67 ++++++------- crabpy_pyramid/routes/crab.py | 119 ++++++++++++++++++------ crabpy_pyramid/tests/test_crab.py | 8 +- crabpy_pyramid/tests/test_functional.py | 12 ++- setup.py | 2 +- 7 files changed, 187 insertions(+), 83 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4a1206d..a2f4c2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +0.8.0 (11-01-2019) +------------------ + +- Caching tween volledig herwerken (#97) + 0.7.0 (2019-01-03) ------------------ diff --git a/crabpy_pyramid/__init__.py b/crabpy_pyramid/__init__.py index 169faaf..30fd211 100644 --- a/crabpy_pyramid/__init__.py +++ b/crabpy_pyramid/__init__.py @@ -21,6 +21,7 @@ ) log = logging.getLogger(__name__) +GENERATE_ETAG_ROUTE_NAMES = set() class ICapakey(Interface): @@ -63,6 +64,13 @@ def _parse_settings(settings): if cache_config: args[short_key_name] = cache_config + # crab wsdl settings + for short_key_name in ('crab.wsdl', ): + key_name = "crabpy.%s" % short_key_name + if key_name in settings: + args[short_key_name] = settings.get(key_name) + + log.debug(settings) log.debug(args) return args @@ -111,7 +119,7 @@ def _build_crab(registry, settings): gateway = CrabGateway(factory, cache_config=cache_config) registry.registerUtility(gateway, ICrab) - return registry.queryUtility(ICapakey) + return registry.queryUtility(ICrab) def get_capakey(registry): @@ -158,20 +166,32 @@ def _get_proxy_settings(settings): return base_settings +def add_route(config, name, pattern, *args, **kwargs): + """ + Adds a pyramid route to the config. All args and kwargs will be + passed on to config.add_route. + + This exists so the default behaviour of including crabpy will still be to + cache all crabpy routes. + """ + config.add_route(name, pattern, *args, **kwargs) + GENERATE_ETAG_ROUTE_NAMES.add(name) + + def conditional_http_tween_factory(handler, registry): """ Tween that adds ETag headers and tells Pyramid to enable conditional responses where appropriate. """ settings = registry.settings if hasattr(registry, 'settings') else {} - not_cacheble_list = [] - if 'not.cachable.list' in settings: - not_cacheble_list = settings.get('not.cachable.list').split() + if 'generate_etag_for.list' in settings: + route_names = settings.get('generate_etag_for.list').split() + GENERATE_ETAG_ROUTE_NAMES.update(route_names) def conditional_http_tween(request): response = handler(request) - if request.path not in not_cacheble_list: + if request.matched_route.name in GENERATE_ETAG_ROUTE_NAMES: # If the Last-Modified header has been set, we want to enable the # conditional response processing. @@ -204,24 +224,27 @@ def includeme(config): base_settings = _get_proxy_settings(settings) # http caching tween - config.add_tween('crabpy_pyramid.conditional_http_tween_factory') + if not settings.get('etag_tween_disabled', False): + config.add_tween('crabpy_pyramid.conditional_http_tween_factory') # create cache root = settings.get('cache.file.root', '/tmp/dogpile_data') if not os.path.exists(root): os.makedirs(root) - capakey_settings = dict(_filter_settings(settings, 'capakey.'), **base_settings) - if capakey_settings['include']: - log.info('Adding CAPAKEY Gateway.') - del capakey_settings['include'] - config.add_renderer('capakey_listjson', capakey_json_list_renderer) - config.add_renderer('capakey_itemjson', capakey_json_item_renderer) - _build_capakey(config.registry, capakey_settings) - config.add_request_method(get_capakey, 'capakey_gateway') - config.add_directive('get_capakey', get_capakey) - config.include('crabpy_pyramid.routes.capakey') - config.scan('crabpy_pyramid.views.capakey') + capakey_settings = dict(_filter_settings(settings, 'capakey.'), + **base_settings) + if 'include' in capakey_settings: + log.info("The 'capakey.include' setting is deprecated. Capakey will " + "always be included.") + log.info('Adding CAPAKEY Gateway.') + config.add_renderer('capakey_listjson', capakey_json_list_renderer) + config.add_renderer('capakey_itemjson', capakey_json_item_renderer) + _build_capakey(config.registry, capakey_settings) + config.add_request_method(get_capakey, 'capakey_gateway') + config.add_directive('get_capakey', get_capakey) + config.include('crabpy_pyramid.routes.capakey') + config.scan('crabpy_pyramid.views.capakey') crab_settings = dict(_filter_settings(settings, 'crab.'), **base_settings) if crab_settings['include']: diff --git a/crabpy_pyramid/routes/capakey.py b/crabpy_pyramid/routes/capakey.py index 4bf43d1..0fcfe85 100644 --- a/crabpy_pyramid/routes/capakey.py +++ b/crabpy_pyramid/routes/capakey.py @@ -4,41 +4,44 @@ .. versionadded:: 0.1.0 """ +import crabpy_pyramid def includeme(config): - config.add_route('list_gemeenten', '/capakey/gemeenten') - config.add_route('get_gemeente', '/capakey/gemeenten/{gemeente_id}') - config.add_route( - 'list_kadastrale_afdelingen_by_gemeente', - '/capakey/gemeenten/{gemeente_id}/afdelingen' - ) - config.add_route('list_kadastrale_afdelingen', '/capakey/afdelingen') - config.add_route( - 'get_kadastrale_afdeling_by_id', - '/capakey/afdelingen/{afdeling_id}' - ) - config.add_route( - 'list_secties_by_afdeling', - '/capakey/afdelingen/{afdeling_id}/secties' - ) - config.add_route( + crabpy_pyramid.add_route(config, + 'list_gemeenten', + '/capakey/gemeenten') + crabpy_pyramid.add_route(config, + 'get_gemeente', + '/capakey/gemeenten/{gemeente_id}') + crabpy_pyramid.add_route(config, + 'list_kadastrale_afdelingen_by_gemeente', + '/capakey/gemeenten/{gemeente_id}/afdelingen') + crabpy_pyramid.add_route(config, + 'list_kadastrale_afdelingen', + '/capakey/afdelingen') + crabpy_pyramid.add_route(config, + 'get_kadastrale_afdeling_by_id', + '/capakey/afdelingen/{afdeling_id}') + crabpy_pyramid.add_route(config, + 'list_secties_by_afdeling', + '/capakey/afdelingen/{afdeling_id}/secties') + crabpy_pyramid.add_route( + config, 'get_sectie_by_id_and_afdeling', - '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}' - ) - config.add_route( + '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}') + crabpy_pyramid.add_route( + config, 'list_percelen_by_sectie', - '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}/percelen' - ) - config.add_route( + '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}/percelen') + crabpy_pyramid.add_route( + config, 'get_perceel_by_sectie_and_id', - '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}/percelen/{perceel_id1}/{perceel_id2}' - ) - config.add_route( - 'get_perceel_by_capakey', - '/capakey/percelen/{capakey1}/{capakey2}' - ) - config.add_route( - 'get_perceel_by_percid', - '/capakey/percelen/{percid}' - ) + '/capakey/afdelingen/{afdeling_id}/secties/{sectie_id}' + '/percelen/{perceel_id1}/{perceel_id2}') + crabpy_pyramid.add_route(config, + 'get_perceel_by_capakey', + '/capakey/percelen/{capakey1}/{capakey2}') + crabpy_pyramid.add_route(config, + 'get_perceel_by_percid', + '/capakey/percelen/{percid}') diff --git a/crabpy_pyramid/routes/crab.py b/crabpy_pyramid/routes/crab.py index cc437f3..ca793f9 100644 --- a/crabpy_pyramid/routes/crab.py +++ b/crabpy_pyramid/routes/crab.py @@ -4,35 +4,96 @@ .. versionadded:: 0.1.0 """ +import crabpy_pyramid def includeme(config): - config.add_route('list_gewesten', '/crab/gewesten') - config.add_route('get_gewest_by_id', '/crab/gewesten/{gewest_id}') - config.add_route('list_provincies', '/crab/gewesten/{gewest_id}/provincies') - config.add_route('get_provincie', '/crab/provincies/{provincie_id}') - config.add_route('list_gemeenten_by_provincie', '/crab/provincies/{provincie_id}/gemeenten') - config.add_route('list_gemeenten_crab', '/crab/gewesten/{gewest_id}/gemeenten') - config.add_route('get_gemeente_crab', '/crab/gemeenten/{gemeente_id}') - config.add_route('list_deelgemeenten', '/crab/gewesten/{gewest_id}/deelgemeenten') - config.add_route('list_deelgemeenten_by_gemeente', '/crab/gemeenten/{gemeente_id}/deelgemeenten') - config.add_route('get_deelgemeente_by_id', '/crab/deelgemeenten/{deelgemeente_id}') - config.add_route('list_straten', '/crab/gemeenten/{gemeente_id}/straten') - config.add_route('get_straat_by_id', '/crab/straten/{straat_id}') - config.add_route('get_wegobject', '/crab/wegobjecten/{wegobject_id}') - config.add_route('list_huisnummers', '/crab/straten/{straat_id}/huisnummers') - config.add_route('get_huisnummer_by_straat_and_label', '/crab/straten/{straat_id}/huisnummers/{huisnummer_label}') - config.add_route('get_huisnummer_by_id', '/crab/huisnummers/{huisnummer_id}') - config.add_route('list_percelen', '/crab/huisnummers/{huisnummer_id}/percelen') - config.add_route('get_perceel_by_id', '/crab/percelen/{perceel_id1}/{perceel_id2}') - config.add_route('list_huisnummers_by_perceel', '/crab/percelen/{perceel_id1}/{perceel_id2}/huisnummers') - config.add_route('list_gebouwen', '/crab/huisnummers/{huisnummer_id}/gebouwen') - config.add_route('get_gebouw_by_id', '/crab/gebouwen/{gebouw_id}') - config.add_route('list_subadressen', '/crab/huisnummers/{huisnummer_id}/subadressen') - config.add_route('get_subadres_by_id', '/crab/subadressen/{subadres_id}') - config.add_route('list_postkantons_by_gemeente', '/crab/gemeenten/{gemeente_id}/postkantons') - config.add_route('list_adresposities_by_huisnummer', '/crab/huisnummers/{huisnummer_id}/adresposities') - config.add_route('list_adresposities_by_subadres', '/crab/subadressen/{subadres_id}/adresposities') - config.add_route('get_adrespositie_by_id', '/crab/adresposities/{adrespositie_id}') - config.add_route('list_landen', '/crab/landen') - config.add_route('get_land_by_id', '/crab/landen/{land_id}') + crabpy_pyramid.add_route(config, + 'list_gewesten', + '/crab/gewesten') + crabpy_pyramid.add_route(config, + 'get_gewest_by_id', + '/crab/gewesten/{gewest_id}') + crabpy_pyramid.add_route(config, + 'list_provincies', + '/crab/gewesten/{gewest_id}/provincies') + crabpy_pyramid.add_route(config, + 'get_provincie', + '/crab/provincies/{provincie_id}') + crabpy_pyramid.add_route(config, + 'list_gemeenten_by_provincie', + '/crab/provincies/{provincie_id}/gemeenten') + crabpy_pyramid.add_route(config, + 'list_gemeenten_crab', + '/crab/gewesten/{gewest_id}/gemeenten') + crabpy_pyramid.add_route(config, + 'get_gemeente_crab', + '/crab/gemeenten/{gemeente_id}') + crabpy_pyramid.add_route(config, + 'list_deelgemeenten', + '/crab/gewesten/{gewest_id}/deelgemeenten') + crabpy_pyramid.add_route(config, + 'list_deelgemeenten_by_gemeente', + '/crab/gemeenten/{gemeente_id}/deelgemeenten') + crabpy_pyramid.add_route(config, + 'get_deelgemeente_by_id', + '/crab/deelgemeenten/{deelgemeente_id}') + crabpy_pyramid.add_route(config, + 'list_straten', + '/crab/gemeenten/{gemeente_id}/straten') + crabpy_pyramid.add_route(config, + 'get_straat_by_id', + '/crab/straten/{straat_id}') + crabpy_pyramid.add_route(config, + 'get_wegobject', + '/crab/wegobjecten/{wegobject_id}') + crabpy_pyramid.add_route(config, + 'list_huisnummers', + '/crab/straten/{straat_id}/huisnummers') + crabpy_pyramid.add_route( + config, + 'get_huisnummer_by_straat_and_label', + '/crab/straten/{straat_id}/huisnummers/{huisnummer_label}') + crabpy_pyramid.add_route(config, + 'get_huisnummer_by_id', + '/crab/huisnummers/{huisnummer_id}') + crabpy_pyramid.add_route(config, + 'list_percelen', + '/crab/huisnummers/{huisnummer_id}/percelen') + crabpy_pyramid.add_route(config, + 'get_perceel_by_id', + '/crab/percelen/{perceel_id1}/{perceel_id2}') + crabpy_pyramid.add_route( + config, + 'list_huisnummers_by_perceel', + '/crab/percelen/{perceel_id1}/{perceel_id2}/huisnummers') + crabpy_pyramid.add_route(config, + 'list_gebouwen', + '/crab/huisnummers/{huisnummer_id}/gebouwen') + crabpy_pyramid.add_route(config, + 'get_gebouw_by_id', + '/crab/gebouwen/{gebouw_id}') + crabpy_pyramid.add_route(config, + 'list_subadressen', + '/crab/huisnummers/{huisnummer_id}/subadressen') + crabpy_pyramid.add_route(config, + 'get_subadres_by_id', + '/crab/subadressen/{subadres_id}') + crabpy_pyramid.add_route(config, + 'list_postkantons_by_gemeente', + '/crab/gemeenten/{gemeente_id}/postkantons') + crabpy_pyramid.add_route(config, + 'list_adresposities_by_huisnummer', + '/crab/huisnummers/{huisnummer_id}/adresposities') + crabpy_pyramid.add_route(config, + 'list_adresposities_by_subadres', + '/crab/subadressen/{subadres_id}/adresposities') + crabpy_pyramid.add_route(config, + 'get_adrespositie_by_id', + '/crab/adresposities/{adrespositie_id}') + crabpy_pyramid.add_route(config, + 'list_landen', + '/crab/landen') + crabpy_pyramid.add_route(config, + 'get_land_by_id', + '/crab/landen/{land_id}') diff --git a/crabpy_pyramid/tests/test_crab.py b/crabpy_pyramid/tests/test_crab.py index e4ff588..c024753 100644 --- a/crabpy_pyramid/tests/test_crab.py +++ b/crabpy_pyramid/tests/test_crab.py @@ -22,8 +22,8 @@ import unittest2 as unittest except ImportError: import unittest # noqa - - + + class TestRegistry(object): def __init__(self, settings=None): @@ -68,12 +68,13 @@ def test_build_crab_default_settings(self): r.registerUtility(G, ICrab) G2 = _build_crab(r, {}) self.assertIsInstance(G, CrabGateway) - self.assertIsInstance(G2, CrabGateway) + self.assertIsInstance(G2, CrabGateway) self.assertEqual(G, G2) def test_build_crab_custom_settings(self): settings = { 'crabpy.cache.file.root': './dogpile_data/', + 'crabpy.crab.wsdl': 'http://crab.agiv.be/wscrab/WsCrab.svc?wsdl', 'crabpy.crab.permanent.backend': 'dogpile.cache.dbm', 'crabpy.crab.permanent.expiration_time': 604800, 'crabpy.crab.permanent.arguments.filename': 'dogpile_data/crab_permanent.dbm', @@ -86,6 +87,7 @@ def test_build_crab_custom_settings(self): } r = TestRegistry(settings) crab_settings = _filter_settings(_parse_settings(settings), 'crab.') + self.assertIn('wsdl', crab_settings) if 'include' in crab_settings: del crab_settings['include'] G = _build_crab(r, crab_settings) diff --git a/crabpy_pyramid/tests/test_functional.py b/crabpy_pyramid/tests/test_functional.py index 3771a89..8a745bb 100644 --- a/crabpy_pyramid/tests/test_functional.py +++ b/crabpy_pyramid/tests/test_functional.py @@ -11,6 +11,7 @@ from pyramid import testing from webtest import TestApp +import crabpy_pyramid from crabpy_pyramid import main @@ -40,7 +41,6 @@ def run_crab_integration_tests(): settings = { 'crabpy.cache.file.root': os.path.join(os.path.dirname(__file__), 'dogpile_data'), - 'crabpy.capakey.include': True, 'crabpy.capakey.cache_config.permanent.backend': 'dogpile.cache.dbm', 'crabpy.capakey.cache_config.permanent.expiration_time': 604800, 'crabpy.capakey.cache_config.permanent.arguments.filename': os.path.join(os.path.dirname(__file__), 'dogpile_data', 'capakey_permanent.dbm'), @@ -364,3 +364,13 @@ def test_http_304_res(self): etag = res.headers['Etag'] res2 = self.testapp.get('/crab/gewesten', headers={'If-None-Match': etag}) self.assertEqual('304 Not Modified', res2.status) + + def test_list_gewesten_not_cached(self): + crabpy_pyramid.GENERATE_ETAG_ROUTE_NAMES.remove('list_gewesten') + try: + res = self.testapp.get('/crab/gewesten') + self.assertEqual('200 OK', res.status) + self.assertNotIn('ETag', res.headers) + finally: + crabpy_pyramid.GENERATE_ETAG_ROUTE_NAMES.add('list_gewesten') + diff --git a/setup.py b/setup.py index 1c416af..29d335c 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ testing_extras = tests_requires + [] setup(name='crabpy_pyramid', - version='0.7.0', + version='0.8.0', description='Bindings for the CRABpy webservices and the Pyramid framework.', long_description=README + '\n\n' + CHANGES, classifiers=[