From bb4eaae1c03d16fc674ef135479ee854ce302d4c Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Sat, 4 Jan 2014 00:32:51 +0200 Subject: [PATCH 1/2] Add support for collection column in pycsw db, so to be able to filter csw results similar to CKAN collections --- bin/ckan_pycsw.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/ckan_pycsw.py b/bin/ckan_pycsw.py index 471c71c6..c9a266cf 100644 --- a/bin/ckan_pycsw.py +++ b/bin/ckan_pycsw.py @@ -17,7 +17,7 @@ def setup_db(pycsw_config): """Setup database tables and indexes""" - from sqlalchemy import Column, Text + from sqlalchemy import Boolean, Column, Text database = pycsw_config.get('repository', 'database') table_name = pycsw_config.get('repository', 'table', 'records') @@ -25,6 +25,7 @@ def setup_db(pycsw_config): ckan_columns = [ Column('ckan_id', Text, index=True), Column('ckan_modified', Text), + Column('ckan_collection', Boolean), ] pycsw.admin.setup_db(database, @@ -63,7 +64,7 @@ def load(pycsw_config, ckan_url): log.info('Started gathering CKAN datasets identifiers: {0}'.format(str(datetime.datetime.now()))) - query = 'api/search/dataset?qjson={"fl":"id,metadata_modified,extras_harvest_object_id,extras_metadata_source", "q":"harvest_object_id:[\\"\\" TO *]", "limit":1000, "start":%s}' + query = 'api/search/dataset?qjson={"fl":"id,metadata_modified,extras_harvest_object_id,extras_metadata_source,extras_collection_package_id", "q":"harvest_object_id:[\\"\\" TO *]", "limit":1000, "start":%s}' start = 0 @@ -83,8 +84,12 @@ def load(pycsw_config, ckan_url): gathered_records[result['id']] = { 'metadata_modified': result['metadata_modified'], 'harvest_object_id': result['extras']['harvest_object_id'], - 'source': result['extras'].get('metadata_source') + 'source': result['extras'].get('metadata_source'), } + is_collection = True + if 'collection_package_id' in result['extras']: + is_collection = False + gathered_records[result['id']]['ckan_collection'] = is_collection start = start + 1000 log.debug('Gathered %s' % start) @@ -188,6 +193,7 @@ def get_record(context, repo, ckan_url, ckan_id, ckan_info): record.identifier = ckan_id record.ckan_id = ckan_id record.ckan_modified = ckan_info['metadata_modified'] + record.ckan_collection = ckan_info['ckan_collection'] return record From 7759a41636fba00209e7e3275f50e24e1e1d9329 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Wed, 8 Jan 2014 01:47:46 +0200 Subject: [PATCH 2/2] Removed extra comma from previous commit --- bin/ckan_pycsw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ckan_pycsw.py b/bin/ckan_pycsw.py index c9a266cf..49f63c33 100644 --- a/bin/ckan_pycsw.py +++ b/bin/ckan_pycsw.py @@ -84,7 +84,7 @@ def load(pycsw_config, ckan_url): gathered_records[result['id']] = { 'metadata_modified': result['metadata_modified'], 'harvest_object_id': result['extras']['harvest_object_id'], - 'source': result['extras'].get('metadata_source'), + 'source': result['extras'].get('metadata_source') } is_collection = True if 'collection_package_id' in result['extras']: