diff --git a/README.md b/README.md index 651f356d..8af8d4a3 100644 --- a/README.md +++ b/README.md @@ -23,21 +23,23 @@ Configuration Set the schemas you want to use with configuration options: ```ini -ckan.plugins = scheming_datasets scheming_groups +ckan.plugins = scheming_datasets # module-path:file to schemas being used scheming.dataset_schemas = ckanext.spatialx:spatialx_schema.json ckanext.spatialx:spatialxy_schema.json -scheming.group_schemas = ckanext.spatialx:group_schema.json -scheming.organization_schemas = ckanext.spatialx:org_schema.json # will try to load "spatialx_schema.json" and "spatialxy_schema.json" -# as dataset schemas and "group_schema.json" as a group schema and -# "org_schema" as an organization schema, all from the directory -# containing the ckanext.spatialx module code +# as dataset schemas # # URLs may also be used, e.g: # # scheming.dataset_schemas = http://example.com/spatialx_schema.json + +# Preset files may be included as well. The default preset setting is: +scheming.presets = ckanext.scheming:presets.json + +# The is_fallback setting may be changed as well. Defaults to false: +scheming.dataset_fallback = false ``` @@ -50,10 +52,12 @@ Example dataset schemas These schemas are included in ckanext-scheming and may be enabled with e.g: `scheming.dataset_schemas = ckanext.scheming:camel_photos.json` +These schemas use [presets](#preset) defined in +[presets.json](ckanext/scheming/presets.json). -Fields ------- +Schema Keys +----------- ### `scheming_version` @@ -62,17 +66,14 @@ Set to `1`. Future versions of ckanext-scheming may use a larger number to indicate a change to the description JSON format. -### `dataset_type`, `group_type` or `organization_type` +### `dataset_type` -These are the "type" fields stored in the dataset, group or organization. -For datasets it is used to set the URL for searching this type of dataset. +This is the "type" field stored in the dataset. +It is also used to set the URL for searching this type of dataset. -Normal datasets would be available under `/dataset`, but datasets with +Normal datasets would be available under the URL `/dataset`, but datasets with the `camel_photos.json` schema above would appear under `/camel-photos` instead. -For organizations this field should be set to `"organization"` as some -parts of CKAN depend on this value not changing. - ### `about_url` @@ -80,22 +81,24 @@ parts of CKAN depend on this value not changing. Its use is optional but highly recommended. -### `dataset_fields` and `resource_fields` or `fields` +### `dataset_fields`, `resource_fields` Fields are specified in the order you -would like them to appear in the dataset, group or organization editing -pages. Datasets have separate lists of dataset and resource fields. -Organizations and groups have a single fields list. +would like them to appear in the dataset and resource editing +pages. Fields you exclude will not be shown to the end user, and will not -be accepted when editing or updating this type of dataset, group or -organization. +be accepted when editing or updating this type of dataset. + + +Field Keys +---------- ### `field_name` -The `field_name` value is the name of an existing CKAN dataset, resource, -group or organization field or a new new extra field. Existing dataset +The `field_name` value is the name of an existing CKAN dataset or resource +field or a new new extra field. Existing dataset field names include: * `name` - the URI for the dataset @@ -112,30 +115,99 @@ New field names should follow the current lowercase_with_underscores This value is available to the form snippet as `field.field_name`. -FIXME: list group/organization fields - ### `label` The `label` value is a human-readable label for this field as it will appear in the dataset editing form. -This label may be a string or an object providing in multiple -languages: +This label may be a string or an object providing multiple +language versions: + +```json +{ + "label": { + "en": "Title", + "fr": "Titre" + }, + "...": "..." +} +``` + +When using a plain string translations will be provided with gettext: + +```json +{ + "label": "Title", + "...": "..." +} +``` + + +### `required` + +Set to `true` for fields that must be included. Set to `false` or +don't include this key for fields that are optional. + +Setting to `true` will mark the field as required in the editing form +and include `not_empty` in the default validators that will be applied +when `validators` is not specified. + +To honor this settings with custom validators include `scheming_required` +as the first validator. `scheming_required` will check the required +setting for this field and apply either the `not_empty` or `ignore_missing` +validator. + + +### `choices` + +The `choices` list must be provided for +select fields. List elements include `label`s for human-readable text for +each element (may be multiple languages like a [field label](#label)) +and `value`s that will be stored in the dataset or resource: ```json { - "en": "Title", - "fr": "Titre" + "preset": "select", + "choices": [ + { + "value": "bactrian", + "label": "Bactrian Camel" + }, + "..." + ], + "...": "..." } ``` -When using a plain string translations will be provided with gettext. + +### `preset` + +A `preset` specifies a set of default values for these field keys. They +are typically used to define validation and snippets for common field +types. + +This extension includes the following presets: + +* `"title"` - title validation and large text form snippet +* `"select"` - validation that choice is from [choices](#choices), + form select box and display snippet +* `"dataset_slug"` - dataset slug validation and form snippet that + autofills the value from the title field +* `"tag_string_autocomplete"` - tag string validation and form autocomplete +* `"dataset_organization"` - organization validation and form select box +* `"resource_url_upload"` - resource url validaton and link/upload form + field +* `"resource_format_autocomplete"` - resource format validation with + format guessing based on url and autocompleting form field + +You may add your own presets by adding them to the `scheming.presets` +configuration setting. ### `form_snippet` The `form_snippet` value is the name of the snippet template to -use for this field in the dataset, group or organization editing form. +use for this field in the dataset or resource editing form. A number of snippets are provided with this extension, but you may also provide your own by creating templates under `scheming/form_snippets/` in a template directory in your @@ -161,6 +233,8 @@ This extension includes the following form snippets: an organization selection field for datasets * [upload.html](ckanext/scheming/templates/scheming/form_snippets/upload.html) - an upload field for resource files +* [select.html](ckanext/scheming/templates/scheming/form_snippets/select.html) - + a select box ### `display_snippet` @@ -204,13 +278,20 @@ This string does not contain arbitrary python code to be executed, you may only use registered validator functions, optionally calling them with static string values provided. -New validators and converters may be added using the IValidators -plugin interface. - This extension automatically adds calls to `convert_to_extras` for new extra fields, so you should not add that to this list. +New validators and converters may be added using the +[IValidators plugin interface](http://docs.ckan.org/en/latest/extensions/plugin-interfaces.html?highlight=ivalidator#ckan.plugins.interfaces.IValidators). + +Validators that need access to other values in this schema (e.g. +to test values against the choices list) May be decorated with +the [scheming.validation.scheming_validator](ckanext/scheming/validation.py) +function. This decorator will make scheming pass this field dict to the +validator and use its return value for validation of the field. + + ### `output_validators` The `output_validators` value is like `validators` but used when @@ -222,28 +303,3 @@ This extension automatically adds calls to `convert_from_extras` for extra fields so you should not add that to this list. -### `choices` - -(not yet implemented) - -The `choices` list must be provided for multiple-choice and -single-choice fields. The `label`s are human-readable text for -the dataset editing form and the `value`s are stored in -the dataset field or are used for tag names in tag vocabularies. - -A validator is automatically added for creating or updating datasets -that only allows values from this list. - - -### `tag_vocabulary` - -(not yet implemented) - -The `tag_vocabulary` value is used for the name of the tag vocabulary -that will store the valid choices for a multiple-choice field. - -Tag vocabularies are global to the CKAN instance so this name should -be made unique, e.g. by prefixing it with a domain name in reverse order -and the name of the schema. - - diff --git a/ckanext/__init__.py b/ckanext/__init__.py index e69de29b..2e2033b3 100644 --- a/ckanext/__init__.py +++ b/ckanext/__init__.py @@ -0,0 +1,7 @@ +# this is a namespace package +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/ckanext/scheming/camel_photos.json b/ckanext/scheming/camel_photos.json index 3d3c6902..57cf27de 100644 --- a/ckanext/scheming/camel_photos.json +++ b/ckanext/scheming/camel_photos.json @@ -6,16 +6,13 @@ { "field_name": "title", "label": "Title", - "form_snippet": "large_text.html", - "validators": "if_empty_same_as(name) unicode", - "form_attrs": {"data-module": "slug-preview-target"}, + "preset": "title", "form_placeholder": "eg. Larry, Peter, Susan" }, { "field_name": "name", "label": "URL", - "form_snippet": "slug.html", - "validators": "not_empty unicode name_validator package_name_validator", + "preset": "dataset_slug", "form_placeholder": "eg. camel-no-5" }, { @@ -24,6 +21,33 @@ "validators": "ignore_missing int_validator", "form_placeholder": "eg. 2" }, + { + "field_name": "category", + "label": "Category", + "preset": "select", + "choices": [ + { + "value": "bactrian", + "label": "Bactrian Camel" + }, + { + "value": "hybrid", + "label": "Hybrid Camel" + }, + { + "value": "f2hybrid", + "label": "F2 Hybrid Camel" + }, + { + "value": "snowwhite", + "label": "Snow-white Dromedary" + }, + { + "value": "black", + "label": "Black Camel" + } + ] + }, { "field_name": "other", "label": {"en": "Other information"}, @@ -34,11 +58,8 @@ { "field_name": "url", "label": "Photo", - "validators": "not_empty unicode remove_whitespace", - "form_snippet": "upload.html", + "preset": "resource_url_upload", "form_placeholder": "http://example.com/my-camel-photo.jpg", - "upload_field": "upload", - "upload_clear": "clear_upload", "upload_label": "Photo" }, { diff --git a/ckanext/scheming/ckan_dataset.json b/ckanext/scheming/ckan_dataset.json index 044daf49..20342fbf 100644 --- a/ckanext/scheming/ckan_dataset.json +++ b/ckanext/scheming/ckan_dataset.json @@ -1,21 +1,19 @@ { "scheming_version": 1, "dataset_type": "dataset", + "about": "A reimplementation of the default CKAN dataset schema", "about_url": "http://github.com/open-data/ckanext-scheming", "dataset_fields": [ { "field_name": "title", "label": "Title", - "validators": "if_empty_same_as(name) unicode", - "form_snippet": "large_text.html", - "form_attrs": {"data-module": "slug-preview-target"}, + "preset": "title", "form_placeholder": "eg. A descriptive title" }, { "field_name": "name", "label": "URL", - "validators": "not_empty unicode name_validator package_name_validator", - "form_snippet": "slug.html", + "preset": "dataset_slug", "form_placeholder": "eg. my-dataset" }, { @@ -27,13 +25,8 @@ { "field_name": "tag_string", "label": "Tags", - "validators": "ignore_missing tag_string_convert", - "form_placeholder": "eg. economy, mental health, government", - "form_attrs": { - "data-module": "autocomplete", - "data-module-tags": "", - "data-module-source": "/api/2/util/tag/autocomplete?incomplete=?" - } + "preset": "tag_string_autocomplete", + "form_placeholder": "eg. economy, mental health, government" }, { "field_name": "license_id", @@ -43,8 +36,7 @@ { "field_name": "owner_org", "label": "Organization", - "validators": "owner_org_validator unicode", - "form_snippet": "organization.html" + "preset": "dataset_organization" }, { "field_name": "url", @@ -92,12 +84,7 @@ { "field_name": "url", "label": "URL", - "validators": "not_empty unicode remove_whitespace", - "form_snippet": "upload.html", - "form_placeholder": "http://example.com/my-data.csv", - "upload_field": "upload", - "upload_clear": "clear_upload", - "upload_label": "File" + "preset": "resource_url_upload" }, { "field_name": "name", @@ -113,12 +100,7 @@ { "field_name": "format", "label": "Format", - "validators": "if_empty_guess_format ignore_missing clean_format unicode", - "form_placeholder": "eg. CSV, XML or JSON", - "form_attrs": { - "data-module": "autocomplete", - "data-module-source": "/api/2/util/resource/format_autocomplete?incomplete=?" - } + "preset": "resource_format_autocomplete" } ] } diff --git a/ckanext/scheming/helpers.py b/ckanext/scheming/helpers.py index 6b1d437f..b4adf491 100644 --- a/ckanext/scheming/helpers.py +++ b/ckanext/scheming/helpers.py @@ -26,6 +26,21 @@ def scheming_language_text(text, _gettext=None, _lang=None): return _gettext(text) +def scheming_choices_label(choices, value): + """ + :param choices: choices list of {"label": .., "value": ..} dicts + :param value: value selected + + Return the label from choices with a matching value, or + the value passed when not found. Result is passed through + scheming_language_text before being returned. + """ + for c in choices: + if c['value'] == value: + return scheming_language_text(c['label']) + return scheming_language_text(value) + + def scheming_field_required(field): """ Return field['required'] or guess based on validators if not present. @@ -35,62 +50,68 @@ def scheming_field_required(field): return 'not_empty' in field.get('validators', '').split() -def scheming_dataset_schemas(): +def scheming_dataset_schemas(expanded=True): """ Return the dict of dataset schemas. Or if scheming_datasets plugin is not loaded return None. """ from ckanext.scheming.plugins import SchemingDatasetsPlugin as p if p.instance: + if expanded: + return p.instance._expanded_schemas return p.instance._schemas -def scheming_get_dataset_schema(dataset_type): +def scheming_get_dataset_schema(dataset_type, expanded=True): """ Return the schema for the dataset_type passed or None if no schema is defined for that dataset_type """ - schemas = scheming_dataset_schemas() + schemas = scheming_dataset_schemas(expanded) if schemas: return schemas.get(dataset_type) -def scheming_group_schemas(): +def scheming_group_schemas(expanded=True): """ Return the dict of group schemas. Or if scheming_groups plugin is not loaded return None. """ from ckanext.scheming.plugins import SchemingGroupsPlugin as p if p.instance: + if expanded: + return p.instance._expanded_schemas return p.instance._schemas -def scheming_get_group_schema(group_type): +def scheming_get_group_schema(group_type, expanded=True): """ Return the schema for the group_type passed or None if no schema is defined for that group_type """ - schemas = scheming_group_schemas() + schemas = scheming_group_schemas(expanded) if schemas: return schemas.get(group_type) -def scheming_organization_schemas(): +def scheming_organization_schemas(expanded=True): """ Return the dict of organization schemas. Or if scheming_organizations plugin is not loaded return None. """ from ckanext.scheming.plugins import SchemingOrganizationsPlugin as p if p.instance: + if expanded: + return p.instance._expanded_schemas return p.instance._schemas -def scheming_get_organization_schema(organization_type): +def scheming_get_organization_schema(organization_type, expanded=True): """ Return the schema for the organization_type passed or None if no schema is defined for that organization_type """ - schemas = scheming_organization_schemas() + schemas = scheming_organization_schemas(expanded) if schemas: return schemas.get(organization_type) diff --git a/ckanext/scheming/logic.py b/ckanext/scheming/logic.py index 24775dd6..79e027ad 100644 --- a/ckanext/scheming/logic.py +++ b/ckanext/scheming/logic.py @@ -19,9 +19,11 @@ def scheming_dataset_schema_show(context, data_dict): Return the scheming schema for a given dataset type :param type: the dataset type + :param expanded: True to expand presets (default) ''' t = get_or_bust(data_dict, 'type') - s = scheming_get_dataset_schema(t) + expanded = data_dict.get('expanded', True) + s = scheming_get_dataset_schema(t, expanded) if s is None: raise ObjectNotFound() return s @@ -39,9 +41,11 @@ def scheming_group_schema_show(context, data_dict): Return the scheming schema for a given group type :param type: the group type + :param expanded: True to expand presets (default) ''' t = get_or_bust(data_dict, 'type') - s = scheming_get_group_schema(t) + expanded = data_dict.get('expanded', True) + s = scheming_get_group_schema(t, expanded) if s is None: raise ObjectNotFound() return s @@ -60,9 +64,11 @@ def scheming_organization_schema_show(context, data_dict): Return the scheming schema for a given organization type :param type: the organization type + :param expanded: True to expand presets (default) ''' t = get_or_bust(data_dict, 'type') - s = scheming_get_organization_schema(t) + expanded = data_dict.get('expanded', True) + s = scheming_get_organization_schema(t, expanded) if s is None: raise ObjectNotFound() return s diff --git a/ckanext/scheming/plugins.py b/ckanext/scheming/plugins.py index 51e10b74..8ffb9c59 100644 --- a/ckanext/scheming/plugins.py +++ b/ckanext/scheming/plugins.py @@ -11,7 +11,8 @@ from ckanext.scheming import helpers from ckanext.scheming.errors import SchemingException -from ckanext.scheming.validation import validators_from_string +from ckanext.scheming.validation import ( + validators_from_string, scheming_choices, scheming_required) from ckanext.scheming.logic import ( scheming_dataset_schema_list, scheming_dataset_schema_show, scheming_group_schema_list, scheming_group_schema_show, @@ -23,9 +24,13 @@ import inspect ignore_missing = get_validator('ignore_missing') +not_missing = get_validator('not_missing') convert_to_extras = get_converter('convert_to_extras') convert_from_extras = get_converter('convert_from_extras') +DEFAULT_PRESETS = 'ckanext.scheming:presets.json' + + class _SchemingMixin(object): """ Store single plugin instances in class variable 'instance' @@ -35,7 +40,9 @@ class _SchemingMixin(object): """ instance = None _helpers_loaded = False + _presets = None _template_dir_added = False + _validators_loaded = False def get_helpers(self): if _SchemingMixin._helpers_loaded: @@ -43,6 +50,7 @@ def get_helpers(self): _SchemingMixin._helpers_loaded = True return { 'scheming_language_text': helpers.scheming_language_text, + 'scheming_choices_label': helpers.scheming_choices_label, 'scheming_field_required': helpers.scheming_field_required, 'scheming_dataset_schemas': helpers.scheming_dataset_schemas, 'scheming_get_dataset_schema': helpers.scheming_get_dataset_schema, @@ -54,24 +62,46 @@ def get_helpers(self): helpers.scheming_get_organization_schema, } + def get_validators(self): + if _SchemingMixin._validators_loaded: + return {} + _SchemingMixin._validators_loaded = True + return { + 'scheming_choices': scheming_choices, + 'scheming_required': scheming_required, + } + def _add_template_directory(self, config): if _SchemingMixin._template_dir_added: return _SchemingMixin._template_dir_added = True add_template_directory(config, 'templates') + def _load_presets(self, config): + if _SchemingMixin._presets is not None: + return + presets = config.get('scheming.presets', DEFAULT_PRESETS).split() + _SchemingMixin._presets = {} + for f in reversed(presets): + for p in _load_schema(f)['presets']: + _SchemingMixin._presets[p['preset_name']] = p['values'] + def update_config(self, config): if self.instance: # reloading plugins, probably in WebTest _SchemingMixin._helpers_loaded = False + _SchemingMixin._validators_loaded = False # record our plugin instance in a place where our helpers # can find it: self._store_instance(self) self._add_template_directory(config) + self._load_presets(config) self._is_fallback = asbool(config.get(self.FALLBACK_OPTION, False)) + self._schema_urls = config.get(self.SCHEMA_OPTION, "").split() self._schemas = _load_schemas(self._schema_urls, self.SCHEMA_TYPE_FIELD) + self._expanded_schemas = _expand_schemas(self._schemas) def is_fallback(self): return self._is_fallback @@ -88,7 +118,7 @@ def group_types(self): def setup_template_variables(self, context, data_dict): group_type = context.get('group_type') if not group_type: - if c.group_dict: + if c.group_dict: group_type = c.group_dict['type'] else: group_type = self.UNSPECIFIED_GROUP_TYPE @@ -102,27 +132,18 @@ def db_to_form_schema_options(self, options): def validate(self, context, data_dict, schema, action): thing, action_type = action.split('_') t = data_dict.get('type') - if not t or t not in self._schemas: # pragma: no cover + if not t or t not in self._schemas: return data_dict, {'type': "Unsupported {thing} type: {t}".format( thing=thing, t=t)} - scheming_schema = self._schemas[t] + scheming_schema = self._expanded_schemas[t] scheming_fields = scheming_schema['fields'] + + get_validators = (_field_output_validators + if action_type == 'show' else _field_validators) + for f in scheming_fields: - if action_type == 'show': - if f['field_name'] not in schema: - validators = [convert_from_extras, ignore_missing] - else: - validators = [ignore_missing] - if 'output_validators' in f: - validators += validators_from_string(f['output_validators']) - else: - if 'validators' in f: - validators = validators_from_string(f['validators']) - else: - validators = [ignore_missing, unicode] - if f['field_name'] not in schema: - validators = validators + [convert_to_extras] - schema[f['field_name']] = validators + schema[f['field_name']] = get_validators(f, + f['field_name'] not in schema) return navl_validate(data_dict, schema, context) @@ -133,6 +154,7 @@ class SchemingDatasetsPlugin(p.SingletonPlugin, DefaultDatasetForm, p.implements(p.ITemplateHelpers) p.implements(p.IDatasetForm, inherit=True) p.implements(p.IActions) + p.implements(p.IValidators) SCHEMA_OPTION = 'scheming.dataset_schemas' FALLBACK_OPTION = 'scheming.dataset_fallback' @@ -164,40 +186,21 @@ def validate(self, context, data_dict, schema, action): """ thing, action_type = action.split('_') t = data_dict.get('type') - if not t or t not in self._schemas: # pragma: no cover + if not t or t not in self._schemas: return data_dict, {'type': [ "Unsupported dataset type: {t}".format(t=t)]} - scheming_schema = self._schemas[t] + scheming_schema = self._expanded_schemas[t] + + get_validators = (_field_output_validators + if action_type == 'show' else _field_validators) for f in scheming_schema['dataset_fields']: - if action_type == 'show': - if f['field_name'] not in schema: - validators = [convert_from_extras, ignore_missing] - else: - validators = [ignore_missing] - if 'output_validators' in f: - validators += validators_from_string(f['output_validators']) - else: - if 'validators' in f: - validators = validators_from_string(f['validators']) - else: - validators = [ignore_missing, unicode] - if f['field_name'] not in schema: - validators = validators + [convert_to_extras] - schema[f['field_name']] = validators + schema[f['field_name']] = get_validators(f, + f['field_name'] not in schema) resource_schema = schema['resources'] for f in scheming_schema['resource_fields']: - if action_type == 'show': - validators = [ignore_missing] - if 'output_validators' in f: - validators += validators_from_string(f['output_validators']) - else: - if 'validators' in f: - validators = validators_from_string(f['validators']) - else: - validators = [ignore_missing, unicode] - resource_schema[f['field_name']] = validators + resource_schema[f['field_name']] = get_validators(f, False) return navl_validate(data_dict, schema, context) @@ -217,6 +220,7 @@ class SchemingGroupsPlugin(p.SingletonPlugin, _GroupOrganizationMixin, p.implements(p.ITemplateHelpers) p.implements(p.IGroupForm, inherit=True) p.implements(p.IActions) + p.implements(p.IValidators) SCHEMA_OPTION = 'scheming.group_schemas' FALLBACK_OPTION = 'scheming.group_fallback' @@ -246,6 +250,7 @@ class SchemingOrganizationsPlugin(p.SingletonPlugin, _GroupOrganizationMixin, p.implements(p.ITemplateHelpers) p.implements(p.IGroupForm, inherit=True) p.implements(p.IActions) + p.implements(p.IValidators) SCHEMA_OPTION = 'scheming.organization_schemas' FALLBACK_OPTION = 'scheming.organization_fallback' @@ -265,8 +270,10 @@ def about_template(self): def get_actions(self): return { - 'scheming_organization_schema_list': scheming_organization_schema_list, - 'scheming_organization_schema_show': scheming_organization_schema_show, + 'scheming_organization_schema_list': + scheming_organization_schema_list, + 'scheming_organization_schema_show': + scheming_organization_schema_show, } @@ -308,3 +315,60 @@ def _load_schema_url(url): raise SchemingException("Could not load %s" % url) return json.loads(tables) + + +def _field_output_validators(f, convert_extras): + """ + Return the output validators for a scheming field f + """ + if convert_extras: + validators = [convert_from_extras, ignore_missing] + else: + validators = [ignore_missing] + if 'output_validators' in f: + validators += validators_from_string(f['output_validators'], f) + return validators + +def _field_validators(f, convert_extras): + """ + Return the validators for a scheming field f + """ + validators = [] + if 'validators' in f: + validators = validators_from_string(f['validators'], f) + elif helpers.scheming_field_required(f): + validators = [not_missing, unicode] + else: + validators = [ignore_missing, unicode] + + if convert_extras: + validators = validators + [convert_to_extras] + return validators + +def _expand_preset(f): + """ + If scheming field f includes a preset value return a new field + based on the preset with values from f overriding any values in the + preset. + + raises SchemingException if the preset given is not found. + """ + if 'preset' not in f: + return f + if f['preset'] not in _SchemingMixin._presets: + raise SchemingException("preset '%s' not defined" % f['preset']) + return dict(_SchemingMixin._presets[f['preset']], **f) + +def _expand_schemas(schemas): + """ + Return a new dict of schemas with all field presets expanded. + """ + out = {} + for name, original in schemas.iteritems(): + s = dict(original) + for fname in ('fields', 'dataset_fields', 'resource_fields'): + if fname not in s: + continue + s[fname] = [_expand_preset(f) for f in s[fname]] + out[name] = s + return out diff --git a/ckanext/scheming/presets.json b/ckanext/scheming/presets.json new file mode 100644 index 00000000..395560d9 --- /dev/null +++ b/ckanext/scheming/presets.json @@ -0,0 +1,72 @@ +{ + "scheming_presets_version": 1, + "about": "these are the default scheming field presets", + "about_url": "http://github.com/open-data/ckanext-scheming#preset", + "presets": [ + { + "preset_name": "title", + "values": { + "validators": "if_empty_same_as(name) unicode", + "form_snippet": "large_text.html", + "form_attrs": { + "data-module": "slug-preview-target" + } + } + }, + { + "preset_name": "dataset_slug", + "values": { + "validators": "not_empty unicode name_validator package_name_validator", + "form_snippet": "slug.html" + } + }, + { + "preset_name": "tag_string_autocomplete", + "values": { + "validators": "ignore_missing tag_string_convert", + "form_attrs": { + "data-module": "autocomplete", + "data-module-tags": "", + "data-module-source": "/api/2/util/tag/autocomplete?incomplete=?" + } + } + }, + { + "preset_name": "dataset_organization", + "values": { + "validators": "owner_org_validator unicode", + "form_snippet": "organization.html" + } + }, + { + "preset_name": "resource_url_upload", + "values": { + "validators": "not_empty unicode remove_whitespace", + "form_snippet": "upload.html", + "form_placeholder": "http://example.com/my-data.csv", + "upload_field": "upload", + "upload_clear": "clear_upload", + "upload_label": "File" + } + }, + { + "preset_name": "resource_format_autocomplete", + "values": { + "validators": "if_empty_guess_format ignore_missing clean_format unicode", + "form_placeholder": "eg. CSV, XML or JSON", + "form_attrs": { + "data-module": "autocomplete", + "data-module-source": "/api/2/util/resource/format_autocomplete?incomplete=?" + } + } + }, + { + "preset_name": "select", + "values": { + "form_snippet": "select.html", + "display_snippet": "select.html", + "validators": "scheming_required scheming_choices" + } + } + ] +} diff --git a/ckanext/scheming/templates/scheming/display_snippets/select.html b/ckanext/scheming/templates/scheming/display_snippets/select.html new file mode 100644 index 00000000..afc06da6 --- /dev/null +++ b/ckanext/scheming/templates/scheming/display_snippets/select.html @@ -0,0 +1 @@ +{{ h.scheming_choices_label(field.choices, data[field.field_name]) }} diff --git a/ckanext/scheming/templates/scheming/form_snippets/select.html b/ckanext/scheming/templates/scheming/form_snippets/select.html new file mode 100644 index 00000000..297acd4d --- /dev/null +++ b/ckanext/scheming/templates/scheming/form_snippets/select.html @@ -0,0 +1,23 @@ +{% import 'macros/form.html' as form %} + +{%- set options=[] -%} +{%- if not h.scheming_field_required(field) -%} + {%- do options.append({'value': ''}) -%} +{%- endif -%} +{%- for c in field.choices -%} + {%- do options.append({ + 'value': c.value, + 'text': h.scheming_language_text(c.label)}) -%} +{%- endfor -%} + +{{ form.select( + field.field_name, + id='field-' + field.field_name, + label=h.scheming_language_text(field.label), + options=options, + selected=data[field.field_name], + error=errors[field.field_name], + classes=['control-medium'], + attrs=field.form_attrs if 'form_attrs' in field else {}, + is_required=h.scheming_field_required(field) + ) }} diff --git a/ckanext/scheming/templates/scheming/snippets/display_field.html b/ckanext/scheming/templates/scheming/snippets/display_field.html index 97c89fff..2fa62e72 100644 --- a/ckanext/scheming/templates/scheming/snippets/display_field.html +++ b/ckanext/scheming/templates/scheming/snippets/display_field.html @@ -4,7 +4,11 @@ {%- set display_snippet = field.display_snippet -%} {%- if not display_snippet -%} - {%- set display_snippet = 'text.html' -%} + {%- if 'choices' in field -%} + {%- set display_snippet = 'select.html' -%} + {%- else -%} + {%- set display_snippet = 'text.html' -%} + {%- endif -%} {%- endif -%} {%- if '/' not in display_snippet -%} diff --git a/ckanext/scheming/tests/test_dataset_display.py b/ckanext/scheming/tests/test_dataset_display.py index 53f04393..3af2f156 100644 --- a/ckanext/scheming/tests/test_dataset_display.py +++ b/ckanext/scheming/tests/test_dataset_display.py @@ -35,3 +35,15 @@ def test_resource_displays_custom_fields(self): response = app.get(url='/dataset/set-two/resource/' + d['resources'][0]['id']) assert_true('Camels in Photo' in response.body) + + def test_choice_field_shows_labels(self): + user = Sysadmin() + d = Dataset( + user=user, + type='camel-photos', + name='with-choice', + category='hybrid', + ) + app = self._get_test_app() + response = app.get(url='/dataset/with-choice') + assert_true('Hybrid Camel' in response.body) diff --git a/ckanext/scheming/tests/test_validation.py b/ckanext/scheming/tests/test_validation.py index 13efa225..f745381c 100644 --- a/ckanext/scheming/tests/test_validation.py +++ b/ckanext/scheming/tests/test_validation.py @@ -1,7 +1,10 @@ -from nose.tools import assert_raises +from nose.tools import assert_raises, assert_equals +from ckanapi import LocalCKAN, ValidationError from ckanext.scheming.errors import SchemingException from ckanext.scheming.validation import get_validator_or_converter +from ckanext.scheming.plugins import ( + SchemingDatasetsPlugin, SchemingGroupsPlugin) class TestGetValidatorOrConverter(object): def test_missing(self): @@ -13,3 +16,34 @@ def test_validator_name(self): def test_converter_name(self): assert get_validator_or_converter('remove_whitespace') + + +class TestChoices(object): + def test_choice_field_only_accepts_given_choices(self): + lc = LocalCKAN() + assert_raises(ValidationError, lc.action.package_create, + type='camel-photos', + name='fred', + category='rocker', + ) + + def test_choice_field_accepts_valid_choice(self): + lc = LocalCKAN() + d = lc.action.package_create( + type='camel-photos', + name='fred', + category='f2hybrid', + ) + assert_equals(d['category'], 'f2hybrid') + + +class TestInvalidType(object): + def test_invalid_dataset_type(self): + p = SchemingDatasetsPlugin.instance + data, errors = p.validate({}, {'type': 'banana'}, {}, 'dataset_show') + assert_equals(list(errors), ['type']) + + def test_invalid_group_type(self): + p = SchemingGroupsPlugin.instance + data, errors = p.validate({}, {'type': 'banana'}, {}, 'dataset_show') + assert_equals(list(errors), ['type']) diff --git a/ckanext/scheming/validation.py b/ckanext/scheming/validation.py index 53beeaed..f85c5a6c 100644 --- a/ckanext/scheming/validation.py +++ b/ckanext/scheming/validation.py @@ -2,8 +2,40 @@ from ckanext.scheming.errors import SchemingException +OneOf = get_validator('OneOf') +ignore_missing = get_validator('ignore_missing') +not_missing = get_validator('not_missing') -def validators_from_string(s): +def scheming_validator(fn): + """ + Decorate a validator that needs to have the scheming fields + passed with this function. When generating navl validator lists + the function decorated will be called passing the field + data to produce the actual validator for each field. + """ + fn.is_a_scheming_validator = True + return fn + + +@scheming_validator +def scheming_choices(field): + """ + Require that one of the field choices values is passed. + """ + return OneOf([c['value'] for c in field['choices']]) + + +@scheming_validator +def scheming_required(field): + """ + not_missing if field['required'] else ignore_missing + """ + if field.get('required'): + return not_missing + return ignore_missing + + +def validators_from_string(s, field): """ convert a schema validators string to a list of validators @@ -16,10 +48,12 @@ def validators_from_string(s): if '(' in p and p[-1] == ')': name, args = p.split('(', 1) args = args[:-1].split(',') # trim trailing ')', break up - v = get_validator_or_converter(name) - out.append(v(*args)) + v = get_validator_or_converter(name)(*args) else: - out.append(get_validator_or_converter(p)) + v = get_validator_or_converter(p) + if getattr(v, 'is_a_scheming_validator', False): + v = v(field) + out.append(v) return out