diff --git a/core/collections/migrations/0065_expansion_unresolved_repo_versions.py b/core/collections/migrations/0065_expansion_unresolved_repo_versions.py new file mode 100644 index 000000000..fb2526a68 --- /dev/null +++ b/core/collections/migrations/0065_expansion_unresolved_repo_versions.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.11 on 2024-10-24 02:09 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('collections', '0064_collection_coll_org_released_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='expansion', + name='unresolved_repo_versions', + field=django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(), default=list, size=None), + ), + ] diff --git a/core/collections/migrations/0066_alter_expansion_unresolved_repo_versions.py b/core/collections/migrations/0066_alter_expansion_unresolved_repo_versions.py new file mode 100644 index 000000000..920ee7b52 --- /dev/null +++ b/core/collections/migrations/0066_alter_expansion_unresolved_repo_versions.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.11 on 2024-10-24 02:42 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('collections', '0065_expansion_unresolved_repo_versions'), + ] + + operations = [ + migrations.AlterField( + model_name='expansion', + name='unresolved_repo_versions', + field=django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(), blank=True, default=list, null=True, size=None), + ), + ] diff --git a/core/collections/models.py b/core/collections/models.py index f7f02c1a4..1e4de411a 100644 --- a/core/collections/models.py +++ b/core/collections/models.py @@ -913,6 +913,20 @@ def resolve_valueset_versions(self): versions.append(version) return versions + @cached_property + def resolve_valueset_versions_with_unresolved(self): + versions = [] + unresolved = [] + if isinstance(self.valueset, list): + for valueset in self.valueset: # pylint: disable=not-an-iterable + if valueset: + version, _ = Collection.resolve_reference_expression(valueset, self.namespace) + if version.id: + versions.append(version) + else: + unresolved.append({'url': valueset, 'namespace': self.namespace, 'type': 'reference.valueset'}) + return versions, unresolved + def clean(self): if not self.is_valid_filter(): raise ValidationError({'filter': ['Invalid filter schema.']}) @@ -1042,6 +1056,7 @@ class Meta: 'collections.Collection', related_name='expansions_resolved_collection_versions_set') resolved_source_versions = models.ManyToManyField( 'sources.Source', related_name='expansions_resolved_source_versions_set') + unresolved_repo_versions = ArrayField(models.JSONField(), default=list, null=True, blank=True) @property def is_auto_generated(self): @@ -1194,10 +1209,11 @@ def delete_expressions(self, expressions): # Deprecated: Old way, must use dele batch_index_resources.apply_async(('concept', concepts_filters), queue='indexing', permanent=False) batch_index_resources.apply_async(('mapping', mappings_filters), queue='indexing', permanent=False) - def add_references(self, references, index=True, is_adding_all=False, attempt_reevaluate=True): # pylint: disable=too-many-locals,too-many-statements + def add_references(self, references, index=True, is_adding_all=False, attempt_reevaluate=True): # pylint: disable=too-many-locals,too-many-statements,too-many-branches include_refs, exclude_refs = self.to_ref_list_separated(references) resolved_valueset_versions = [] resolved_system_versions = [] + unresolved_repo_versions = [] _system_version_cache = {} if not is_adding_all: @@ -1223,6 +1239,8 @@ def add_references(self, references, index=True, is_adding_all=False, attempt_re _system_version_cache[_cache_key] = version if version.id: include_system_versions.append(version) + else: + unresolved_repo_versions.append({'url': _system, 'type': 'expansion.parameters.system-version'}) def get_ref_system_version(ref): if ref.system: @@ -1235,7 +1253,10 @@ def get_ref_system_version(ref): def get_ref_results(ref): nonlocal resolved_valueset_versions nonlocal resolved_system_versions - resolved_valueset_versions += ref.resolve_valueset_versions + nonlocal unresolved_repo_versions + _resolved_valueset_versions, _unresolved_valueset_versions = ref.resolve_valueset_versions_with_unresolved + resolved_valueset_versions += _resolved_valueset_versions + unresolved_repo_versions += _unresolved_valueset_versions ref_system_versions = [] should_use_ref_system_version = True for _system_version in include_system_versions: @@ -1247,6 +1268,15 @@ def get_ref_results(ref): _system_version = get_ref_system_version(ref) if _system_version: ref_system_versions.append(_system_version) + elif ref.system: + unresolved_repo_versions.append( + { + 'url': ref.system, + 'namespace': ref.namespace, + 'version': ref.version, + 'type': 'reference.system' + } + ) if ref_system_versions: ref_system_versions = list(set(ref_system_versions)) @@ -1286,6 +1316,9 @@ def get_ref_results(ref): self.resolved_collection_versions.add(*compact(resolved_valueset_versions)) self.resolved_source_versions.add(*compact(resolved_system_versions)) + if unresolved_repo_versions: + self.unresolved_repo_versions = unresolved_repo_versions + self.save() self.dedupe_resources() if index: self.index_resources(index_concepts, index_mappings) @@ -1344,6 +1377,8 @@ def calculate_uri(self): def clean(self): if not self.parameters: self.parameters = default_expansion_parameters() + if not self.unresolved_repo_versions: + self.unresolved_repo_versions = [] super().clean() diff --git a/core/collections/serializers.py b/core/collections/serializers.py index 6e21d2a47..dd062e74f 100644 --- a/core/collections/serializers.py +++ b/core/collections/serializers.py @@ -590,7 +590,7 @@ class Meta: model = Expansion fields = ( 'mnemonic', 'id', 'parameters', 'canonical_url', 'url', 'summary', 'created_on', 'created_by', - 'is_processing', 'resolved_collection_versions', 'resolved_source_versions' + 'is_processing', 'resolved_collection_versions', 'resolved_source_versions', 'unresolved_repo_versions' ) def __init__(self, *args, **kwargs): diff --git a/core/collections/tests/tests.py b/core/collections/tests/tests.py index a734c619f..caa3fbe76 100644 --- a/core/collections/tests/tests.py +++ b/core/collections/tests/tests.py @@ -1080,6 +1080,7 @@ def test_add_references_task(self): collection.id, 'sourcemappings' ) + self.assertEqual(len(added_references), 4) self.assertEqual(errors, {}) self.assertListEqual( @@ -1099,6 +1100,65 @@ def test_add_references_task(self): sorted(list(expansion.mappings.values_list('uri', flat=True))), sorted([mapping1.url, mapping2.get_latest_version().url]) ) + self.assertEqual(expansion.unresolved_repo_versions, []) + self.assertEqual(expansion.resolved_collection_versions.count(), 0) + self.assertEqual(expansion.resolved_source_versions.count(), 3) + self.assertEqual( + sorted(list(expansion.resolved_source_versions.values_list('uri', flat=True))), + sorted([concept1.parent.uri, concept2.parent.uri, mapping2.parent.uri]) + ) + + added_references, errors = add_references( + collection.created_by.id, + [ + {'system': 'http://foo-system.com', 'namespace': 'barbar', 'code': 'bar'}, + {'system': 'http://foo-system2.com|v1', + 'valueset': ['http://foo-valueset.com', '/orgs/Org/collections/Collection/123/'], + 'code': 'bar'}, + ], + collection.id, + 'sourcemappings' + ) + expansion.refresh_from_db() + + self.assertEqual(len(added_references), 2) + self.assertEqual(errors, {}) + self.assertListEqual( + sorted(list( + collection.references.values_list('expression', flat=True) + )), + sorted([ + concept1.get_latest_version().url, + concept2.get_latest_version().url, + mapping1.url, + mapping2.get_latest_version().url, + 'http://foo-system.com/concepts/bar/', + 'http://foo-system2.com|v1/concepts/bar/' + ]) + ) + self.assertEqual( + sorted(list(expansion.concepts.values_list('uri', flat=True))), + sorted([concept1.get_latest_version().url, concept2.get_latest_version().url]) + ) + self.assertEqual( + sorted(list(expansion.mappings.values_list('uri', flat=True))), + sorted([mapping1.url, mapping2.get_latest_version().url]) + ) + self.assertEqual( + expansion.unresolved_repo_versions, + [ + {'url': 'http://foo-system.com', 'type': 'reference.system', 'version': None, 'namespace': 'barbar'}, + {'url': 'http://foo-valueset.com', 'type': 'reference.valueset', 'namespace': None}, + {'url': '/orgs/Org/collections/Collection/123/', 'type': 'reference.valueset', 'namespace': None}, + {'url': 'http://foo-system2.com|v1', 'type': 'reference.system', 'version': None, 'namespace': None} + ] + ) + self.assertEqual(expansion.resolved_collection_versions.count(), 0) + self.assertEqual(expansion.resolved_source_versions.count(), 3) + self.assertEqual( + sorted(list(expansion.resolved_source_versions.values_list('uri', flat=True))), + sorted([concept1.parent.uri, concept2.parent.uri, mapping2.parent.uri]) + ) @patch('core.collections.models.Collection.index_children') @patch('core.common.tasks.export_collection')