OpenConceptLab/ocl_issues#1962 | Expansion to persist unresolved repo…

… versions

core/collections/migrations/0065_expansion_unresolved_repo_versions.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-10-24 02:09
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('collections', '0064_collection_coll_org_released_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='expansion',
+            name='unresolved_repo_versions',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(), default=list, size=None),
+        ),
+    ]

core/collections/migrations/0066_alter_expansion_unresolved_repo_versions.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-10-24 02:42
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('collections', '0065_expansion_unresolved_repo_versions'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='expansion',
+            name='unresolved_repo_versions',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.JSONField(), blank=True, default=list, null=True, size=None),
+        ),
+    ]

core/collections/models.py

@@ -913,6 +913,20 @@ def resolve_valueset_versions(self):
                         versions.append(version)
         return versions
 
+    @cached_property
+    def resolve_valueset_versions_with_unresolved(self):
+        versions = []
+        unresolved = []
+        if isinstance(self.valueset, list):
+            for valueset in self.valueset:  # pylint: disable=not-an-iterable
+                if valueset:
+                    version, _ = Collection.resolve_reference_expression(valueset, self.namespace)
+                    if version.id:
+                        versions.append(version)
+                    else:
+                        unresolved.append({'url': valueset, 'namespace': self.namespace, 'type': 'reference.valueset'})
+        return versions, unresolved
+
     def clean(self):
         if not self.is_valid_filter():
             raise ValidationError({'filter': ['Invalid filter schema.']})
@@ -1042,6 +1056,7 @@ class Meta:
         'collections.Collection', related_name='expansions_resolved_collection_versions_set')
     resolved_source_versions = models.ManyToManyField(
         'sources.Source', related_name='expansions_resolved_source_versions_set')
+    unresolved_repo_versions = ArrayField(models.JSONField(), default=list, null=True, blank=True)
 
     @property
     def is_auto_generated(self):
@@ -1194,10 +1209,11 @@ def delete_expressions(self, expressions):  # Deprecated: Old way, must use dele
             batch_index_resources.apply_async(('concept', concepts_filters), queue='indexing', permanent=False)
             batch_index_resources.apply_async(('mapping', mappings_filters), queue='indexing', permanent=False)
 
-    def add_references(self, references, index=True, is_adding_all=False, attempt_reevaluate=True):  # pylint: disable=too-many-locals,too-many-statements
+    def add_references(self, references, index=True, is_adding_all=False, attempt_reevaluate=True):  # pylint: disable=too-many-locals,too-many-statements,too-many-branches
         include_refs, exclude_refs = self.to_ref_list_separated(references)
         resolved_valueset_versions = []
         resolved_system_versions = []
+        unresolved_repo_versions = []
         _system_version_cache = {}
 
         if not is_adding_all:
@@ -1223,6 +1239,8 @@ def add_references(self, references, index=True, is_adding_all=False, attempt_re
                 _system_version_cache[_cache_key] = version
                 if version.id:
                     include_system_versions.append(version)
+                else:
+                    unresolved_repo_versions.append({'url': _system, 'type': 'expansion.parameters.system-version'})
 
         def get_ref_system_version(ref):
             if ref.system:
@@ -1235,7 +1253,10 @@ def get_ref_system_version(ref):
         def get_ref_results(ref):
             nonlocal resolved_valueset_versions
             nonlocal resolved_system_versions
-            resolved_valueset_versions += ref.resolve_valueset_versions
+            nonlocal unresolved_repo_versions
+            _resolved_valueset_versions, _unresolved_valueset_versions = ref.resolve_valueset_versions_with_unresolved
+            resolved_valueset_versions += _resolved_valueset_versions
+            unresolved_repo_versions += _unresolved_valueset_versions
             ref_system_versions = []
             should_use_ref_system_version = True
             for _system_version in include_system_versions:
@@ -1247,6 +1268,15 @@ def get_ref_results(ref):
                 _system_version = get_ref_system_version(ref)
                 if _system_version:
                     ref_system_versions.append(_system_version)
+                elif ref.system:
+                    unresolved_repo_versions.append(
+                        {
+                            'url': ref.system,
+                            'namespace': ref.namespace,
+                            'version': ref.version,
+                            'type': 'reference.system'
+                        }
+                    )
             if ref_system_versions:
                 ref_system_versions = list(set(ref_system_versions))
 
@@ -1286,6 +1316,9 @@ def get_ref_results(ref):
 
         self.resolved_collection_versions.add(*compact(resolved_valueset_versions))
         self.resolved_source_versions.add(*compact(resolved_system_versions))
+        if unresolved_repo_versions:
+            self.unresolved_repo_versions = unresolved_repo_versions
+            self.save()
         self.dedupe_resources()
         if index:
             self.index_resources(index_concepts, index_mappings)
@@ -1344,6 +1377,8 @@ def calculate_uri(self):
     def clean(self):
         if not self.parameters:
             self.parameters = default_expansion_parameters()
+        if not self.unresolved_repo_versions:
+            self.unresolved_repo_versions = []
 
         super().clean()
 

core/collections/serializers.py

@@ -590,7 +590,7 @@ class Meta:
         model = Expansion
         fields = (
             'mnemonic', 'id', 'parameters', 'canonical_url', 'url', 'summary', 'created_on', 'created_by',
-            'is_processing', 'resolved_collection_versions', 'resolved_source_versions'
+            'is_processing', 'resolved_collection_versions', 'resolved_source_versions', 'unresolved_repo_versions'
         )
 
     def __init__(self, *args, **kwargs):

core/collections/tests/tests.py

@@ -1080,6 +1080,7 @@ def test_add_references_task(self):
             collection.id,
             'sourcemappings'
         )
+
         self.assertEqual(len(added_references), 4)
         self.assertEqual(errors, {})
         self.assertListEqual(
@@ -1099,6 +1100,65 @@ def test_add_references_task(self):
             sorted(list(expansion.mappings.values_list('uri', flat=True))),
             sorted([mapping1.url, mapping2.get_latest_version().url])
         )
+        self.assertEqual(expansion.unresolved_repo_versions, [])
+        self.assertEqual(expansion.resolved_collection_versions.count(), 0)
+        self.assertEqual(expansion.resolved_source_versions.count(), 3)
+        self.assertEqual(
+            sorted(list(expansion.resolved_source_versions.values_list('uri', flat=True))),
+            sorted([concept1.parent.uri, concept2.parent.uri, mapping2.parent.uri])
+        )
+
+        added_references, errors = add_references(
+            collection.created_by.id,
+            [
+                {'system': 'http://foo-system.com', 'namespace': 'barbar', 'code': 'bar'},
+                {'system': 'http://foo-system2.com|v1',
+                 'valueset': ['http://foo-valueset.com', '/orgs/Org/collections/Collection/123/'],
+                 'code': 'bar'},
+            ],
+            collection.id,
+            'sourcemappings'
+        )
+        expansion.refresh_from_db()
+
+        self.assertEqual(len(added_references), 2)
+        self.assertEqual(errors, {})
+        self.assertListEqual(
+            sorted(list(
+                collection.references.values_list('expression', flat=True)
+            )),
+            sorted([
+                concept1.get_latest_version().url,
+                concept2.get_latest_version().url,
+                mapping1.url,
+                mapping2.get_latest_version().url,
+                'http://foo-system.com/concepts/bar/',
+                'http://foo-system2.com|v1/concepts/bar/'
+            ])
+        )
+        self.assertEqual(
+            sorted(list(expansion.concepts.values_list('uri', flat=True))),
+            sorted([concept1.get_latest_version().url, concept2.get_latest_version().url])
+        )
+        self.assertEqual(
+            sorted(list(expansion.mappings.values_list('uri', flat=True))),
+            sorted([mapping1.url, mapping2.get_latest_version().url])
+        )
+        self.assertEqual(
+            expansion.unresolved_repo_versions,
+            [
+                {'url': 'http://foo-system.com', 'type': 'reference.system', 'version': None, 'namespace': 'barbar'},
+                {'url': 'http://foo-valueset.com', 'type': 'reference.valueset', 'namespace': None},
+                {'url': '/orgs/Org/collections/Collection/123/', 'type': 'reference.valueset', 'namespace': None},
+                {'url': 'http://foo-system2.com|v1', 'type': 'reference.system', 'version': None, 'namespace': None}
+            ]
+        )
+        self.assertEqual(expansion.resolved_collection_versions.count(), 0)
+        self.assertEqual(expansion.resolved_source_versions.count(), 3)
+        self.assertEqual(
+            sorted(list(expansion.resolved_source_versions.values_list('uri', flat=True))),
+            sorted([concept1.parent.uri, concept2.parent.uri, mapping2.parent.uri])
+        )
 
     @patch('core.collections.models.Collection.index_children')
     @patch('core.common.tasks.export_collection')