Merge pull request eregs#35 from cmc333333/list-all-regs

Allow all versions of all regs to be listed

regcore/db/__init__.py

@@ -30,9 +30,10 @@ def bulk_put(self, regs, version, root_label):
            Each should have the provided version"""
         raise NotImplementedError
 
-    def listing(self, label):
-        """Documentation method. List regulation versions that match this
-        label, sorted"""
+    def listing(self, label=None):
+        """Documentation method. Return a list of (version, label) pairs for
+        regulation objects that match the provided label (or all root regs),
+        sorted by version"""
         raise NotImplementedError
 
 

regcore/db/django_models.py

@@ -32,6 +32,7 @@ def _transform(self, reg, version):
                           text=reg['text'],
                           title=reg.get('title', ''),
                           node_type=reg['node_type'],
+                          root=(len(reg['label']) == 1),
                           children=reg['children'])
 
     def bulk_put(self, regs, version, root_label):
@@ -42,11 +43,17 @@ def bulk_put(self, regs, version, root_label):
         Regulation.objects.bulk_create(map(
             lambda r: self._transform(r, version), regs), batch_size=100)
 
-    def listing(self, label):
-        """List regulation versions that match this label"""
-        query = Regulation.objects.filter(label_string=label).only('version')
-        query = query.order_by('version')
-        versions = [v for v, in query.values_list('version')]  # Flattens
+    def listing(self, label=None):
+        """List regulation version-label pairs that match this label (or are
+        root, if label is None)"""
+        if label is None:
+            query = Regulation.objects.filter(root=True)
+        else:
+            query = Regulation.objects.filter(label_string=label)
+
+        query = query.only('version', 'label_string').order_by('version')
+        # Flattens
+        versions = [v for v in query.values_list('version', 'label_string')]
         return versions
 
 

regcore/db/es.py

@@ -33,20 +33,25 @@ def _transform(self, reg, version):
         node['label_string'] = '-'.join(node['label'])
         node['regulation'] = node['label'][0]
         node['id'] = version + '/' + node['label_string']
+        node['root'] = len(node['label']) == 1
         return node
 
     def bulk_put(self, regs, version, root_label):
         """Store all reg objects"""
         self.es.bulk_index(settings.ELASTIC_SEARCH_INDEX, 'reg_tree',
                            map(lambda r: self._transform(r, version), regs))
 
-    def listing(self, label):
-        """List regulation versions that match this label"""
-        query = {'match': {'label_string': label}}
-        query = {'fields': ['version'], 'query': query}
+    def listing(self, label=None):
+        """List regulation version-label pairs that match this label (or are
+        root, if label is None)"""
+        if label is None:
+            query = {'match': {'root': True}}
+        else:
+            query = {'match': {'label_string': label}}
+        query = {'fields': ['label_string', 'version'], 'query': query}
         result = self.es.search(query, index=settings.ELASTIC_SEARCH_INDEX,
                                 doc_type='reg_tree', size=100)
-        return sorted(res['fields']['version']
+        return sorted((res['fields']['version'], res['fields']['label_string'])
                       for res in result['hits']['hits'])
 
 

regcore/migrations/0003_index_charset.py

@@ -44,3 +44,43 @@ def forwards(self, orm):
             self._longtext(table, 'text')
             self._longtext(table, 'title')
             self._longtext(table, 'children')
+
+    models = {
+        u'regcore.diff': {
+            'Meta': {'unique_together': "(('label', 'old_version', 'new_version'),)", 'object_name': 'Diff', 'index_together': "(('label', 'old_version', 'new_version'),)"},
+            'diff': ('regcore.fields.CompressedJSONField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'new_version': ('django.db.models.fields.SlugField', [], {'max_length': '20'}),
+            'old_version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        },
+        u'regcore.layer': {
+            'Meta': {'unique_together': "(('version', 'name', 'label'),)", 'object_name': 'Layer', 'index_together': "(('version', 'name', 'label'),)"},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'layer': ('regcore.fields.CompressedJSONField', [], {}),
+            'name': ('django.db.models.fields.SlugField', [], {'max_length': '20'}),
+            'version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        },
+        u'regcore.notice': {
+            'Meta': {'object_name': 'Notice'},
+            'cfr_part': ('django.db.models.fields.SlugField', [], {'max_length': '10'}),
+            'document_number': ('django.db.models.fields.SlugField', [], {'max_length': '20', 'primary_key': 'True'}),
+            'effective_on': ('django.db.models.fields.DateField', [], {'null': 'True'}),
+            'fr_url': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'notice': ('regcore.fields.CompressedJSONField', [], {}),
+            'publication_date': ('django.db.models.fields.DateField', [], {})
+        },
+        u'regcore.regulation': {
+            'Meta': {'unique_together': "(('version', 'label_string'),)", 'object_name': 'Regulation', 'index_together': "(('version', 'label_string'),)"},
+            'children': ('regcore.fields.CompressedJSONField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label_string': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'node_type': ('django.db.models.fields.SlugField', [], {'max_length': '10'}),
+            'text': ('django.db.models.fields.TextField', [], {}),
+            'title': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        }
+    }
+
+    complete_apps = ['regcore']

regcore/migrations/0004_auto__add_field_regulation_root.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Regulation.root'
+        db.add_column(u'regcore_regulation', 'root',
+                      self.gf('django.db.models.fields.BooleanField')(default=False, db_index=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'Regulation.root'
+        db.delete_column(u'regcore_regulation', 'root')
+
+
+    models = {
+        u'regcore.diff': {
+            'Meta': {'unique_together': "(('label', 'old_version', 'new_version'),)", 'object_name': 'Diff', 'index_together': "(('label', 'old_version', 'new_version'),)"},
+            'diff': ('regcore.fields.CompressedJSONField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'new_version': ('django.db.models.fields.SlugField', [], {'max_length': '20'}),
+            'old_version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        },
+        u'regcore.layer': {
+            'Meta': {'unique_together': "(('version', 'name', 'label'),)", 'object_name': 'Layer', 'index_together': "(('version', 'name', 'label'),)"},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'layer': ('regcore.fields.CompressedJSONField', [], {}),
+            'name': ('django.db.models.fields.SlugField', [], {'max_length': '20'}),
+            'version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        },
+        u'regcore.notice': {
+            'Meta': {'object_name': 'Notice'},
+            'cfr_part': ('django.db.models.fields.SlugField', [], {'max_length': '10'}),
+            'document_number': ('django.db.models.fields.SlugField', [], {'max_length': '20', 'primary_key': 'True'}),
+            'effective_on': ('django.db.models.fields.DateField', [], {'null': 'True'}),
+            'fr_url': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'notice': ('regcore.fields.CompressedJSONField', [], {}),
+            'publication_date': ('django.db.models.fields.DateField', [], {})
+        },
+        u'regcore.regulation': {
+            'Meta': {'unique_together': "(('version', 'label_string'),)", 'object_name': 'Regulation', 'index_together': "(('version', 'label_string'),)"},
+            'children': ('regcore.fields.CompressedJSONField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'label_string': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'node_type': ('django.db.models.fields.SlugField', [], {'max_length': '10'}),
+            'root': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'text': ('django.db.models.fields.TextField', [], {}),
+            'title': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'version': ('django.db.models.fields.SlugField', [], {'max_length': '20'})
+        }
+    }
+
+    complete_apps = ['regcore']

regcore/models.py

@@ -10,6 +10,7 @@ class Regulation(models.Model):
     title = models.TextField(blank=True)
     node_type = models.SlugField(max_length=10)
     children = CompressedJSONField()
+    root = models.BooleanField(default=False, db_index=True)
 
     class Meta:
         index_together = (('version', 'label_string'),)

regcore/tests/db_django_models_tests.py

@@ -29,7 +29,18 @@ def test_listing(self):
                    node_type='ty', children=[]).save()
 
         results = self.dmr.listing('a-b')
-        self.assertEqual(['333', 'aaa', 'four', 'ver1'], results)
+        self.assertEqual([('333', 'a-b'), ('aaa', 'a-b'), ('four', 'a-b'),
+                          ('ver1', 'a-b')], results)
+
+        Regulation(version='ver1', label_string='1111', text='aaaa',
+                   node_type='ty', root=True, children=[]).save()
+        Regulation(version='ver2', label_string='1111', text='bbbb',
+                   node_type='ty', root=True, children=[]).save()
+        Regulation(version='ver3', label_string='1111', text='cccc',
+                   node_type='ty', root=False, children=[]).save()
+
+        results = self.dmr.listing()
+        self.assertEqual([('ver1', '1111'), ('ver2', '1111')], results)
 
 
 class DMRegulationsTest(TestCase, ReusableDMRegulations):
@@ -38,30 +49,43 @@ def setUp(self):
         self.dmr = DMRegulations()
 
     def test_bulk_put(self):
-        nodes = [
-            {'text': 'some text', 'label': ['111', '2'], 'children': [],
-             'node_type': 'tyty'},
-            {'text': 'other', 'label': ['111', '3'], 'children': [],
-             'node_type': 'tyty2'}]
+        n2 = {'text': 'some text', 'label': ['111', '2'], 'children': [],
+              'node_type': 'tyty'}
+        n3 = {'text': 'other', 'label': ['111', '3'], 'children': [],
+              'node_type': 'tyty2'}
+        # Use a copy of the children
+        root = {'text': 'root', 'label': ['111'], 'node_type': 'tyty3',
+                'children': [dict(n2), dict(n3)]}
+        nodes = [root, n2, n3]
         self.dmr.bulk_put(nodes, 'verver', '111')
 
         regs = Regulation.objects.all().order_by('text')
 
-        self.assertEqual(2, len(regs))
+        self.assertEqual(3, len(regs))
 
         self.assertEqual('verver', regs[0].version)
         self.assertEqual('111-3', regs[0].label_string)
         self.assertEqual('other', regs[0].text)
         self.assertEqual('', regs[0].title)
         self.assertEqual('tyty2', regs[0].node_type)
         self.assertEqual([], regs[0].children)
+        self.assertFalse(regs[0].root)
 
         self.assertEqual('verver', regs[1].version)
-        self.assertEqual('111-2', regs[1].label_string)
-        self.assertEqual('some text', regs[1].text)
+        self.assertEqual('111', regs[1].label_string)
+        self.assertEqual('root', regs[1].text)
         self.assertEqual('', regs[1].title)
-        self.assertEqual('tyty', regs[1].node_type)
-        self.assertEqual([], regs[1].children)
+        self.assertEqual('tyty3', regs[1].node_type)
+        self.assertEqual(2, len(regs[1].children))
+        self.assertTrue(regs[1].root)
+
+        self.assertEqual('verver', regs[2].version)
+        self.assertEqual('111-2', regs[2].label_string)
+        self.assertEqual('some text', regs[2].text)
+        self.assertEqual('', regs[2].title)
+        self.assertEqual('tyty', regs[2].node_type)
+        self.assertEqual([], regs[2].children)
+        self.assertFalse(regs[2].root)
 
     def test_bulk_put_overwrite(self):
         nodes = [{'text': 'other', 'label': ['111', '3'], 'children': [],

regcore/tests/db_es_tests.py

@@ -32,36 +32,54 @@ def test_get_success(self, es):
     @patch('regcore.db.es.ElasticSearch')
     def test_bulk_put(self, es):
         esr = ESRegulations()
-        nodes = [
-            {'text': 'some text', 'label': ['111', '2'], 'children': []},
-            {'text': 'other', 'label': ['111', '3'], 'children': []}]
+        n2 = {'text': 'some text', 'label': ['111', '2'], 'children': []}
+        n3 = {'text': 'other', 'label': ['111', '3'], 'children': []}
+        # Use a copy of the children
+        root = {'text': 'root', 'label': ['111'], 'children': [dict(n2),
+                                                               dict(n3)]}
+        nodes = [root, n2, n3]
         esr.bulk_put(nodes, 'verver', '111')
         self.assertTrue(es.return_value.bulk_index.called)
         args = es.return_value.bulk_index.call_args[0]
         self.assertEqual(3, len(args))
         self.assertEqual('reg_tree', args[1])
 
-        nodes[0]['version'] = 'verver'
-        nodes[0]['regulation'] = '111'
-        nodes[0]['label_string'] = '111-2'
-        nodes[0]['id'] = 'verver/111-2'
-        nodes[1]['version'] = 'verver'
-        nodes[1]['label_string'] = '111-3'
-        nodes[1]['regulation'] = '111'
-        nodes[1]['id'] = 'verver/111-3'
+        root['version'] = 'verver'
+        root['regulation'] = '111'
+        root['label_string'] = '111'
+        root['id'] = 'verver/111'
+        root['root'] = True
+        n2['version'] = 'verver'
+        n2['regulation'] = '111'
+        n2['label_string'] = '111-2'
+        n2['id'] = 'verver/111-2'
+        n2['root'] = False
+        n3['version'] = 'verver'
+        n3['label_string'] = '111-3'
+        n3['regulation'] = '111'
+        n3['id'] = 'verver/111-3'
+        n3['root'] = False
 
         self.assertEqual(nodes, args[2])
 
     @patch('regcore.db.es.ElasticSearch')
     def test_listing(self, es):
         es.return_value.search.return_value = {'hits': {'hits': [
-            {'fields': {'version': 'ver1'}}, {'fields': {'version': 'aaa'}},
-            {'fields': {'version': '333'}}, {'fields': {'version': 'four'}},
+            {'fields': {'version': 'ver1', 'label_string': 'lll'}},
+            {'fields': {'version': 'aaa', 'label_string': 'lll'}},
+            {'fields': {'version': '333', 'label_string': 'lll'}},
+            {'fields': {'version': 'four', 'label_string': 'lll'}},
         ]}}
         esr = ESRegulations()
         results = esr.listing('lll')
+        self.assertFalse('root' in str(es.return_value.search.call_args[0][0]))
         self.assertTrue('ll' in str(es.return_value.search.call_args[0][0]))
-        self.assertEqual(['333', 'aaa', 'four', 'ver1'], results)
+        self.assertEqual([('333', 'lll'), ('aaa', 'lll'), ('four', 'lll'),
+                          ('ver1', 'lll')], results)
+
+        results = esr.listing()
+        self.assertTrue('root' in str(es.return_value.search.call_args[0][0]))
+        self.assertFalse('ll' in str(es.return_value.search.call_args[0][0]))
 
 
 class ESLayersTest(TestCase):

regcore/tests/db_splitter_tests.py

@@ -51,10 +51,12 @@ def test_bulk_put(self, es):
         nodes[0]['label_string'] = '111-2'
         nodes[0]['regulation'] = '111'
         nodes[0]['id'] = 'verver/111-2'
+        nodes[0]['root'] = False
         nodes[1]['version'] = 'verver'
         nodes[1]['label_string'] = '111-3'
         nodes[1]['regulation'] = '111'
         nodes[1]['id'] = 'verver/111-3'
+        nodes[1]['root'] = False
 
         self.assertEqual(nodes, args[2])
 

regcore/urls.py

@@ -56,6 +56,7 @@ def seg(label):
     by_verb_url(r'^regulation/%s/%s$' % (seg('label_id'), seg('version')),
                 'regulation', mapping['regulation']),
     by_verb_url(r'^notice$', 'notices', mapping['notices']),
+    by_verb_url(r'^regulation$', 'all-reg-versions', mapping['reg-versions']),
     by_verb_url(r'^regulation/%s$' % seg('label_id'),
                 'reg-versions', mapping['reg-versions']),
     by_verb_url(r'^search$', 'search', mapping['search'])

regcore_read/tests/views_regulation_tests.py

@@ -42,7 +42,8 @@ def test_listing(self, db):
             {'document_number': '20', 'effective_on': '2011-11-11'},
             {'document_number': '25', 'effective_on': '2011-11-11'}
         ]
-        db.Regulations.return_value.listing.return_value = ['10', '15', '20']
+        db.Regulations.return_value.listing.return_value = [
+            ('10', 'lablab'), ('15', 'lablab'), ('20', 'lablab')]
 
         response = Client().get(url)
         self.assertEqual(200, response.status_code)
@@ -55,3 +56,31 @@ def test_listing(self, db):
             if ver['version'] == '20' and ver['by_date'] == '2011-11-11':
                 found[2] = True
         self.assertEqual(found, [True, True, True])
+
+    @patch('regcore_read.views.regulation.db')
+    def test_listing_all(self, db):
+        url = '/regulation'
+        db.Notices.return_value.listing.return_value = [
+            {'document_number': '10', 'effective_on': '2010-10-10'},
+            {'document_number': '15', 'effective_on': '2010-10-10'},
+            {'document_number': '12'},
+            {'document_number': '20', 'effective_on': '2011-11-11'},
+            {'document_number': '25', 'effective_on': '2011-11-11'}
+        ]
+        db.Regulations.return_value.listing.return_value = [
+            ('10', '1111'), ('15', '1111'), ('20', '1212')]
+
+        response = Client().get(url)
+        self.assertEqual(200, response.status_code)
+        found = [False, False, False]
+        for ver in json.loads(response.content)['versions']:
+            if (ver['version'] == '10' and 'by_date' not in ver
+                    and ver['regulation'] == '1111'):
+                found[0] = True
+            if (ver['version'] == '15' and ver['by_date'] == '2010-10-10'
+                    and ver['regulation'] == '1111'):
+                found[1] = True
+            if (ver['version'] == '20' and ver['by_date'] == '2011-11-11'
+                    and ver['regulation'] == '1212'):
+                found[2] = True
+        self.assertEqual(found, [True, True, True])