Merge pull request #227 from OnroerendErfgoed/214_collect_usage_stats

214 collect usage stats

atramhasis/alembic/versions/1ad2b6fbcf22_visit_log.py

@@ -0,0 +1,35 @@
+"""visit_log
+
+Revision ID: 1ad2b6fbcf22
+Revises: 441c5a16ef8
+Create Date: 2015-07-27 13:29:04.840631
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1ad2b6fbcf22'
+down_revision = '441c5a16ef8'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('conceptscheme_visit_log',
+                    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
+                    sa.Column('conceptscheme_id', sa.String(), nullable=False),
+                    sa.Column('visited_at', sa.DateTime(), nullable=False),
+                    sa.Column('origin', sa.String, nullable=False)
+                    )
+    op.create_table('concept_visit_log',
+                    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
+                    sa.Column('concept_id', sa.Integer(), nullable=False),
+                    sa.Column('conceptscheme_id', sa.String(), nullable=False),
+                    sa.Column('visited_at', sa.DateTime(), nullable=False),
+                    sa.Column('origin', sa.String, nullable=False)
+                    )
+
+
+def downgrade():
+    op.drop_table('concept_visit_log')
+    op.drop_table('conceptscheme_visit_log')

atramhasis/audit.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+from atramhasis.data.models import (
+    ConceptschemeVisitLog,
+    ConceptVisitLog
+)
+from pyramid.response import Response
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def _origin_from_request(request):
+    if request.url.endswith('.csv'):
+        return 'CSV'
+    elif request.url.endswith('.rdf') \
+            or 'application/rdf+xml' in request.accept \
+            or 'text/turtle' in request.accept \
+            or 'application/x-turtle' in request.accept:
+        return 'RDF'
+    elif 'text/html' in request.accept:
+        return 'HTML'
+    elif 'application/json' in request.accept:
+        return 'REST'
+    else:
+        return None
+
+
+def _origin_from_response(response):
+    if response.content_type == 'text/html':
+        return 'HTML'
+    elif response.content_type == 'application/json':
+        return 'REST'
+    elif response.content_type == 'application/rdf+xml' \
+            or response.content_type == 'text/turtle' \
+            or response.content_type == 'application/x-turtle':
+        return 'RDF'
+    elif response.content_type == 'text/csv':
+        return 'CSV'
+    else:
+        return None
+
+
+def audit(fn):
+    '''
+    use this decorator to audit an operation and to log the visit
+
+    * CSV routes with .csv extensions accept all mime types,
+      the response is not of the `pyramid.response.Response` type,
+      the origin is derived from the `pyramid.request.Request.url` extension.
+    * RDF routes with .rdf, .ttl extensions accept all mime types,
+      the origin is derived form the response content type.
+    * REST and HTML the view results are not of the `pyramid.response.Response` type,
+      the origin is derived from the accept header.
+    '''
+
+    def advice(parent_object, *args, **kw):
+        request = parent_object.request
+        audit_manager = request.data_managers['audit_manager']
+
+        if 'c_id' in request.matchdict.keys():
+            visit_log = ConceptVisitLog(
+                concept_id=request.matchdict['c_id'],
+                conceptscheme_id=request.matchdict['scheme_id']
+            )
+        elif 'scheme_id' not in request.matchdict.keys():
+            log.error('Misuse of the audit decorator. The url must at least contain a {scheme_id} parameter')
+            return fn(parent_object, *args, **kw)
+        else:
+            visit_log = ConceptschemeVisitLog(conceptscheme_id=request.matchdict['scheme_id'])
+        response = fn(parent_object, *args, **kw)
+
+        if isinstance(response, Response):
+            visit_log.origin = _origin_from_response(response)
+        else:
+            visit_log.origin = _origin_from_request(request)
+
+        audit_manager.save(visit_log)
+
+        return response
+
+    return advice

atramhasis/data/datamanagers.py

@@ -213,3 +213,19 @@ def get_all_sorted(self, sort_coll, sort_desc):
 
     def count_languages(self, language_tag):
         return self.session.query(Language).filter_by(id=language_tag).count()
+
+
+class AuditManager(DataManager):
+    '''
+    A data manager for logging the visit.
+    '''
+
+    def save(self, visit_log):
+        '''
+        save a certain visit
+        :param visit_log: log of visit to save
+        :return: The saved visit log
+        '''
+        self.session.add(visit_log)
+        self.session.flush()
+        return visit_log

atramhasis/data/db.py

@@ -2,7 +2,7 @@
 '''
 Module that sets up the datamanagers and the database connections.
 '''
-from atramhasis.data.datamanagers import SkosManager, ConceptSchemeManager, LanguagesManager
+from atramhasis.data.datamanagers import SkosManager, ConceptSchemeManager, LanguagesManager, AuditManager
 from .models import Base
 from skosprovider_sqlalchemy.models import Base as SkosBase
 
@@ -24,13 +24,14 @@ def data_managers(request):
     skos_manager = SkosManager(session)
     conceptscheme_manager = ConceptSchemeManager(session)
     languages_manager = LanguagesManager(session)
+    audit_manager = AuditManager(session)
 
     def cleanup(request):
         session.close()
     request.add_finished_callback(cleanup)
 
     return {'skos_manager': skos_manager, 'conceptscheme_manager': conceptscheme_manager,
-            'languages_manager': languages_manager}
+            'languages_manager': languages_manager, 'audit_manager': audit_manager}
 
 
 def includeme(config):

atramhasis/data/models.py

@@ -1,3 +1,28 @@
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime)
+from sqlalchemy.sql import (
+    func
+)
 
-Base = declarative_base()
+Base = declarative_base()
+
+
+class ConceptschemeVisitLog(Base):
+    __tablename__ = 'conceptscheme_visit_log'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    conceptscheme_id = Column(String(25), nullable=False)
+    visited_at = Column(DateTime, default=func.now(), nullable=False)
+    origin = Column(String(25), nullable=False)
+
+
+class ConceptVisitLog(Base):
+    __tablename__ = 'concept_visit_log'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    concept_id = Column(Integer, nullable=False)
+    conceptscheme_id = Column(String(25), nullable=False)
+    visited_at = Column(DateTime, default=func.now(), nullable=False)
+    origin = Column(String(25), nullable=False)

atramhasis/routes.py

@@ -30,6 +30,8 @@ def includeme(config):
     config.add_route('scheme_root', pattern='/conceptschemes/{scheme_id}/c/', accept='text/html')
     config.add_route('scheme_tree', pattern='/conceptschemes/{scheme_id}/tree', accept='application/json')
     config.add_route('search_result_export', pattern='/conceptschemes/{scheme_id}/c.csv')
+    config.add_route('atramhasis.get_conceptscheme', pattern='/conceptschemes/{scheme_id}', accept='application/json',
+                     request_method="GET")
     config.add_route('atramhasis.get_concept', pattern='/conceptschemes/{scheme_id}/c/{c_id}',
                      accept='application/json', request_method="GET")
     config.add_route('atramhasis.add_concept', pattern='/conceptschemes/{scheme_id}/c', accept='application/json',

atramhasis/views/crud.py

@@ -16,6 +16,7 @@
 from atramhasis.protected_resources import protected_operation
 from atramhasis.utils import from_thing, internal_providers_only
 from atramhasis.views import invalidate_scheme_cache
+from atramhasis.audit import audit
 
 
 @view_defaults(accept='application/json', renderer='skosrenderer_verbose')
@@ -64,6 +65,14 @@ def _validate_concept(self, json_concept, conceptscheme_id):
                 e.asdict()
             )
 
+    @audit
+    @view_config(route_name='atramhasis.get_conceptscheme', permission='view')
+    def get_conceptscheme(self):
+        # is the same as the pyramid_skosprovider get_conceptscheme function, but wrapped with the audit function
+        from pyramid_skosprovider.views import ProviderView
+        return ProviderView(self.request).get_conceptscheme()
+
+    @audit
     @view_config(route_name='atramhasis.get_concept', permission='view')
     def get_concept(self):
         '''

atramhasis/views/rdf.py

@@ -3,6 +3,7 @@
 from skosprovider_rdf import utils
 
 from atramhasis.errors import SkosRegistryNotFoundException, ConceptSchemeNotFoundException
+from atramhasis.audit import audit
 
 
 @view_defaults()
@@ -21,6 +22,7 @@ def __init__(self, request):
         if not self.provider:
             raise ConceptSchemeNotFoundException(self.scheme_id)   # pragma: no cover
 
+    @audit
     @view_config(route_name='atramhasis.rdf_full_export')
     @view_config(route_name='atramhasis.rdf_full_export_ext')
     def rdf_full_export(self):
@@ -30,6 +32,7 @@ def rdf_full_export(self):
         response.content_disposition = 'attachment; filename="%s-full.rdf"' % (str(self.scheme_id),)
         return response
 
+    @audit
     @view_config(route_name='atramhasis.rdf_full_export_turtle')
     @view_config(route_name='atramhasis.rdf_full_export_turtle_x')
     @view_config(route_name='atramhasis.rdf_full_export_turtle_ext')
@@ -40,6 +43,7 @@ def rdf_full_export_turtle(self):
         response.content_disposition = 'attachment; filename="%s-full.ttl"' % (str(self.scheme_id),)
         return response
 
+    @audit
     @view_config(route_name='atramhasis.rdf_conceptscheme_export')
     @view_config(route_name='atramhasis.rdf_conceptscheme_export_ext')
     def rdf_conceptscheme_export(self):
@@ -59,6 +63,7 @@ def rdf_conceptscheme_export_turtle(self):
         response.content_disposition = 'attachment; filename="%s.ttl"' % (str(self.scheme_id),)
         return response
 
+    @audit
     @view_config(route_name='atramhasis.rdf_individual_export')
     @view_config(route_name='atramhasis.rdf_individual_export_ext')
     def rdf_individual_export(self):
@@ -68,6 +73,7 @@ def rdf_individual_export(self):
         response.content_disposition = 'attachment; filename="%s.rdf"' % (str(self.c_id),)
         return response
 
+    @audit
     @view_config(route_name='atramhasis.rdf_individual_export_turtle')
     @view_config(route_name='atramhasis.rdf_individual_export_turtle_x')
     @view_config(route_name='atramhasis.rdf_individual_export_turtle_ext')

atramhasis/views/views.py

@@ -11,6 +11,7 @@
 
 from atramhasis.errors import SkosRegistryNotFoundException, ConceptSchemeNotFoundException, ConceptNotFoundException
 from atramhasis.views import tree_region, invalidate_scheme_cache, invalidate_cache
+from atramhasis.audit import audit
 
 
 def labels_to_string(labels, ltype):
@@ -95,6 +96,7 @@ def conceptschemes_view(self):
 
         return {'conceptschemes': conceptschemes}
 
+    @audit
     @view_config(route_name='conceptscheme', renderer='atramhasis:templates/conceptscheme.jinja2')
     def conceptscheme_view(self):
         '''
@@ -118,6 +120,7 @@ def conceptscheme_view(self):
 
         return {'conceptscheme': scheme}
 
+    @audit
     @view_config(route_name='concept', renderer='atramhasis:templates/concept.jinja2')
     def concept_view(self):
         '''
@@ -191,6 +194,7 @@ def set_locale_cookie(self):
                             max_age=31536000)  # max_age = year
         return response
 
+    @audit
     @view_config(route_name='search_result_export', renderer='csv')
     def results_csv(self):
         header = ['conceptscheme', 'id', 'uri', 'type', 'label', 'prefLabels', 'altLabels', 'definition', 'broader',
@@ -285,7 +289,6 @@ def create_treeid(self, parent_tree_id, concept_id):
         else:
             return parent_tree_id + "." + str(concept_id)
 
-
     @view_config(route_name='scheme_root', renderer='atramhasis:templates/concept.jinja2')
     def results_tree_html(self):
         scheme_id = self.request.matchdict['scheme_id']

requirements-dev.txt

@@ -14,6 +14,7 @@ py==1.4.26
 coveralls==0.4.4
 webtest==2.0.16
 mock==1.0.1
+testfixtures==4.1.2
 
 # Documentation
 Sphinx==1.2.3