From 3267e29a04c73a10df014c434997cea084596125 Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 11:19:49 +0300 Subject: [PATCH 1/7] inital refactor of settings --- pip-requires.txt | 1 + ureport_project/manage.py | 18 +- ureport_project/settings/__init__.py | 0 .../{settings.py => settings/base.py} | 317 +++++++++++++----- ureport_project/settings/dev.py | 11 + ureport_project/settings/production.py | 79 +++++ 6 files changed, 334 insertions(+), 92 deletions(-) create mode 100644 ureport_project/settings/__init__.py rename ureport_project/{settings.py => settings/base.py} (52%) create mode 100644 ureport_project/settings/dev.py create mode 100644 ureport_project/settings/production.py diff --git a/pip-requires.txt b/pip-requires.txt index 70482d7..20bbc27 100644 --- a/pip-requires.txt +++ b/pip-requires.txt @@ -53,3 +53,4 @@ xlrd==0.9.2 xlutils==1.6.0 xlwt==0.7.5 requests==2.0.1 +raven==4.0.2 diff --git a/ureport_project/manage.py b/ureport_project/manage.py index 3ca05ca..b2e1087 100755 --- a/ureport_project/manage.py +++ b/ureport_project/manage.py @@ -1,7 +1,17 @@ #!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8 +import os +import sys -from django.core.management import execute_manager -import settings if __name__ == "__main__": - execute_manager(settings) + """ + you can add something like this here: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.production") + + but its better to set the environment variable with a deploy script + + and just pass --settings=settings.dev to manage.py when running locally + """ + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/ureport_project/settings/__init__.py b/ureport_project/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ureport_project/settings.py b/ureport_project/settings/base.py similarity index 52% rename from ureport_project/settings.py rename to ureport_project/settings/base.py index 86d19e5..cd8f75d 100644 --- a/ureport_project/settings.py +++ b/ureport_project/settings/base.py @@ -7,46 +7,55 @@ # PATH CONFIGURATION # # -------------------------------------------------------------------- # -import sys, os - - -filedir = os.path.dirname(__file__) -sys.path.append(os.path.join(filedir)) -sys.path.append(os.path.join(filedir, 'rapidsms', 'lib')) -sys.path.append(os.path.join(filedir, 'rapidsms_auth')) -sys.path.append(os.path.join(filedir, 'rapidsms_contact')) -sys.path.append(os.path.join(filedir, 'rapidsms_cvs')) -sys.path.append(os.path.join(filedir, 'rapidsms_generic')) -sys.path.append(os.path.join(filedir, 'rapidsms_geoserver')) -sys.path.append(os.path.join(filedir, 'rapidsms_httprouter_src')) -sys.path.append(os.path.join(filedir, 'rapidsms_polls')) -sys.path.append(os.path.join(filedir, 'rapidsms_script')) -sys.path.append(os.path.join(filedir, 'rapidsms_uregister')) -sys.path.append(os.path.join(filedir, 'rapidsms_ureport')) -sys.path.append(os.path.join(filedir, 'rapidsms_uganda_common')) -sys.path.append(os.path.join(filedir, 'rapidsms_unregister')) -sys.path.append(os.path.join(filedir, 'rapidsms_xforms_src')) -sys.path.append(os.path.join(filedir, 'rapidsms_tracking')) -sys.path.append(os.path.join(filedir, 'django_eav')) -sys.path.append(os.path.join(filedir, 'rapidsms_uganda_ussd')) -sys.path.append(os.path.join(filedir, 'rapidsms_message_classifier')) - -sys.path.append(os.path.join(filedir, 'qos_monitor')) +import os +import sys + + +PROJECT_PATH = os.path.join(os.path.dirname(__file__), os.pardir) + +sys.path.append(os.path.join(PROJECT_PATH)) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms', 'lib')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_auth')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_contact')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_cvs')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_generic')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_geoserver')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_httprouter_src')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_polls')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_script')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_uregister')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_ureport')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_uganda_common')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_unregister')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_xforms_src')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_tracking')) +sys.path.append(os.path.join(PROJECT_PATH, 'django_eav')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_uganda_ussd')) +sys.path.append(os.path.join(PROJECT_PATH, 'rapidsms_message_classifier')) +sys.path.append(os.path.join(PROJECT_PATH, 'qos_monitor')) gettext = lambda s: s # -------------------------------------------------------------------- # # MAIN CONFIGURATION # # -------------------------------------------------------------------- # -BAD_WORDS = ['poop', ] #list of profanities +BAD_WORDS = ['poop', ] # list of profanities TIME_ZONE = "Africa/Kampala" EMAIL_HOST_USER = '' EMAIL_HOST = '127.0.0.1' OPT_IN_WORDS = ['join'] -OPT_IN_WORDS_LUO=["donyo","dony","donyo","doyo",] -OPT_IN_CONFIRMATION = gettext("Welcome to UReport! Ureport is a community of %(citizen)s youth that are dedicated to working for positive change in their communities. Stay tuned for more info.") % {"citizen": gettext("Ugandan")} -OPT_OUT_WORDS = ['stop', 'unjoin', 'quit','giki'] -OPT_OUT_CONFIRMATION = gettext("Your UReport opt out is confirmed.If you made a mistake,or you want your voice to be heard again,text in JOIN and send it to 8500!All SMS messages are free") +OPT_IN_WORDS_LUO = ["donyo", "dony", "donyo", "doyo"] +OPT_IN_CONFIRMATION = gettext("Welcome to UReport! Ureport is a community of " + "%(citizen)s youth that are dedicated to " + "working for positive change in their " + "communities. Stay tuned for more info.")\ + % {"citizen": gettext("Ugandan")} + +OPT_OUT_WORDS = ['stop', 'unjoin', 'quit', 'giki'] +OPT_OUT_CONFIRMATION = gettext("Your UReport opt out is confirmed.If you " + "made a mistake,or you want your voice to be " + "heard again,text in JOIN and send it to " + "8500!All SMS messages are free") # map bounding box MIN_LON = '29.55322265625' @@ -54,7 +63,8 @@ MIN_LAT = '-1.0326589311777759' MAX_LAT = '4.280680030820496' # map categorized color pallete -CATEGORY_COLORS = ['#AA4643', '#4572A7', '#89A54E', '#80699B', '#3D96AE', '#DB843D', '#92A8CD', '#A47D7C', '#B5CA92'] +CATEGORY_COLORS = ['#AA4643', '#4572A7', '#89A54E', '#80699B', '#3D96AE', + '#DB843D', '#92A8CD', '#A47D7C', '#B5CA92'] ADMINS = ( ) @@ -70,13 +80,13 @@ # see: http://docs.djangoproject.com/en/dev/ref/settings/#databases DATABASES = { 'default': { - 'ENGINE' : 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'ureport', - 'HOST': 'dbserver', 'USER': 'postgres', } } + # the rapidsms backend configuration is designed to resemble django's # database configuration, as a nested dict of (name, configuration). # @@ -136,9 +146,9 @@ "celery", "djcelery", #"permission", - # nothing after south + # nothing after south "south", - "django_nose" #Except nose must come after south so that it runs migrations properly!! + "django_nose" # Except nose must come after south so that migrations run!! ] SMS_APPS = [ @@ -154,17 +164,17 @@ # tabbed navigation. when adding an app to INSTALLED_APPS, you may wish # to add it here, also, to expose it in the rapidsms ui. RAPIDSMS_TABS = [ - ("rapidsms-dashboard", "Home"), - ("ureport-about", "About"), - ("polls-summary", "Polls"), - ("ureport-stories", "Stories"), + ("rapidsms-dashboard", "Home"), + ("ureport-about", "About"), + ("polls-summary", "Polls"), + ("ureport-stories", "Stories"), ] AUTHENTICATED_TABS = [ ("ureport-polls", "Poll Admin"), ("messagelog", "Message Log"), ("ureport-contact", "uReporters"), - ("flaggedmessages", "Flagged Messages"), + ("flaggedmessages", "Flagged Messages"), ] # -------------------------------------------------------------------- # @@ -172,11 +182,6 @@ # -------------------------------------------------------------------- # -# debug mode is turned on as default, since rapidsms is under heavy -# development at the moment, and full stack traces are very useful -# when reporting bugs. don't forget to turn this off in production. -DEBUG = TEMPLATE_DEBUG = True - # after login (which is handled by django.contrib.auth), redirect to the # dashboard rather than 'accounts/profile' (the default). LOGIN_REDIRECT_URL = "/" @@ -198,12 +203,12 @@ # see: http://docs.djangoproject.com/en/dev/ref/contrib/sites/ SITE_ID = 1 -# this is used for geoserver to tell which website this viz should be for (and prevents clashing of -# polls across different websites with the same id +# this is used for geoserver to tell which website this viz should be for +# (and prevents clashing of polls across different websites with the same id) DEPLOYMENT_ID = 1 #model containing blacklisted contacts -BLACKLIST_MODEL= "unregister.Blacklist" +BLACKLIST_MODEL = "unregister.Blacklist" # these weird dependencies should be handled by their respective apps, # but they're not, so here they are. most of them are for django admin. @@ -220,20 +225,174 @@ MIDDLEWARE_CLASSES = ( 'ureport.middleware.access_log.UreportAccessLogMiddleware', - 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.common.CommonMiddleware', - 'django.middleware.cache.FetchFromCacheMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.middleware.cache.CacheMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.transaction.TransactionMiddleware', - #'tracking.middleware.UserTrackingMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - # "ureport.middleware.permissions.RequirePermissionMiddleware", - "uganda_common.middleware.accessmiddleware.accessmiddleware.AccessMiddleWare" + 'uganda_common.middleware.accessmiddleware.accessmiddleware.AccessMiddleWare', + #'tracking.middleware.UserTrackingMiddleware', + #'ureport.middleware.permissions.RequirePermissionMiddleware', ) +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + 'handlers': { + 'null': { + 'level': 'DEBUG', + 'class': 'django.utils.log.NullHandler', + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'mail_admins': { + 'level': 'ERROR', + 'class': 'django.utils.log.AdminEmailHandler' + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': True, + }, + } +} + +# -------------------------------------------------------------------- # +# CELERY CONFIGURATION # +# -------------------------------------------------------------------- # +#BROKER_URL = 'amqp://guest:guest@localhost:5672//' +#CELERY_ALWAYS_EAGER = True + +CELERY_RESULT_BACKEND = 'amqp' +CELERY_RESULT_PERSISTENT = False +CELERY_TASK_RESULT_EXPIRES = 10 +CELERY_DISABLE_RATE_LIMITS = True +CELERY_SEND_TASK_ERROR_EMAILS = True + +CELERY_DEFAULT_QUEUE = 'default' +CELERY_QUEUES = { + 'default': { + 'binding_key': 'task.#', + }, + 'upload_responses': { + 'binding_key': 'upload_responses.#', + }, + 'classify_excel': { + 'binding_key': 'classify_excel.#', + }, + 'message_export': { + 'binding_key': 'message_export.#', + }, + 'start_poll': { + 'binding_key': 'start_poll.#', + }, + 'process_message': { + 'binding_key': 'process_message.#', + }, + 'handle_incoming': { + 'binding_key': 'handle_incoming.#', + }, + 'reprocess_responses': { + 'binding_key': 'reprocess_responses.#', + }, + 'push_to_mtrac': { + 'binding_key': 'push_to_mtrac.#' + }, + 'process_uploaded_contacts': { + 'binding_key': 'process_uploaded_contacts.#' + }, + 'process_assign_group': { + 'binding_key': 'process_assign_group.#' + }, + 'export_poll': { + 'binding_key': 'export_poll.#' + }, + 'extract_gen_reports': { + 'binding_key': 'extract_gen_reports.#' + } +} +CELERY_DEFAULT_EXCHANGE = 'tasks' +CELERY_DEFAULT_EXCHANGE_TYPE = 'topic' +CELERY_DEFAULT_ROUTING_KEY = 'task.default' +#CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler" + +BROKER_HOST = "127.0.0.1" +BROKER_PORT = 5672 +BROKER_USER = "ureport" +BROKER_PASSWORD = "ureport" +BROKER_VHOST = "ureport" + +CELERY_ROUTES = { + 'message_classifier.tasks.upload_responses': { + 'queue': 'upload_responses', + 'routing_key': 'upload_responses.result' + }, + 'message_classifier.tasks.classify_excel': { + 'queue': 'classify_excel', + 'routing_key': 'classify_excel.result' + }, + 'message_classifier.tasks.message_export': { + 'queue': 'message_export', + 'routing_key': 'message_export.result' + }, + 'ureport.tasks.start_poll': { + 'queue': 'start_poll', + 'routing_key': 'start_poll.result' + }, + 'ureport.tasks.process_message': { + 'queue': 'process_message', + 'routing_key': 'process_message.result' + }, + 'rapidsms_httprouter.tasks.handle_incoming': { + 'queue': 'handle_incoming', + 'routing_key': 'handle_incoming.result' + }, + 'ureport.tasks.reprocess_responses': { + 'queue': 'reprocess_responses', + 'routing_key': 'reprocess_responses.result' + }, + 'ureport.tasks.push_to_mtrac': { + 'queue': 'push_to_mtrac', + 'routing_key': 'push_to_mtrac.result' + }, + 'ureport.tasks.process_uploaded_contacts': { + 'queue': 'process_uploaded_contacts', + 'routing_key': 'process_uploaded_contacts.result' + }, + 'ureport.tasks.process_assign_group': { + 'queue': 'process_assign_group', + 'routing_key': 'process_assign_group.result' + }, + 'ureport.tasks.export_poll': { + 'queue': 'export_poll', + 'routing_key': 'export_poll.result' + }, + 'ureport.tasks.extract_gen_reports': { + 'queue': 'extract_gen_reports', + 'routing_key': 'extract_gen_reports.result' + } + +} + +SERVER_EMAIL = 'Ureport Background Tasks' + +CELERY_IMPORTS = ('ureport.tasks', 'message_classifier.tasks', + 'rapidsms_httprouter.tasks') + # -------------------------------------------------------------------- # # HERE BE DRAGONS! # # these settings are pure hackery, and will go away soon # @@ -260,22 +419,6 @@ # the project-level url patterns ROOT_URLCONF = "urls" -#caching stuff -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:11211', - 'TIMEOUT': 6000, - 'OPTIONS': { - 'MAX_ENTRIES': 1000 - } - } -} - -CACHE_MIDDLEWARE_ALIAS = 'default' -CACHE_MIDDLEWARE_SECONDS = 2 * 60 * 60 -CACHE_MIDDLEWARE_KEY_PREFIX = '' - ALLOWED = ( r'/accounts/login(.*)$', r'/accounts/logout(.*)$', @@ -284,8 +427,7 @@ r'/about/$', r'/signup/$', r'^/bestviz(.*)' - - ) +) #AUTHENTICATION_BACKENDS = ( # 'django.contrib.auth.backends.ModelBackend', @@ -294,7 +436,7 @@ # ) USE_I18N = True -INITIAL_USSD_SCREEN='ussd_root' +INITIAL_USSD_SCREEN = 'ussd_root' MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' @@ -307,13 +449,17 @@ ## http://hustoknow.blogspot.com/2011/02/setting-up-django-nose-on-hudson.html # https://nose.readthedocs.org/en/latest/plugins/cover.html # this is supposed to exclude a dir '--exclude-dir=', -NOSE_ARGS = ('--with-xunit', '--xunit-file=target/reports/unit-test/nosetests.ureport.xml', - '--with-coverage', '--cover-html', '--cover-html-dir=target/reports/unit-test/coverage', '--cover-package=poll,uganda_common,unregister,message_classifier,contact,rapidsms_httprouter,rapidsms_ureport', - '--verbosity=2') - -MAP_BOUNDS = {'bottom_left': {'lon': 28.6963, 'lat': -1.8015}, - 'top_right': {'lon': 35.8592, 'lat': 4.9} - } +NOSE_ARGS = ('--with-xunit', '--with-coverage', + '--cover-html', '--verbosity=2', + '--xunit-file=target/reports/unit-test/nosetests.ureport.xml', + '--cover-html-dir=target/reports/unit-test/coverage', + '--cover-package=poll,uganda_common,unregister,message_classifier,contact,rapidsms_httprouter,rapidsms_ureport', + ) + +MAP_BOUNDS = { + 'bottom_left': {'lon': 28.6963, 'lat': -1.8015}, + 'top_right': {'lon': 35.8592, 'lat': 4.9} +} #MAP_BOUNDS SHOULD MOVE INTO HERE (BEFORE GEOSERVER IS KILLED) MAP_ARGS = {'center': {'longitude': 32.27, 'latitude': 0.95}, @@ -321,20 +467,17 @@ } COUNTRY_SPECIFIC_TOKENS = { - "district" : "district" + "district": "district" } LOCALE_PATHS = ( - os.path.join(filedir, 'locale'), + os.path.join(PROJECT_PATH, 'locale'), ) # since we might hit the database from any thread during testing, the # in-memory sqlite database isn't sufficient. it spawns a separate # virtual database for each thread, and syncdb is only called for the # first. this leads to confusing "no such table" errors. We create # a named temporary instance instead. -import os -import tempfile -import sys PAGINATION_LIMIT = None @@ -348,7 +491,7 @@ } try: - if os.environ.has_key('LOCAL_SETTINGS'): + if 'LOCAL_SETTINGS' in os.environ: # the LOCAL_SETTINGS environment variable is used by the build server sys.path.insert(0, os.path.dirname(os.environ['LOCAL_SETTINGS'])) from settings_test import * @@ -358,5 +501,3 @@ pass # if 'test' in sys.argv: # DATABASES['default'] = {'ENGINE': 'django.db.backends.sqlite3'} - - diff --git a/ureport_project/settings/dev.py b/ureport_project/settings/dev.py new file mode 100644 index 0000000..4dc35a9 --- /dev/null +++ b/ureport_project/settings/dev.py @@ -0,0 +1,11 @@ +from .base import * + +try: + # attempt to load local settings, if they exist + from .local import * +except: + pass + +DEBUG = TEMPLATE_DEBUG = False + +GEOSERVER_URL = '' diff --git a/ureport_project/settings/production.py b/ureport_project/settings/production.py new file mode 100644 index 0000000..6a86968 --- /dev/null +++ b/ureport_project/settings/production.py @@ -0,0 +1,79 @@ +import copy +from .base import * + +DEBUG = TEMPLATE_DEBUG = False +GEOSERVER_URL = '' + +# -------------------------------------------------------------------- # +# CACHE CONFIGURATION # +# -------------------------------------------------------------------- # +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': '127.0.0.1:11211', + 'TIMEOUT': 6000, + 'OPTIONS': { + 'MAX_ENTRIES': 1000 + } + } +} + +CACHE_MIDDLEWARE_ALIAS = 'default' +CACHE_MIDDLEWARE_SECONDS = 2 * 60 * 60 +CACHE_MIDDLEWARE_KEY_PREFIX = '' + +CACHING_MIDDLEWARES = ( + 'django.middleware.cache.CacheMiddleware', + 'django.middleware.cache.FetchFromCacheMiddleware', + 'django.middleware.cache.UpdateCacheMiddleware', +) + +MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + CACHING_MIDDLEWARES + + +# -------------------------------------------------------------------- # +# SENTRY CONFIGURATION # +# -------------------------------------------------------------------- # +LOGGING['handlers']['sentry'] = { + 'level': 'ERROR', + 'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler', +} +LOGGING['root'] = { + 'level': 'WARNING', + 'handlers': ['sentry'], +} +LOGGING['loggers']['raven'] = { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, +} +LOGGING['loggers']['sentry.errors'] = { + 'level': 'DEBUG', + 'handlers': ['console'], + 'propagate': False, +} + +# raven docs say to put SentryResponseErrorIdMiddleware +# 'as high in the chain as possible' +# so this will insert raven into the top of the base +# settings.py file's MIDDLEWARE_CLASSES +TEMP = list(copy.copy(MIDDLEWARE_CLASSES)) +TEMP.insert(0, 'raven.contrib.django.raven_compat.' + 'middleware.SentryResponseErrorIdMiddleware') +TEMP.append('raven.contrib.django.raven_compat.' + 'middleware.Sentry404CatchMiddleware') +MIDDLEWARE_CLASSES = tuple(TEMP) + +INSTALLED_APPS += ("raven.contrib.django.raven_compat",) + +RAVEN_CONFIG = { + 'dsn': 'https://fa310d8c078c40cb9881fdb787ba21c1:' + '42d834d6923740ee88e1027e7e213941@sentry.unicefuganda.org/2', +} + +SENTRY_URL = "https://sentry.unicefuganda.org/t4d/ureport/" + +CELERY_QUEUES["sentry"] = { + "exchange": "default", + "binding_key": "sentry" +} From 93c84ffc81021016d19a44a8f9be324e61dbf1ec Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:00:44 +0300 Subject: [PATCH 2/7] update sentry conf --- ureport_project/settings/base.py | 3 +++ ureport_project/settings/production.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ureport_project/settings/base.py b/ureport_project/settings/base.py index cd8f75d..bccaead 100644 --- a/ureport_project/settings/base.py +++ b/ureport_project/settings/base.py @@ -274,6 +274,9 @@ # -------------------------------------------------------------------- # # CELERY CONFIGURATION # # -------------------------------------------------------------------- # +# TODO this duplicates celeryconfig.py! do we need that file? +# if djcelery will pick up celery conf here? + #BROKER_URL = 'amqp://guest:guest@localhost:5672//' #CELERY_ALWAYS_EAGER = True diff --git a/ureport_project/settings/production.py b/ureport_project/settings/production.py index 6a86968..d4ed21f 100644 --- a/ureport_project/settings/production.py +++ b/ureport_project/settings/production.py @@ -66,12 +66,13 @@ INSTALLED_APPS += ("raven.contrib.django.raven_compat",) +# TODO we should support SSL RAVEN_CONFIG = { - 'dsn': 'https://fa310d8c078c40cb9881fdb787ba21c1:' + 'dsn': 'http://fa310d8c078c40cb9881fdb787ba21c1:' '42d834d6923740ee88e1027e7e213941@sentry.unicefuganda.org/2', } -SENTRY_URL = "https://sentry.unicefuganda.org/t4d/ureport/" +SENTRY_URL = "http://sentry.unicefuganda.org/t4d/ureport/" CELERY_QUEUES["sentry"] = { "exchange": "default", From 9e7f103cd9020340976627e06ae28e43f5a2b30e Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:22:12 +0300 Subject: [PATCH 3/7] update logging conf --- ureport_project/settings/base.py | 111 +++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 5 deletions(-) diff --git a/ureport_project/settings/base.py b/ureport_project/settings/base.py index bccaead..5b78279 100644 --- a/ureport_project/settings/base.py +++ b/ureport_project/settings/base.py @@ -252,22 +252,122 @@ 'level': 'DEBUG', 'class': 'django.utils.log.NullHandler', }, + 'command': { + 'class': 'logging.FileHandler', + 'filename': '/var/log/ureport/command.log', + 'formatter': 'simple' + }, 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', 'formatter': 'simple' }, - 'mail_admins': { - 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler' + 'application_log_file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': '/var/log/ureport/ureport_application.log', + 'formatter': 'verbose', + 'backupCount': 50, + 'maxBytes': 2 ** 20, + }, + 'application_access_log_file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': '/var/log/ureport/ureport_access.log', + 'formatter': 'verbose', + 'backupCount': 50, + 'maxBytes': 2 ** 20, }, }, 'loggers': { + 'django': { + 'handlers': ['application_log_file'], + 'level': 'ERROR', + 'propagate': True, + }, + 'ureport': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django.middleware': { + 'handlers': ['application_log_file'], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.db.backends': { + 'handlers': ['application_log_file'], + 'level': 'ERROR', + 'propagate': True, + }, 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['application_log_file'], 'level': 'ERROR', 'propagate': True, }, + 'ureport.app': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'ureport.tasks': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'ureport.views.poll_views': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'ureport.middleware.access_log': { + 'handlers': ['application_access_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'poll.app': { + 'handlers': ['application_log_file'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'poll.models': { + 'handlers': ['application_log_file'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'rapidsms_httprouter.models': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'rapidsms_httprouter.views': { + 'handlers': ['application_log_file'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'rapidsms_httprouter.router': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'unregister': { + 'handlers': ['application_log_file'], + 'level': 'INFO', + 'propagate': True, + }, + 'script.utils.outgoing': { + 'handlers': ['application_log_file'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'script.managers': { + 'handlers': ['application_log_file'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'command': { + 'handlers': ['command'], + 'level': 'DEBUG', + 'propogate': True + } } } @@ -456,7 +556,8 @@ '--cover-html', '--verbosity=2', '--xunit-file=target/reports/unit-test/nosetests.ureport.xml', '--cover-html-dir=target/reports/unit-test/coverage', - '--cover-package=poll,uganda_common,unregister,message_classifier,contact,rapidsms_httprouter,rapidsms_ureport', + '--cover-package=poll,uganda_common,unregister,' + 'message_classifier,contact,rapidsms_httprouter,rapidsms_ureport', ) MAP_BOUNDS = { From ea4016514cf245d0bd433761a3c62ee3ff6c0cb1 Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:28:30 +0300 Subject: [PATCH 4/7] use debug mode in dev --- ureport_project/settings/dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ureport_project/settings/dev.py b/ureport_project/settings/dev.py index 4dc35a9..bb72cd6 100644 --- a/ureport_project/settings/dev.py +++ b/ureport_project/settings/dev.py @@ -6,6 +6,6 @@ except: pass -DEBUG = TEMPLATE_DEBUG = False +DEBUG = TEMPLATE_DEBUG = True GEOSERVER_URL = '' From 31b481eeba3a2d667776b05c041291e95d650328 Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:37:12 +0300 Subject: [PATCH 5/7] update settings in wsgi_app --- ureport_project/wsgi_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ureport_project/wsgi_app.py b/ureport_project/wsgi_app.py index 584e515..c8f757c 100644 --- a/ureport_project/wsgi_app.py +++ b/ureport_project/wsgi_app.py @@ -7,7 +7,7 @@ #print sys.path os.environ["CELERY_LOADER"] = "django" -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings.production' import sys print sys.path From e80d4cbd5dfe312d06608bf21bcc0494b499be27 Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:44:56 +0300 Subject: [PATCH 6/7] move celery conf to settings --- ureport_project/celeryconfig.py | 124 ------------------------------- ureport_project/settings/base.py | 7 +- ureport_project/settings/dev.py | 1 + 3 files changed, 4 insertions(+), 128 deletions(-) delete mode 100644 ureport_project/celeryconfig.py diff --git a/ureport_project/celeryconfig.py b/ureport_project/celeryconfig.py deleted file mode 100644 index b498737..0000000 --- a/ureport_project/celeryconfig.py +++ /dev/null @@ -1,124 +0,0 @@ -#BROKER_URL = 'amqp://guest:guest@localhost:5672//' - -#CELERY_ALWAYS_EAGER = True - - - -CELERY_RESULT_BACKEND = 'amqp' -CELERY_RESULT_PERSISTENT = False -CELERY_TASK_RESULT_EXPIRES = 10 -CELERY_DISABLE_RATE_LIMITS = True -CELERY_SEND_TASK_ERROR_EMAILS = True - -CELERY_DEFAULT_QUEUE = 'default' -CELERY_QUEUES = { - 'default': { - 'binding_key': 'task.#', - }, - 'upload_responses': { - 'binding_key': 'upload_responses.#', - }, - 'classify_excel': { - 'binding_key': 'classify_excel.#', - }, - 'message_export': { - 'binding_key': 'message_export.#', - }, - 'start_poll': { - 'binding_key': 'start_poll.#', - }, - 'process_message': { - 'binding_key': 'process_message.#', - }, - 'handle_incoming': { - 'binding_key': 'handle_incoming.#', - }, - 'reprocess_responses': { - 'binding_key': 'reprocess_responses.#', - }, - 'push_to_mtrac': { - 'binding_key': 'push_to_mtrac.#' - }, - 'process_uploaded_contacts':{ - 'binding_key': 'process_uploaded_contacts.#' - }, - 'process_assign_group':{ - 'binding_key': 'process_assign_group.#' - },'export_poll':{ - 'binding_key': 'export_poll.#' - },'extract_gen_reports':{ - 'binding_key': 'extract_gen_reports.#' - } -} -CELERY_DEFAULT_EXCHANGE = 'tasks' -CELERY_DEFAULT_EXCHANGE_TYPE = 'topic' -CELERY_DEFAULT_ROUTING_KEY = 'task.default' -#CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler" - -BROKER_HOST = "127.0.0.1" -BROKER_PORT = 5672 -BROKER_USER = "ureport" -BROKER_PASSWORD = "ureport" -BROKER_VHOST = "ureport" - -CELERY_ROUTES = { - 'message_classifier.tasks.upload_responses': { - 'queue': 'upload_responses', - 'routing_key': 'upload_responses.result' - }, - 'message_classifier.tasks.classify_excel': { - 'queue': 'classify_excel', - 'routing_key': 'classify_excel.result' - }, - 'message_classifier.tasks.message_export': { - 'queue': 'message_export', - 'routing_key': 'message_export.result' - }, - 'ureport.tasks.start_poll': { - 'queue': 'start_poll', - 'routing_key': 'start_poll.result' - }, - 'ureport.tasks.process_message': { - 'queue': 'process_message', - 'routing_key': 'process_message.result' - }, - 'rapidsms_httprouter.tasks.handle_incoming': { - 'queue': 'handle_incoming', - 'routing_key': 'handle_incoming.result' - }, - 'ureport.tasks.reprocess_responses': { - 'queue': 'reprocess_responses', - 'routing_key': 'reprocess_responses.result' - }, - 'ureport.tasks.push_to_mtrac': { - 'queue': 'push_to_mtrac', - 'routing_key': 'push_to_mtrac.result' - }, - 'ureport.tasks.process_uploaded_contacts': { - 'queue': 'process_uploaded_contacts', - 'routing_key': 'process_uploaded_contacts.result' - }, - 'ureport.tasks.process_assign_group': { - 'queue': 'process_assign_group', - 'routing_key': 'process_assign_group.result' - }, - 'ureport.tasks.export_poll': { - 'queue': 'export_poll', - 'routing_key': 'export_poll.result' - }, - 'ureport.tasks.extract_gen_reports': { - 'queue': 'extract_gen_reports', - 'routing_key': 'extract_gen_reports.result' - } - -} - -SERVER_EMAIL = 'Ureport Background Tasks' - -ADMINS = ( - ('Kenneth Matovu', 'kbonky@gmail.com'), - ('Arga', 'arghacha@thoughtworks.com'), -) - - -CELERY_IMPORTS = ('ureport.tasks','message_classifier.tasks','rapidsms_httprouter.tasks') \ No newline at end of file diff --git a/ureport_project/settings/base.py b/ureport_project/settings/base.py index 5b78279..ac8b029 100644 --- a/ureport_project/settings/base.py +++ b/ureport_project/settings/base.py @@ -2,6 +2,8 @@ # vim: ai ts=4 sts=4 et sw=4 # encoding=utf-8 +import djcelery +djcelery.setup_loader() # -------------------------------------------------------------------- # # PATH CONFIGURATION # @@ -374,11 +376,8 @@ # -------------------------------------------------------------------- # # CELERY CONFIGURATION # # -------------------------------------------------------------------- # -# TODO this duplicates celeryconfig.py! do we need that file? -# if djcelery will pick up celery conf here? -#BROKER_URL = 'amqp://guest:guest@localhost:5672//' -#CELERY_ALWAYS_EAGER = True +# djcelery can load config from here, so this replaces celeryconfig.py CELERY_RESULT_BACKEND = 'amqp' CELERY_RESULT_PERSISTENT = False diff --git a/ureport_project/settings/dev.py b/ureport_project/settings/dev.py index bb72cd6..e78609c 100644 --- a/ureport_project/settings/dev.py +++ b/ureport_project/settings/dev.py @@ -7,5 +7,6 @@ pass DEBUG = TEMPLATE_DEBUG = True +CELERY_ALWAYS_EAGER = True GEOSERVER_URL = '' From 75c053c06979c0f0efd6a7860df2613fb5cd310c Mon Sep 17 00:00:00 2001 From: ewheeler Date: Thu, 2 Jan 2014 12:50:01 +0300 Subject: [PATCH 7/7] update import in ci_settings --- ureport_project/ci_settings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ureport_project/ci_settings.py b/ureport_project/ci_settings.py index 4ec8517..3063a8c 100644 --- a/ureport_project/ci_settings.py +++ b/ureport_project/ci_settings.py @@ -1,4 +1,4 @@ -from settings import * +from settings.base import * CELERY_ALWAYS_EAGER = True ADMIN_UNREGISTER = True IBM_TABLES_MANAGED = True @@ -106,21 +106,21 @@ 'handlers': ['application_log_file'], 'level': 'ERROR', 'propagate': True, - + }, 'app/ureport' : { 'handlers': ['application_log_file'], 'level': 'INFO', 'propagate': True, - + }, 'app/ureport' : { 'handlers': ['application_log_file'], 'level': 'INFO', 'propagate': True, - + }, 'django.middleware': { @@ -128,7 +128,7 @@ 'level': 'ERROR', 'propagate': True, }, - + 'django.db.backends': { 'handlers': ['application_log_file'], 'level': 'ERROR', @@ -145,7 +145,7 @@ 'level': 'INFO', 'propagate': True, }, - + 'ureport.tasks': { 'handlers': ['application_log_file'], 'level': 'INFO',