diff --git a/leasing/report/lease/lease_statistic_report.py b/leasing/report/lease/lease_statistic_report.py index 0954f3c1..e52a7f28 100644 --- a/leasing/report/lease/lease_statistic_report.py +++ b/leasing/report/lease/lease_statistic_report.py @@ -502,6 +502,17 @@ def get_data(self, input_data): Q(end_date__isnull=True) | Q(end_date__gte=datetime.date.today()) ) + # Skip the leases where have not set the period type of base amount in contact rents + # The report fails because contract rents contain items where isn't defined the period type for the base amount + # TODO: Review this with the specialist + no_period = [] + for lease in qs: + for rent in lease.rents.all(): + for cr in rent.contract_rents.all(): + if not cr.base_amount_period: + no_period.append(lease.id) + qs = qs.exclude(id__in=no_period) + return qs def generate_report(self, user, input_data): diff --git a/leasing/report/report_base.py b/leasing/report/report_base.py index 5ceaf008..bdc00d3e 100644 --- a/leasing/report/report_base.py +++ b/leasing/report/report_base.py @@ -68,8 +68,8 @@ class ReportBase: # If the column labels should be automatically added as the first row automatic_excel_column_labels = True - def __init__(self): - self.form = None + # The query form model of report + form = None @classmethod def get_output_fields_metadata(cls): @@ -88,6 +88,10 @@ def get_form(self, data=None): self.form instance attribute and returns it.""" self.form = ReportFormBase(data, input_fields=self.input_fields) + # This has been set to None as the report doesn't require any form rendering + # and it causes pickle error in Django Q async tasks. + self.form.renderer = None + return self.form def get_input_data(self, request): diff --git a/leasing/tests/report/test_reports.py b/leasing/tests/report/test_reports.py new file mode 100644 index 00000000..7b87a0ab --- /dev/null +++ b/leasing/tests/report/test_reports.py @@ -0,0 +1,57 @@ +from multiprocessing import Event, Value + +import pytest +from django.core import mail +from django_q.brokers import get_broker +from django_q.cluster import monitor, pusher, worker +from django_q.queues import Queue +from django_q.tasks import queue_size + +from leasing.report.lease.lease_statistic_report import LeaseStatisticReport + + +@pytest.fixture(autouse=True) +def use_q_cluster_testing(settings): + settings.Q_CLUSTER = { + "name": "DjangORM", + "cpu_affinity": 1, + "testing": True, + "log_level": "DEBUG", + "orm": "default", + } + + +@pytest.mark.django_db(transaction=True) +def test_simple_async_report_send(rf, admin_user): + broker = get_broker() + assert broker.queue_size() == 0 + + request = rf.get("/") + request.query_params = {} + request.user = admin_user + + report = LeaseStatisticReport() + response = report.get_response(request) + assert response.data + assert broker.queue_size() == 1 + + # Run async task + task_queue = Queue() + result_queue = Queue() + event = Event() + event.set() + pusher(task_queue, event, broker=broker) + assert task_queue.qsize() == 1 + assert queue_size(broker=broker) == 0 + task_queue.put("STOP") + worker(task_queue, result_queue, Value("f", -1)) + assert task_queue.qsize() == 0 + assert result_queue.qsize() == 1 + result_queue.put("STOP") + monitor(result_queue) + assert result_queue.qsize() == 0 + broker.delete_queue() + + # Test report file have been sent via email + assert len(mail.outbox) == 1 + assert len(mail.outbox[0].attachments) == 1 diff --git a/local_settings.py.template b/local_settings.py.template new file mode 100644 index 00000000..7fb753d5 --- /dev/null +++ b/local_settings.py.template @@ -0,0 +1,11 @@ +Q_CLUSTER = { + "name": "DjangORM", + "timeout": 90, + "retry": 60 * 60, # 1 hour + "orm": "default", + "error_reporter": { + "sentry": { + "dsn": "https://******@sentry.io/" + } + } +} diff --git a/mvj/settings.py b/mvj/settings.py index 4ef4b21b..1d43810c 100644 --- a/mvj/settings.py +++ b/mvj/settings.py @@ -149,11 +149,11 @@ def get_git_revision_hash(): "laske_export", "field_permissions", "batchrun", - "django_q", "constance", "constance.backends.database", "sanitized_dump", "utils", + "django_q", ] if DEBUG: diff --git a/requirements-dev.txt b/requirements-dev.txt index c8861d96..6812475a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,70 +4,179 @@ # # pip-compile requirements-dev.in # -appdirs==1.4.3 # via -c requirements.txt, black -attrs==19.3.0 # via -c requirements.txt, black, pytest -backcall==0.1.0 # via ipython -black==19.10b0 # via -r requirements-dev.in -click==7.1.2 # via black -coverage==5.0.4 # via pytest-cov -debugpy==1.1.0 # via -r requirements-dev.in -decorator==4.4.2 # via ipython, traitlets -django-debug-toolbar==2.2 # via -r requirements-dev.in -django-extensions==2.2.9 # via -r requirements-dev.in -django-stubs==1.5.0 # via -r requirements-dev.in -django==2.2.13 # via -c requirements.txt, django-debug-toolbar, django-stubs -entrypoints==0.3 # via flake8 -et-xmlfile==1.0.1 # via openpyxl -factory-boy==2.12.0 # via -r requirements-dev.in, pytest-factoryboy -faker==4.0.2 # via factory-boy -flake8-polyfill==1.0.2 # via pep8-naming -flake8-print==3.1.4 # via -r requirements-dev.in -flake8==3.7.9 # via -r requirements-dev.in, flake8-polyfill, flake8-print -importlib-metadata==2.0.0 # via pluggy, pytest -inflection==0.3.1 # via pytest-factoryboy -ipython-genutils==0.2.0 # via traitlets -ipython==7.13.0 # via -r requirements-dev.in -isort==5.6.4 # via -r requirements-dev.in -jdcal==1.4.1 # via openpyxl -jedi==0.16.0 # via ipython -mccabe==0.6.1 # via flake8 -more-itertools==8.2.0 # via pytest -mypy-extensions==0.4.3 # via mypy -mypy==0.770 # via -r requirements-dev.in, django-stubs -openpyxl==3.0.3 # via -r requirements-dev.in -packaging==20.3 # via pytest -parso==0.6.2 # via jedi -pathspec==0.8.0 # via black -pep8-naming==0.10.0 # via -r requirements-dev.in -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.5 # via ipython -ptyprocess==0.6.0 # via pexpect -py==1.8.1 # via pytest -pycodestyle==2.5.0 # via flake8, flake8-print -pydocstyle==5.0.2 # via -r requirements-dev.in -pyflakes==2.1.1 # via flake8 -pygments==2.6.1 # via ipython -pyparsing==2.4.6 # via packaging -pytest-cov==2.8.1 # via -r requirements-dev.in -pytest-django==3.9.0 # via -r requirements-dev.in -pytest-factoryboy==2.0.3 # via -r requirements-dev.in -pytest==5.4.1 # via -r requirements-dev.in, pytest-cov, pytest-django, pytest-factoryboy -python-dateutil==2.6.0 # via -c requirements.txt, faker -pytz==2019.3 # via -c requirements.txt, django -regex==2020.5.14 # via black -six==1.14.0 # via -c requirements.txt, django-extensions, flake8-print, packaging, python-dateutil, traitlets -snowballstemmer==2.0.0 # via pydocstyle -sqlparse==0.3.1 # via -c requirements.txt, django, django-debug-toolbar -text-unidecode==1.3 # via faker -toml==0.10.1 # via black -traitlets==4.3.3 # via ipython -typed-ast==1.4.1 # via black, mypy -typing-extensions==3.7.4.2 # via django-stubs, mypy -wcwidth==0.1.9 # via -c requirements.txt, prompt-toolkit, pytest -werkzeug==1.0.1 # via -r requirements-dev.in -zipp==3.3.0 # via importlib-metadata +appdirs==1.4.3 + # via + # -c requirements.txt + # black +attrs==19.3.0 + # via + # -c requirements.txt + # black + # pytest +backcall==0.1.0 + # via ipython +black==19.10b0 + # via -r requirements-dev.in +click==7.1.2 + # via black +coverage==5.0.4 + # via pytest-cov +debugpy==1.1.0 + # via -r requirements-dev.in +decorator==4.4.2 + # via + # ipython + # traitlets +django-debug-toolbar==2.2 + # via -r requirements-dev.in +django-extensions==2.2.9 + # via -r requirements-dev.in +django-stubs==1.5.0 + # via -r requirements-dev.in +django==2.2.13 + # via + # -c requirements.txt + # django-debug-toolbar + # django-stubs +entrypoints==0.3 + # via flake8 +et-xmlfile==1.0.1 + # via openpyxl +factory-boy==2.12.0 + # via + # -r requirements-dev.in + # pytest-factoryboy +faker==4.0.2 + # via factory-boy +flake8-polyfill==1.0.2 + # via pep8-naming +flake8-print==3.1.4 + # via -r requirements-dev.in +flake8==3.7.9 + # via + # -r requirements-dev.in + # flake8-polyfill + # flake8-print +importlib-metadata==2.0.0 + # via + # pluggy + # pytest +inflection==0.3.1 + # via pytest-factoryboy +ipython-genutils==0.2.0 + # via traitlets +ipython==7.13.0 + # via -r requirements-dev.in +isort==5.6.4 + # via -r requirements-dev.in +jdcal==1.4.1 + # via openpyxl +jedi==0.16.0 + # via ipython +mccabe==0.6.1 + # via flake8 +more-itertools==8.2.0 + # via pytest +mypy-extensions==0.4.3 + # via mypy +mypy==0.770 + # via + # -r requirements-dev.in + # django-stubs +openpyxl==3.0.3 + # via -r requirements-dev.in +packaging==20.3 + # via pytest +parso==0.6.2 + # via jedi +pathspec==0.8.0 + # via black +pep8-naming==0.10.0 + # via -r requirements-dev.in +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pluggy==0.13.1 + # via pytest +prompt-toolkit==3.0.5 + # via ipython +ptyprocess==0.6.0 + # via pexpect +py==1.8.1 + # via pytest +pycodestyle==2.5.0 + # via + # flake8 + # flake8-print +pydocstyle==5.0.2 + # via -r requirements-dev.in +pyflakes==2.1.1 + # via flake8 +pygments==2.6.1 + # via ipython +pyparsing==2.4.6 + # via packaging +pytest-cov==2.8.1 + # via -r requirements-dev.in +pytest-django==3.9.0 + # via -r requirements-dev.in +pytest-factoryboy==2.0.3 + # via -r requirements-dev.in +pytest==5.4.1 + # via + # -r requirements-dev.in + # pytest-cov + # pytest-django + # pytest-factoryboy +python-dateutil==2.6.0 + # via + # -c requirements.txt + # faker +pytz==2019.3 + # via + # -c requirements.txt + # django +regex==2020.5.14 + # via black +six==1.14.0 + # via + # -c requirements.txt + # django-extensions + # flake8-print + # packaging + # python-dateutil + # traitlets +snowballstemmer==2.0.0 + # via pydocstyle +sqlparse==0.3.1 + # via + # -c requirements.txt + # django + # django-debug-toolbar +text-unidecode==1.3 + # via faker +toml==0.10.1 + # via black +traitlets==4.3.3 + # via ipython +typed-ast==1.4.1 + # via + # black + # mypy +typing-extensions==3.7.4.2 + # via + # django-stubs + # mypy +wcwidth==0.1.9 + # via + # -c requirements.txt + # prompt-toolkit + # pytest +werkzeug==1.0.1 + # via -r requirements-dev.in +zipp==3.3.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.in b/requirements.in index 368e6f34..67fe1c3d 100644 --- a/requirements.in +++ b/requirements.in @@ -15,7 +15,7 @@ django-filter django-helusers django-model-utils django-modeltranslation -django-q +django-q[sentry] django-safedelete django-sanitized-dump django-sequences diff --git a/requirements.txt b/requirements.txt index a2d39375..ff31bbec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,86 +4,235 @@ # # pip-compile requirements.in # -appdirs==1.4.3 # via zeep -arrow==0.15.5 # via django-q -attrs==19.3.0 # via zeep -bcrypt==3.1.7 # via paramiko -beautifulsoup4==4.8.2 # via -r requirements.in -blessed==1.17.4 # via django-q -cached-property==1.5.1 # via zeep -certifi==2019.11.28 # via requests, sentry-sdk -cffi==1.14.0 # via bcrypt, cryptography, pynacl -chardet==3.0.4 # via requests -coreapi==2.3.3 # via django-rest-swagger, openapi-codec -coreschema==0.0.4 # via coreapi -cryptography==3.2 # via paramiko -database-sanitizer==1.1.0 # via django-sanitized-dump -dataclasses==0.6 # via -r requirements.in -defusedxml==0.6.0 # via zeep -django-admin-rangefilter==0.6.3 # via -r requirements.in -django-anymail==7.0.0 # via -r requirements.in -django-auditlog==0.4.7 # via -r requirements.in -django-constance[database]==2.7.0 # via -r requirements.in -django-cors-headers==3.2.1 # via -r requirements.in -django-countries==6.1.2 # via -r requirements.in -django-crispy-forms==1.9.0 # via -r requirements.in -django-enumfields==2.0.0 # via -r requirements.in -django-environ==0.4.5 # via -r requirements.in -django-filter==2.2.0 # via -r requirements.in -django-helusers==0.5.6 # via -r requirements.in -django-jsonfield-compat==0.4.4 # via -r requirements.in -django-jsonfield==1.4.0 # via django-auditlog -django-model-utils==4.0.0 # via -r requirements.in -django-modeltranslation==0.16.1 # via -r requirements.in -django-picklefield==2.1.1 # via django-constance, django-q -django-q==1.2.1 # via -r requirements.in -django-rest-swagger==2.2.0 # via -r requirements.in -django-safedelete==0.5.4 # via -r requirements.in -django-sanitized-dump==1.2.0 # via -r requirements.in -django-sequences==2.4 # via -r requirements.in -django==2.2.13 # via -r requirements.in, django-anymail, django-cors-headers, django-filter, django-helusers, django-jsonfield, django-model-utils, django-modeltranslation, django-picklefield, django-q, django-safedelete, djangorestframework, drf-oidc-auth -djangorestframework-gis==0.15 # via -r requirements.in -djangorestframework==3.11.0 # via -r requirements.in, django-rest-swagger, djangorestframework-gis, drf-oidc-auth -docutils==0.16 # via python-daemon -docxtpl==0.6.3 # via -r requirements.in -drf-oidc-auth==0.9 # via django-helusers -ecdsa==0.15 # via python-jose -future==0.18.2 # via pyjwkest -idna==2.9 # via requests -isodate==0.6.0 # via zeep -itypes==1.1.0 # via coreapi -jinja2==2.11.1 # via coreschema, docxtpl -lockfile==0.12.2 # via python-daemon -lxml==4.5.0 # via docxtpl, python-docx, zeep -markupsafe==1.1.1 # via jinja2 -openapi-codec==1.3.2 # via django-rest-swagger -paramiko==2.7.1 # via pysftp -psycopg2-binary==2.8.4 # via -r requirements.in -pyasn1==0.4.8 # via python-jose, rsa -pycparser==2.20 # via cffi -pycryptodomex==3.9.7 # via pyjwkest -pyjwkest==1.4.2 # via drf-oidc-auth -pynacl==1.3.0 # via paramiko -pysftp==0.2.9 # via -r requirements.in -python-daemon==2.2.4 # via -r requirements.in -python-dateutil==2.6.0 # via -r requirements.in, arrow, django-auditlog -python-docx==0.8.7 # via docxtpl -python-jose==3.1.0 # via django-helusers -pytz==2019.3 # via django, zeep -pyyaml==5.3.1 # via database-sanitizer, django-sanitized-dump -requests-toolbelt==0.9.1 # via zeep -requests==2.23.0 # via -r requirements.in, coreapi, django-anymail, django-helusers, pyjwkest, requests-toolbelt, zeep -rsa==4.0 # via python-jose -sentry-sdk==0.14.3 # via -r requirements.in -simplejson==3.17.0 # via django-rest-swagger -six==1.14.0 # via bcrypt, blessed, cryptography, database-sanitizer, django-anymail, django-jsonfield, django-modeltranslation, django-sanitized-dump, docxtpl, ecdsa, isodate, pyjwkest, pynacl, python-dateutil, python-jose, zeep -soupsieve==2.0 # via beautifulsoup4 -sqlparse==0.3.1 # via django -uritemplate==3.0.1 # via coreapi -urllib3==1.25.8 # via requests, sentry-sdk -wcwidth==0.1.9 # via blessed -xlsxwriter==1.2.8 # via -r requirements.in -zeep==3.4.0 # via -r requirements.in +appdirs==1.4.3 + # via zeep +arrow==0.15.5 + # via django-q +attrs==19.3.0 + # via zeep +bcrypt==3.1.7 + # via paramiko +beautifulsoup4==4.8.2 + # via -r requirements.in +blessed==1.17.4 + # via django-q +cached-property==1.5.1 + # via zeep +certifi==2019.11.28 + # via + # requests + # sentry-sdk +cffi==1.14.0 + # via + # bcrypt + # cryptography + # pynacl +chardet==3.0.4 + # via requests +coreapi==2.3.3 + # via + # django-rest-swagger + # openapi-codec +coreschema==0.0.4 + # via coreapi +cryptography==3.2 + # via paramiko +database-sanitizer==1.1.0 + # via django-sanitized-dump +dataclasses==0.6 + # via -r requirements.in +defusedxml==0.6.0 + # via zeep +django-admin-rangefilter==0.6.3 + # via -r requirements.in +django-anymail==7.0.0 + # via -r requirements.in +django-auditlog==0.4.7 + # via -r requirements.in +django-constance[database]==2.7.0 + # via -r requirements.in +django-cors-headers==3.2.1 + # via -r requirements.in +django-countries==6.1.2 + # via -r requirements.in +django-crispy-forms==1.9.0 + # via -r requirements.in +django-enumfields==2.0.0 + # via -r requirements.in +django-environ==0.4.5 + # via -r requirements.in +django-filter==2.2.0 + # via -r requirements.in +django-helusers==0.5.6 + # via -r requirements.in +django-jsonfield-compat==0.4.4 + # via -r requirements.in +django-jsonfield==1.4.0 + # via django-auditlog +django-model-utils==4.0.0 + # via -r requirements.in +django-modeltranslation==0.16.1 + # via -r requirements.in +django-picklefield==2.1.1 + # via + # django-constance + # django-q +django-q-sentry==0.1.4 + # via django-q +django-q[sentry]==1.3.4 + # via -r requirements.in +django-rest-swagger==2.2.0 + # via -r requirements.in +django-safedelete==0.5.4 + # via -r requirements.in +django-sanitized-dump==1.2.0 + # via -r requirements.in +django-sequences==2.4 + # via -r requirements.in +django==2.2.13 + # via + # -r requirements.in + # django-anymail + # django-cors-headers + # django-filter + # django-helusers + # django-jsonfield + # django-model-utils + # django-modeltranslation + # django-picklefield + # django-q + # django-safedelete + # djangorestframework + # drf-oidc-auth +djangorestframework-gis==0.15 + # via -r requirements.in +djangorestframework==3.11.0 + # via + # -r requirements.in + # django-rest-swagger + # djangorestframework-gis + # drf-oidc-auth +docutils==0.16 + # via python-daemon +docxtpl==0.6.3 + # via -r requirements.in +drf-oidc-auth==0.9 + # via django-helusers +ecdsa==0.15 + # via python-jose +future==0.18.2 + # via pyjwkest +idna==2.9 + # via requests +isodate==0.6.0 + # via zeep +itypes==1.1.0 + # via coreapi +jinja2==2.11.1 + # via + # coreschema + # docxtpl +lockfile==0.12.2 + # via python-daemon +lxml==4.5.0 + # via + # docxtpl + # python-docx + # zeep +markupsafe==1.1.1 + # via jinja2 +openapi-codec==1.3.2 + # via django-rest-swagger +paramiko==2.7.1 + # via pysftp +psycopg2-binary==2.8.4 + # via -r requirements.in +pyasn1==0.4.8 + # via + # python-jose + # rsa +pycparser==2.20 + # via cffi +pycryptodomex==3.9.7 + # via pyjwkest +pyjwkest==1.4.2 + # via drf-oidc-auth +pynacl==1.3.0 + # via paramiko +pysftp==0.2.9 + # via -r requirements.in +python-daemon==2.2.4 + # via -r requirements.in +python-dateutil==2.6.0 + # via + # -r requirements.in + # arrow + # django-auditlog +python-docx==0.8.7 + # via docxtpl +python-jose==3.1.0 + # via django-helusers +pytz==2019.3 + # via + # django + # zeep +pyyaml==5.3.1 + # via + # database-sanitizer + # django-sanitized-dump +requests-toolbelt==0.9.1 + # via zeep +requests==2.23.0 + # via + # -r requirements.in + # coreapi + # django-anymail + # django-helusers + # pyjwkest + # requests-toolbelt + # zeep +rsa==4.0 + # via python-jose +sentry-sdk==0.16.5 + # via + # -r requirements.in + # django-q-sentry +simplejson==3.17.0 + # via django-rest-swagger +six==1.14.0 + # via + # bcrypt + # blessed + # cryptography + # database-sanitizer + # django-anymail + # django-jsonfield + # django-modeltranslation + # django-sanitized-dump + # docxtpl + # ecdsa + # isodate + # pyjwkest + # pynacl + # python-dateutil + # python-jose + # zeep +soupsieve==2.0 + # via beautifulsoup4 +sqlparse==0.3.1 + # via django +uritemplate==3.0.1 + # via coreapi +urllib3==1.25.8 + # via + # requests + # sentry-sdk +wcwidth==0.1.9 + # via blessed +xlsxwriter==1.2.8 + # via -r requirements.in +zeep==3.4.0 + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools