diff --git a/ChangeLog b/ChangeLog index 32214f78..5fe14eed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,11 @@ CHANGES ======= -UPDATE: Update jsonschema dep 2.5.0 to 2.5.1 +Upgrade Python (2 -> 3), #242 + Django (1.11 -> 2.2.24) #310 + DjangoRestFramework (3.9 -> 3.11) #309 + requests (2.22.0 -> 2.25.1) + python-ldap (2.5.2 -> 3.3.1) +UPDATE: Update jsonschema dep 2.5.0 to 2.6.0 Upgrade STH xacml policies with orion frns 3.8.0 diff --git a/DOCKER.md b/DOCKER.md index 8c9496c6..5af7546b 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -104,7 +104,7 @@ Additionally, the following environment variables are available for orchestrator | PROCESSES | | 6 | | THREADS | | 8 | | HARAKIRI | | 80 | -| HTTP_TIMEOUT | | 200 | +| HTTP_TIMEOUT | | 200 | | MAX_REQUESTS | | 250 | | UWSGI_BUFFER_SIZE | | 4096 | | KEYSTONE_HOST | KEYSTONE.host | localhost | diff --git a/Dockerfile b/Dockerfile index 137e7868..151809e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV ORCHESTRATOR_USER_UID 10001 ENV ORCHESTRATOR_VERSION 3.8.0 -ENV python_lib /var/env-orchestrator/lib/python2.7/site-packages +ENV python_lib /var/env-orchestrator/lib/python3.6/site-packages ENV DJANGO_SETTINGS_MODULE settings ENV PYTHONPATH "${PYTHONPATH}:/opt/orchestrator" @@ -20,7 +20,7 @@ RUN \ adduser --comment "${ORCHESTRATOR_USER}" -u ${ORCHESTRATOR_USER_UID} ${ORCHESTRATOR_USER} && \ # Install dependencies yum install -y epel-release && yum update -y epel-release && \ - yum install -y yum-plugin-remove-with-leaves python python-pip python-devel openldap-devel python-virtualenv gcc ssh && \ + yum install -y yum-plugin-remove-with-leaves python3 python3-pip python3-devel openldap-devel python3-virtualenv gcc ssh && \ yum install -y tcping findutils sed && \ mkdir -p $python_lib/iotp-orchestrator && \ mkdir -p $python_lib/iotp-orchestrator/bin && \ @@ -28,7 +28,7 @@ RUN \ cp -rp /opt/sworchestrator/bin $python_lib/iotp-orchestrator && \ chmod 755 $python_lib/iotp-orchestrator/bin/orchestrator-entrypoint.sh && \ chown -R ${ORCHESTRATOR_USER}:${ORCHESTRATOR_USER} $python_lib/iotp-orchestrator && \ - pip install -r $python_lib/iotp-orchestrator/requirements.txt && \ + pip3 install -r $python_lib/iotp-orchestrator/requirements.txt && \ find $python_lib/iotp-orchestrator -name "*.pyc" -delete && \ ln -s $python_lib/iotp-orchestrator /opt/orchestrator && \ ln -s /opt/orchestrator/orchestrator/commands /opt/orchestrator/bin/ && \ diff --git a/INSTALL.md b/INSTALL.md index 3759676e..f8cb9204 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -12,7 +12,7 @@ Due to its stateless behavior, iotp-orchestrator does not require any backup pro * Internet access * Connectivity to http://github.com * [Git](http://git-scm.com/) - * Python 2.7 (or upper) and pip + * Python 3 and pip3 * C and C++ compilation platform: gcc, g++, make and python headers * UNIX platforms but not mandatory * WSGI Web server (Apache2/Nginx) but not mandatory @@ -42,13 +42,13 @@ cd src Then install all dependencies by running: ``` -pip install -r requirements.txt +pip3 install -r requirements.txt ``` Start server in 8084 port using django web server: ``` -python manage.py runserver 8084 --settings=settings.dev +python3 manage.py runserver 8084 --settings=settings.dev ``` or using another web server like uWSGI diff --git a/README.md b/README.md index 7e4ee0b0..3983a22e 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,8 @@ Orchestrator is used to: - Create, List, Modify LDAP Groups Orchestrator is based mainly on: -- Python 2.7 needed +- Python 3 needed - Django / DjangoRestFramework -- httplib Orchestrator relies on these other IoT parts: - Identity Manager: Keystone (mandatory) diff --git a/TESTS.md b/TESTS.md index 4b0904fa..1b00b3f3 100644 --- a/TESTS.md +++ b/TESTS.md @@ -4,7 +4,7 @@ - [Unit Tests](./src/tests/core/test_idm.py) Run unit tests with: ``` - $ python ./test_idm.py + $ python3 ./test_idm.py ``` @@ -12,21 +12,21 @@ Run unit tests with: Run e2e tests with: ``` - $ python ./test_api.py + $ python3 ./test_api.py ``` - [End2End LDAP Tests](./src/tests/api/test_ldap_api.py) Run e2e LDAP tests with: ``` - $ python ./test_ldap_api.py + $ python3 ./test_ldap_api.py ``` - [Comamnd Tests](./src/tests/api/test_commands.py) Run scripts tests with: ``` - $ python ./test_commands.py + $ python3 ./test_commands.py ``` There is a config file to run Orchestrator build into Travis CI service: diff --git a/orchestrator.spec b/orchestrator.spec index ee18c7a0..7789a140 100644 --- a/orchestrator.spec +++ b/orchestrator.spec @@ -18,11 +18,8 @@ Prefix: /opt BuildArch: noarch %define _target_os Linux -%define python_lib /var/env-orchestrator/lib/python2.6/site-packages -%if 0%{?with_python27} -%define python_lib /var/env-orchestrator/lib/python2.7/site-packages -%define __python /usr/bin/python2.7 -%endif # if with_python27 +%define python_lib /var/env-orchestrator/lib/python3.6/site-packages +%define __python /usr/bin/python3.6 %description IoT Platform Orchestrator diff --git a/package-orchestrator.sh b/package-orchestrator.sh index 2e7016fc..21a39ab9 100755 --- a/package-orchestrator.sh +++ b/package-orchestrator.sh @@ -35,7 +35,6 @@ source $BASE/get_version_string.sh string=$(get_rpm_version_string) VERSION_VALUE=${string% *} RELEASE_VALUE=${string#* } -PYTHON27_VALUE=0 args=("$@") ELEMENTS=${#args[@]} @@ -48,9 +47,6 @@ for (( i=0;i<$ELEMENTS;i++)); do if [ "$arg" == "--with-release" ]; then RELEASE_VALUE=${args[${i}+1]} fi - if [ "$arg" == "--with-python27" ]; then - PYTHON27_VALUE=1 - fi done @@ -63,6 +59,5 @@ rpmbuild -bb orchestrator.spec \ --define "_topdir $RPM_DIR" \ --define "_root $BASE"\ --define "_project_user $ORCHESTRATOR_USER"\ - --define "with_python27 $PYTHON27_VALUE"\ --define "_version $VERSION_VALUE"\ --define "_release $RELEASE_VALUE" diff --git a/requirements.txt b/requirements.txt index 98c2b0e0..1e3a810e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -Django==1.11.29 -djangorestframework==3.9.4 -jsonschema==2.5.1 +Django==2.2.24 +djangorestframework==3.11.2 +jsonschema==2.6.0 uwsgi==2.0.19.1 -requests==2.22.0 -python-ldap==2.5.2 +requests==2.25.1 +python-ldap==3.3.1 pymongo==3.7.2 diff --git a/src/orchestrator/api/ldap_view.py b/src/orchestrator/api/ldap_view.py index ead9b6ca..6e1e4561 100644 --- a/src/orchestrator/api/ldap_view.py +++ b/src/orchestrator/api/ldap_view.py @@ -54,6 +54,9 @@ def post(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -98,8 +101,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -111,6 +113,9 @@ def get(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -151,8 +156,7 @@ def get(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -164,6 +168,9 @@ def put(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -188,8 +195,7 @@ def put(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -202,6 +208,9 @@ def delete(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -225,8 +234,7 @@ def delete(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -250,6 +258,9 @@ def post(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -272,8 +283,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -298,6 +308,9 @@ def post(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -330,8 +343,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -343,6 +355,9 @@ def get(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -378,8 +393,7 @@ def get(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -392,6 +406,9 @@ def put(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -415,8 +432,7 @@ def put(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -429,6 +445,9 @@ def delete(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -451,8 +470,7 @@ def delete(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) diff --git a/src/orchestrator/api/parsers.py b/src/orchestrator/api/parsers.py index ad1a803f..6014aa00 100644 --- a/src/orchestrator/api/parsers.py +++ b/src/orchestrator/api/parsers.py @@ -49,7 +49,7 @@ def parse(self, stream, media_type=None, parser_context=None): data, schemas.json[parser_context['view'].schema_name]) except (ValueError, jsonschema.exceptions.ValidationError) as error: - logger.debug(error.message) - raise ParseError(detail=error.message) + logger.debug(str(error.args[0])) + raise ParseError(detail=str(error.args[0])) else: return data diff --git a/src/orchestrator/api/views.py b/src/orchestrator/api/views.py index a6ae054e..cab638b9 100644 --- a/src/orchestrator/api/views.py +++ b/src/orchestrator/api/views.py @@ -128,8 +128,7 @@ def get(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -171,8 +170,7 @@ def put(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -223,8 +221,7 @@ def delete(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -287,8 +284,7 @@ def post(self, request, *args, **kw): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -359,8 +355,7 @@ def get(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -411,8 +406,7 @@ def put(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -469,8 +463,7 @@ def delete(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -536,8 +529,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -589,8 +581,7 @@ def delete(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -632,7 +623,7 @@ def put(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0}'.format(error.message), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -673,8 +664,7 @@ def get(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -717,8 +707,7 @@ def post(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -775,8 +764,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -820,8 +808,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -874,8 +861,7 @@ def delete(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -917,7 +903,7 @@ def put(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0}'.format(error.message), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -958,8 +944,7 @@ def get(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1016,8 +1001,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1059,8 +1043,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1119,8 +1102,7 @@ def get(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1171,8 +1153,7 @@ def post(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1218,8 +1199,7 @@ def delete(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1279,8 +1259,7 @@ def get(self, request, service_id, role_id, policy_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1327,8 +1306,7 @@ def delete(self, request, service_id, role_id, policy_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1383,8 +1361,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1429,8 +1406,7 @@ def get(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1491,8 +1467,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1570,8 +1545,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1650,8 +1624,7 @@ def delete(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1712,8 +1685,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1791,8 +1763,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1871,8 +1842,7 @@ def delete(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1933,8 +1903,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2025,8 +1994,7 @@ def get(self, request, service_id, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2106,8 +2074,7 @@ def post(self, request, service_id, subservice_id=None, iot_module=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2187,8 +2154,7 @@ def delete(self, request, service_id, subservice_id=None, iot_module=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2281,8 +2247,7 @@ def get(self, request): except ParseError as error: response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST ) self.collectMetrics(service_start, service_name, subservice_name, request, response, flow) diff --git a/src/orchestrator/common/util.py b/src/orchestrator/common/util.py index 8b33f993..8524a0e8 100644 --- a/src/orchestrator/common/util.py +++ b/src/orchestrator/common/util.py @@ -21,11 +21,11 @@ # # Author: IoT team # -import urllib2 +import urllib.request import base64 import json import csv -import StringIO +import io import requests import logging import time @@ -98,12 +98,12 @@ def rest_request(self, url, method, user=None, password=None, if data: if json_data: - request = urllib2.Request( - url, data=json.dumps(data)) + request = urllib.request.Request( + url=url, data=json.dumps(data).encode()) else: - request = urllib2.Request(url, data=data) + request = urllib.request.Request(url=url, body=data) else: - request = urllib2.Request(url) + request = urllib.request.Request(url) request.get_method = lambda: method if json_data: @@ -142,8 +142,8 @@ def rest_request(self, url, method, user=None, password=None, res = None try: - res = urllib2.urlopen(request) - except urllib2.HTTPError, e: + res = urllib.request.urlopen(request) + except urllib.request.HTTPError as e: res = e data = res.read() try: @@ -164,9 +164,9 @@ def rest_request(self, url, method, user=None, password=None, except ValueError: res.msg = data - except Exception, e: - print e - except urllib2.URLError, e: + except Exception as e: + self.logger.warn("exception %s", e) + except urllib.request.URLError as e: data = None res = e res.code = 500 @@ -242,19 +242,23 @@ def rest_request2(self, url, method, user=None, password=None, try: if not auth: - res = requests.post(url, - headers=headers, - data=rdata, - verify=False) + res = requests.request(method=method, + url=url, + headers=headers, + data=rdata, + verify=False) else: - res = requests.post(url, - auth=auth, - headers=headers, - data=rdata, - verify=False) - - except Exception, e: - print e + res = requests.request(method=method, + url=url, + auth=auth, + headers=headers, + data=rdata, + verify=False) + res.code = res.status_code + res.msg = res.reason + + except Exception as e: + self.logger.warn("exception %s", e) if settings.ORC_EXTENDED_METRICS: self.collectOutgoingMetrics(service_start, rdata, headers, res) @@ -276,7 +280,7 @@ def collectOutgoingMetrics(self, service_start, data_request, headers_request, r # Check headers self.sum["outgoingTransactionResponseSize"] += len(json.dumps(data_response)) + len(str(response.headers.headers)) if 'headers' in response and 'headers' in response.headers else 0 self.sum["serviceTimeTotal"] += (service_stop - service_start) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR collecting outgoing metrics %s", ex) def getOutgoingMetrics(self): @@ -294,7 +298,7 @@ def __init__(self): @staticmethod def read_devices(CSV): devices = {} - csvreader = csv.reader(StringIO.StringIO(CSV), + csvreader = csv.reader(io.StringIO(CSV), delimiter=',', #quotechar='"', skipinitialspace=True) diff --git a/src/orchestrator/core/flow/Domains.py b/src/orchestrator/core/flow/Domains.py index b298d3c0..b8f47977 100644 --- a/src/orchestrator/core/flow/Domains.py +++ b/src/orchestrator/core/flow/Domains.py @@ -72,7 +72,7 @@ def domains(self, self.logger.debug("DOMAINS=%s" % json.dumps(DOMAINS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -140,7 +140,7 @@ def get_domain(self, self.logger.debug("DOMAIN=%s" % json.dumps(DOMAIN, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -213,7 +213,7 @@ def update_domain(self, self.logger.debug("DOMAIN=%s" % json.dumps(DOMAIN, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -345,7 +345,7 @@ def delete_domain(self, # try: self.idm.removeRoles(ADMIN_TOKEN, DOMAIN_ID) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("Deleting roles in a slow way") roles = self.idm.getDomainRoles(ADMIN_TOKEN, DOMAIN_ID) @@ -368,7 +368,7 @@ def delete_domain(self, self.logger.debug("DOMAIN=%s" % DOMAIN) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -487,7 +487,7 @@ def getDomainRolePolicies(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -596,7 +596,7 @@ def activate_module(self, subscriptionid = cb_res['subscriptionId'] self.logger.debug("subscription id=%s" % subscriptionid) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -690,7 +690,7 @@ def deactivate_module(self, sub['id']) break - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -771,7 +771,7 @@ def list_activated_modules(self, modules = self.cb.extract_modules_from_subscriptions(self, IOTMODULES, cb_res) self.logger.debug("modules=%s" % json.dumps(modules, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Groups.py b/src/orchestrator/core/flow/Groups.py index 8d77c693..03a69088 100644 --- a/src/orchestrator/core/flow/Groups.py +++ b/src/orchestrator/core/flow/Groups.py @@ -94,7 +94,7 @@ def groups(self, self.logger.debug("SERVICE_GROUPS=%s" % json.dumps(SERVICE_GROUPS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -162,7 +162,7 @@ def group(self, GROUP_ID) self.logger.debug("DETAIL_GROUP=%s" % json.dumps(DETAIL_GROUP, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -254,7 +254,7 @@ def updateGroup(self, GROUP_ID, GROUP_DATA_VALUE) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -343,7 +343,7 @@ def removeGroup(self, self.idm.removeGroup(SERVICE_ADMIN_TOKEN, GROUP_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -428,7 +428,7 @@ def createNewServiceGroup(self, NEW_SERVICE_GROUP_DESCRIPTION) self.logger.debug("ID of group %s: %s" % (NEW_SERVICE_GROUP_NAME, ID_GROUP)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/LdapGroupHelper.py b/src/orchestrator/core/flow/LdapGroupHelper.py index 83e3b5b0..eb7f32a7 100644 --- a/src/orchestrator/core/flow/LdapGroupHelper.py +++ b/src/orchestrator/core/flow/LdapGroupHelper.py @@ -62,7 +62,7 @@ def createNewGroup(self, return {} else: raise Exception(400, "None group was created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating group %s: %s" % ( NEW_GROUP_NAME, ex)) @@ -93,7 +93,7 @@ def deleteGroup(self, return {} else: raise Exception(400, "None group deleted in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR deleting group %s: %s" % ( GROUP_NAME, ex)) @@ -123,7 +123,7 @@ def listGroups(self, return res else: raise Exception(404, "None groups were retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving groups %s: %s" % ( FILTER, ex)) @@ -155,7 +155,7 @@ def getGroupDetailByAdmin(self, return group else: raise Exception(400, "None group detail was retrieved from ldap: group %s" % (group)) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving group detail %s: %s" % ( GROUP_NAME, ex)) @@ -190,7 +190,7 @@ def updateGroup(self, return res else: raise Exception(400, "None group was updated in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR updating group %s: %s" % ( GROUP_NAME, ex)) diff --git a/src/orchestrator/core/flow/LdapUserHelper.py b/src/orchestrator/core/flow/LdapUserHelper.py index 7b76dcd9..dcb56135 100644 --- a/src/orchestrator/core/flow/LdapUserHelper.py +++ b/src/orchestrator/core/flow/LdapUserHelper.py @@ -85,7 +85,7 @@ def createNewUser(self, return {} else: raise Exception(400, "None user was created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_USER_NAME, ex)) @@ -117,7 +117,7 @@ def askForCreateNewUser(self, return res else: raise Exception(400, "None user was asked to be created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR asking for create user %s: %s" % ( NEW_USER_NAME, ex)) @@ -154,7 +154,7 @@ def deleteUser(self, return {} else: raise Exception(400, "None user deleted in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR deleting user %s: %s" % ( USER_NAME, ex)) @@ -184,7 +184,7 @@ def listUsers(self, return res else: raise Exception(404, "None users were retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving users %s: %s" % ( FILTER, ex)) @@ -211,7 +211,7 @@ def getUserDetail(self, return res else: raise Exception(400, "None user detail was retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving user detail %s: %s" % ( USER_NAME, ex)) @@ -252,7 +252,7 @@ def getUserDetailByAdmin(self, return user else: raise Exception(400, "None user detail was retrieved from ldap: user %s groups %s" % (user, groups)) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving user detail %s: %s" % ( USER_NAME, ex)) @@ -279,7 +279,7 @@ def authUser(self, return res else: raise Exception(401, "None user was auth by ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR authenticating user %s: %s" % ( USER_NAME, ex)) @@ -318,7 +318,7 @@ def updateUser(self, NEW_USER_NAME, GROUP_NAME) self.logger.debug("res=%s" % res) - elif USERNAME and USER_PASSWORD: + elif USER_NAME and USER_PASSWORD: res = self.ldap.updateUserByHimself( USER_NAME, USER_PASSWORD, @@ -330,7 +330,7 @@ def updateUser(self, return res else: raise Exception(400, "None user was updated in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR updating user %s: %s" % ( USER_NAME, ex)) diff --git a/src/orchestrator/core/flow/Projects.py b/src/orchestrator/core/flow/Projects.py index 43105945..6ed4c508 100644 --- a/src/orchestrator/core/flow/Projects.py +++ b/src/orchestrator/core/flow/Projects.py @@ -88,7 +88,7 @@ def projects(self, self.logger.debug("PROJECTS=%s" % json.dumps(PROJECTS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -164,7 +164,7 @@ def get_project(self, try: PROJECT = self.idm.getProject(ADMIN_TOKEN, PROJECT_ID) - except Exception, ex: + except Exception as ex: PROJECT = { 'project': { 'description': PROJECT_NAME, @@ -177,7 +177,7 @@ def get_project(self, PROJECT['project']['name']=PROJECT_NAME self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -269,7 +269,7 @@ def update_project(self, self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -384,7 +384,7 @@ def delete_project(self, self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -520,7 +520,7 @@ def activate_module(self, subscriptionid = cb_res['subscriptionId'] self.logger.debug("subscription id=%s" % subscriptionid) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -638,7 +638,7 @@ def deactivate_module(self, sub['id']) break - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -741,7 +741,7 @@ def list_activated_modules(self, modules = self.cb.extract_modules_from_subscriptions(self, IOTMODULES, cb_res) self.logger.debug("modules=%s" % json.dumps(modules, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Roles.py b/src/orchestrator/core/flow/Roles.py index 0c3d0c47..022a5669 100644 --- a/src/orchestrator/core/flow/Roles.py +++ b/src/orchestrator/core/flow/Roles.py @@ -102,7 +102,7 @@ def roles(self, self.logger.debug("ROLES=%s" % json.dumps(ROLES, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -292,7 +292,7 @@ def roles_assignments(self, self.logger.debug("ROLES=%s" % json.dumps(role_assignments_expanded, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -424,7 +424,7 @@ def assignRoleServiceUser(self, SERVICE_ID, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -561,7 +561,7 @@ def assignRoleSubServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -673,7 +673,7 @@ def assignInheritRoleServiceUser(self, SERVICE_USER_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -786,7 +786,7 @@ def revokeRoleServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -915,7 +915,7 @@ def revokeRoleSubServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1026,7 +1026,7 @@ def revokeInheritRoleServiceUser(self, SERVICE_ID, SERVICE_USER_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1219,7 +1219,7 @@ def roles_assignments_groups(self, self.logger.debug("ROLES=%s" % json.dumps(role_assignments_expanded, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1352,7 +1352,7 @@ def assignRoleServiceGroup(self, SERVICE_ID, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1490,7 +1490,7 @@ def assignRoleSubServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1603,7 +1603,7 @@ def assignInheritRoleServiceGroup(self, SERVICE_GROUP_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1717,7 +1717,7 @@ def revokeRoleServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1847,7 +1847,7 @@ def revokeRoleSubServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1958,7 +1958,7 @@ def revokeInheritRoleServiceGroup(self, SERVICE_ID, SERVICE_GROUP_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2047,7 +2047,7 @@ def removeRole(self, SERVICE_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2161,7 +2161,7 @@ def setPolicyRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2273,7 +2273,7 @@ def removePolicyFromRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2384,7 +2384,7 @@ def getPolicyFromRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Users.py b/src/orchestrator/core/flow/Users.py index c17861fc..948e606f 100644 --- a/src/orchestrator/core/flow/Users.py +++ b/src/orchestrator/core/flow/Users.py @@ -107,7 +107,7 @@ def users(self, # Listar los usuarios de un Subservicio - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -175,7 +175,7 @@ def user(self, USER_ID) self.logger.debug("DETAIL_USER=%s" % json.dumps(DETAIL_USER, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 562bd980..afeac530 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -40,8 +40,6 @@ from settings.dev import IOTMODULES from settings import dev as settings - - class FlowBase(object): def __init__(self, KEYSTONE_PROTOCOL, @@ -72,7 +70,7 @@ def __init__(self, FROM=None): # Generate Transaction ID - self.TRANSACTION_ID = uuid.uuid4() + self.TRANSACTION_ID = str(uuid.uuid4()) if not CORRELATOR_ID: self.CORRELATOR_ID = self.TRANSACTION_ID @@ -154,31 +152,41 @@ def composeErrorCode(self, ex): # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] # print(exc_type, fname, exc_tb.tb_lineno) res = {"error": str(ex), "code": 500} - if isinstance(ex.args, tuple) and ( - (len(ex.args) > 0) and - not isinstance(ex.args[0], tuple)): # Python 2.6 - res['code'] = ex.args[0] - if res['code'] == 400 and len(ex.args) > 1 and \ - ex.args[1].startswith('SPASSWORD'): - res['error'] = ex.args[1] - elif isinstance(ex.message, tuple): # Python 2.7 - res['code'] = ex.message[0] - if res['code'] == 400 and len(ex.message) > 1 and \ - ex.message[1].startswith('SPASSWORD'): - res['error'] = ex.message[1] - return res, None, None + try: + if isinstance(ex.args, tuple) and ( + (len(ex.args) > 0)): + if not isinstance(ex.args[0], tuple): # Python 2.6 + res['code'] = ex.args[0] + if res['code'] == 400 and len(ex.args) > 1 and \ + ex.args[1].startswith('SPASSWORD'): + res['error'] = ex.args[1] + else: # Python 3 + res['code'] = ex.args[0][0] + if res['code'] == 400 and len(ex.args[0]) > 1 and \ + ex.args[0][1].startswith('SPASSWORD'): + res['error'] = ex.args[0][1] + elif isinstance(ex.message, tuple): # Python 2.7 + res['code'] = ex.message[0] + if res['code'] == 400 and len(ex.message) > 1 and \ + ex.message[1].startswith('SPASSWORD'): + res['error'] = ex.message[1] + finally: + return res, None, None def logError(self, logger, error_code, ex): ''' Log as error level error_code if is < 400 or > 500 ''' - if (error_code[0]['code'] < 400 or error_code[0]['code'] > 500): - logger.error(ex) - else: + try: + if (isinstance(error_code[0]['code'], int) and + (error_code[0]['code'] < 400 or error_code[0]['code'] >= 500)): + logger.error(ex) + else: + logger.debug(ex) + except: logger.debug(ex) - def get_endpoint_iot_module(self, iot_module): assert iot_module in IOTMODULES if iot_module in self.endpoints: @@ -218,7 +226,7 @@ def ensure_service_name(self, USER_TOKEN, SERVICE_ID, SERVICE_NAME): SERVICE_NAME = self.idm.getDomainNameFromToken( USER_TOKEN, SERVICE_ID) - except Exception, ex: + except Exception as ex: # This op could be executed by cloud_admin user SERVICE = self.idm.getDomain(USER_TOKEN, SERVICE_ID) @@ -235,7 +243,7 @@ def ensure_subservice_name(self, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID) - except Exception, ex: + except Exception as ex: # This op could be executed by cloud_admin user SUBSERVICE = self.idm.getProject(USER_TOKEN, SUBSERVICE_ID) @@ -263,7 +271,7 @@ def get_extended_token(self, USER_TOKEN): token_extended['project'] = \ token_detail['token']['project']['name'][1:] - except Exception, ex: + except Exception as ex: # Probably expired? token_extended = { "token": USER_TOKEN, @@ -283,7 +291,7 @@ def collectComponentMetrics(self): all.append(self.perseo.PerseoRestOperations.getOutgoingMetrics()) # TODO: Take care of the following operation takes too much time self.sum = reduce(lambda x, y: dict((k, v + y[k]) for k, v in x.iteritems()), all) - except Exception, ex: + except Exception as ex: self.logger.error("ERROR collecting component metrics %s", ex) def getFlowMetrics(self): diff --git a/src/orchestrator/core/flow/createNewService.py b/src/orchestrator/core/flow/createNewService.py index 57008579..9c5c915d 100644 --- a/src/orchestrator/core/flow/createNewService.py +++ b/src/orchestrator/core/flow/createNewService.py @@ -113,7 +113,7 @@ def createNewService(self, NEW_SERVICE_ADMIN_PASSWORD, NEW_SERVICE_ADMIN_EMAIL, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_SERVICE_ADMIN_USER, ex)) @@ -121,7 +121,7 @@ def createNewService(self, try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) return self.composeErrorCode(ex) @@ -160,7 +160,7 @@ def createNewService(self, ID_NEW_SERVICE_ROLE_SERVICECUSTOMER] = self.idm.createDomainRoles(NEW_SERVICE_ADMIN_TOKEN, ROLES, ID_DOM1) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("creating roles in a slow way") ID_NEW_SERVICE_ROLE_SUBSERVICEADMIN = self.idm.createDomainRole( @@ -210,7 +210,7 @@ def createNewService(self, ID_NEW_SERVICE_ROLE_ADMIN_T] = self.idm.createDomainRoles(NEW_SERVICE_ADMIN_TOKEN, ROLES, ID_DOM1) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("creating roles in a slow way") ID_NEW_SERVICE_ROLE_SUBSERVICEADMIN_T = self.idm.createDomainRole( @@ -383,14 +383,14 @@ def createNewService(self, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating groups %s" % ( ex)) self.logger.info("removing uncomplete created domain %s" % ID_DOM1) try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) return self.composeErrorCode(ex) @@ -442,13 +442,13 @@ def createNewService(self, self.mongodb.createSTHIndexes(NEW_SERVICE_NAME, '') - except Exception, ex: + except Exception as ex: if ID_DOM1: self.logger.info("removing uncomplete created domain %s" % ID_DOM1) try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) diff --git a/src/orchestrator/core/flow/createNewServiceRole.py b/src/orchestrator/core/flow/createNewServiceRole.py index 193969cd..a5d20177 100644 --- a/src/orchestrator/core/flow/createNewServiceRole.py +++ b/src/orchestrator/core/flow/createNewServiceRole.py @@ -127,7 +127,7 @@ def createNewServiceRole(self, ID_ROLE, POLICY_FILE_NAME='policy-keypass-customer2.xml') - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/createNewServiceUser.py b/src/orchestrator/core/flow/createNewServiceUser.py index ca202f71..ca4d5697 100644 --- a/src/orchestrator/core/flow/createNewServiceUser.py +++ b/src/orchestrator/core/flow/createNewServiceUser.py @@ -107,7 +107,7 @@ def createNewServiceUser(self, NEW_USER_DESCRIPTION) self.logger.debug("ID of user %s: %s" % (NEW_USER_NAME, ID_USER)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/createNewSubService.py b/src/orchestrator/core/flow/createNewSubService.py index 01f33807..f1ee3aa8 100644 --- a/src/orchestrator/core/flow/createNewSubService.py +++ b/src/orchestrator/core/flow/createNewSubService.py @@ -125,7 +125,7 @@ def createNewSubService(self, NEW_SUBSERVICE_ADMIN_PASSWORD, NEW_SUBSERVICE_ADMIN_EMAIL, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_SUBSERVICE_ADMIN_USER, ex)) @@ -151,7 +151,7 @@ def createNewSubService(self, self.mongodb.createSTHIndexes(SERVICE_NAME, NEW_SUBSERVICE_NAME) - except Exception, ex: + except Exception as ex: if ID_PRO1: self.logger.info("removing uncomplete created project %s" % ID_PRO1) self.idm.disableProject(SERVICE_ADMIN_TOKEN, SERVICE_ID, ID_PRO1) diff --git a/src/orchestrator/core/flow/createTrustToken.py b/src/orchestrator/core/flow/createTrustToken.py index f8e5a6b5..5da361ac 100644 --- a/src/orchestrator/core/flow/createTrustToken.py +++ b/src/orchestrator/core/flow/createTrustToken.py @@ -178,7 +178,7 @@ def createTrustToken(self, self.logger.debug("ID of Trust %s" % (ID_TRUST)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -275,7 +275,7 @@ def getTrustsUserTrustee(self, self.logger.debug("Trusts %s" % (TRUSTS)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/removeUser.py b/src/orchestrator/core/flow/removeUser.py index 342e49a0..ac95b67c 100644 --- a/src/orchestrator/core/flow/removeUser.py +++ b/src/orchestrator/core/flow/removeUser.py @@ -104,7 +104,7 @@ def removeUser(self, USER_ID) # self.logger.debug("ID of user %s: %s" % (USER_NAME, ID_USER)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/updateUser.py b/src/orchestrator/core/flow/updateUser.py index 4bcabdb1..8acb964f 100644 --- a/src/orchestrator/core/flow/updateUser.py +++ b/src/orchestrator/core/flow/updateUser.py @@ -105,7 +105,7 @@ def updateUser(self, USER_ID, USER_DATA_VALUE) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -197,7 +197,7 @@ def changeUserPassword(self, SERVICE_USER_PASSWORD, NEW_USER_PASSWORD) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/keypass.py b/src/orchestrator/core/keypass.py index 717a85ae..1480dbd9 100644 --- a/src/orchestrator/core/keypass.py +++ b/src/orchestrator/core/keypass.py @@ -55,7 +55,7 @@ def __init__(self, self.policy_dir = os.path.dirname(policies.__file__) def checkAccC(self): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/', method='GET', data=None) @@ -82,7 +82,7 @@ def provisionPolicyByContent(self, SERVICE_ROLE_ID, POLICY_CONTENT): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='POST', json_data=False, @@ -97,7 +97,7 @@ def deleteTenantPolicies(self, SERVICE_NAME, SERVICE_ADMIN_TOKEN): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1', method='DELETE', json_data=False, @@ -111,7 +111,7 @@ def getRolePolicies(self, SERVICE_ADMIN_TOKEN, SERVICE_ROLE_ID): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='GET', json_data=False, @@ -119,7 +119,7 @@ def getRolePolicies(self, fiware_service=SERVICE_NAME) assert res.code == 200, (res.code, res.msg) - body_data = res.read() + body_data = res.content.decode() logger.debug("data response: %s" % body_data) return body_data @@ -129,7 +129,7 @@ def getRolePolicy(self, SERVICE_ROLE_ID, POLICY_NAME): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+ SERVICE_ROLE_ID + '/policy/' + POLICY_NAME, method='GET', json_data=False, @@ -137,7 +137,7 @@ def getRolePolicy(self, fiware_service=SERVICE_NAME) assert res.code == 200, (res.code, res.msg) - body_data = res.read() + body_data = res.content.decode() logger.debug("data response: %s" % body_data) return body_data @@ -146,7 +146,7 @@ def deleteRolePolicies(self, SERVICE_ADMIN_TOKEN, SERVICE_ROLE_ID): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='DELETE', json_data=False, @@ -161,7 +161,7 @@ def deleteRolePolicy(self, SERVICE_ROLE_ID, POLICY_NAME): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+ SERVICE_ROLE_ID + '/policy/' + POLICY_NAME, method='DELETE', json_data=False, diff --git a/src/orchestrator/core/mailer.py b/src/orchestrator/core/mailer.py index ab6f1ed2..540d076e 100644 --- a/src/orchestrator/core/mailer.py +++ b/src/orchestrator/core/mailer.py @@ -99,7 +99,7 @@ def sendMail(self, to=None, subject=None, text=None): try: server.sendmail(self.smtp_from, dest, msg) - except Exception, ex: # try to avoid catching Exception unless you have too + except Exception as ex: # try to avoid catching Exception unless you have too logger.error('MAILER003: SMTP sendmail error %s' % ex) return { "error": "SMTP sendmail error %s" % ex } finally: diff --git a/src/orchestrator/core/mongo.py b/src/orchestrator/core/mongo.py index 6c4bbb15..32834ba8 100644 --- a/src/orchestrator/core/mongo.py +++ b/src/orchestrator/core/mongo.py @@ -58,7 +58,7 @@ def createOrionIndexes(self, SERVICE_NAME): ("_id.id", pymongo.ASCENDING), ("_id.type", pymongo.ASCENDING)]) db.entities.create_index("creDate") - except Exception, e: + except Exception as e: logger.warn("createIndex database %s exception: %s" % (databaseName,e)) @@ -78,7 +78,7 @@ def createSTHIndexes(self, SERVICE_NAME, SUBSERVICE_NAME): ("entityType", pymongo.ASCENDING), ("attrName", pymongo.ASCENDING), ("recvTime", pymongo.ASCENDING)]) - except Exception, e: + except Exception as e: logger.warn("createIndex database %s exception: %s" % (databaseName,e)) def removeDatabases(self, SERVICE_NAME): @@ -87,5 +87,5 @@ def removeDatabases(self, SERVICE_NAME): self.client.drop_database(databaseName) databaseName = 'sth_' + SERVICE_NAME self.client.drop_database(databaseName) - except Exception, e: + except Exception as e: logger.warn("remove database %s exception: %s" % (databaseName,e)) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index b660b767..aaa39134 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -47,12 +47,12 @@ def __init__(self, self.LDAP_BASEDN = LDAP_BASEDN def checkLdap(self): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) # Just for check connection conn.simple_bind_s("","") def bindAdmin(self, USERNAME, PASSWORD): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) conn.protocol_version = ldap.VERSION3 username = "cn=" + USERNAME + "," + self.LDAP_BASEDN logger.debug("bind admin %s" % username) @@ -60,7 +60,7 @@ def bindAdmin(self, USERNAME, PASSWORD): return conn def bindUser(self, USERNAME, PASSWORD): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) conn.protocol_version = ldap.VERSION3 username = "uid=" + USERNAME + ", ou=users," + self.LDAP_BASEDN logger.debug("bind user %s" % username) @@ -82,24 +82,27 @@ def createUser(self, conn = self.bindAdmin(LDAP_ADMIN_USER, LDAP_ADMIN_PASSWORD) dn = "uid=" + NEW_USER_NAME + ",ou=users," + self.LDAP_BASEDN mymodlist = { - "objectClass": ["top", "posixAccount", "shadowAccount", - "organizationalPerson", "inetOrgPerson"], - "uid": [ str(NEW_USER_NAME) ], - "cn": [ str(NEW_USER_NAME) ], - "uidNumber": ["5000"], - "gidNumber": ["10000"], - "loginShell": ["/bin/bash"], - "homeDirectory": ["/home/"+ str(NEW_USER_NAME)], - "mail": str(NEW_USER_EMAIL), - "sn": str(NEW_USER_NAME), - "userPassword": str(NEW_USER_PASSWORD) + "objectClass": [ b"top", + b"posixAccount", + b"shadowAccount", + b"organizationalPerson", + b"inetOrgPerson"], + "uid": [ NEW_USER_NAME.encode('utf-8') ], + "cn": [ NEW_USER_NAME.encode('utf-8') ], + "uidNumber": [b"5000"], + "gidNumber": [b"10000"], + "loginShell": [b"/bin/bash"], + "homeDirectory": [("/home/" + NEW_USER_NAME).encode('utf-8')], + "mail": NEW_USER_EMAIL.encode('utf-8'), + "sn": NEW_USER_NAME.encode('utf-8'), + "userPassword": NEW_USER_PASSWORD.encode('utf-8') } logger.debug("create user mymodlist: %s" % mymodlist) result = conn.add_s(dn, ldap.modlist.addModlist(mymodlist)) logger.debug("ldap create user %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("createUser exception: %s" % e) return { "error": e } @@ -114,7 +117,7 @@ def deleteUserByAdmin(self, logger.debug("ldap delete user by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteUserByAdmin exception: %s" % e) return { "error": e } @@ -128,7 +131,7 @@ def deleteUserByHimself(self, logger.debug("ldap delete user by himself %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteUserByHimself exception: %s" % e) return { "error": e } @@ -139,7 +142,7 @@ def authUser(self, conn = self.bindUser(USER_NAME, USER_PASSWORD) self.unbind(conn) return { "details": "OK" } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("authUser exception: %s" % e) return { "error": e } @@ -172,7 +175,7 @@ def listUsers(self, else: res = { "error": FILTER + " not found" } return res - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("listUsers exception: %s" % e) return { "error": e } @@ -200,13 +203,13 @@ def assignGroupUser(self, old_value['member'] = oldgroupMembers new_value['member'] = groupMembers # Add new group member - new_value['member'].append('uid=' + str(USER_NAME) +',ou=users,' + self.LDAP_BASEDN) + new_value['member'].append(str('uid=' + str(USER_NAME) +',ou=users,' + self.LDAP_BASEDN).encode('utf-8')) mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) logger.debug("ldap assing group user %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("assignGroupUser exception: %s" % e) return { "error": e } @@ -228,12 +231,12 @@ def getUserGroups(self, result_dn = result[0] result_attrs = result[1] if ('cn' in result_attrs and len(result_attrs['cn']) > 0): - groups.append(result_attrs['cn'][0]) + groups.append(result_attrs['cn'][0].decode('utf-8')) logger.debug("ldap groups of user: %s" % json.dumps(groups)) self.unbind(conn) return { "details": groups } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("getUserGroups exception: %s" % e) return { "error": e } @@ -256,10 +259,11 @@ def getUserDetail(self, else: if result_type == ldap.RES_SEARCH_ENTRY: result = result_data[0] - logger.debug("ldap get user detail %s" % json.dumps(result)) + details = {key:[v.decode('utf-8') for v in values] for key, values in result[1].items()} + logger.debug("ldap get user detail %s" % json.dumps(details)) self.unbind(conn) - return { "details": result } - except ldap.LDAPError, e: + return { "details": details } + except ldap.LDAPError as e: logger.warn("getUserDetail exception: %s" % e) return { "error": e } @@ -284,17 +288,17 @@ def updateUserByAdmin(self, new_value[attr] = USER_DATA[userattr] mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update user by admin %s" % json.dumps(result)) + logger.debug("ldap update user by admin %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } - except ldap.LDAPError, e: + return { "details": result[0] } + except ldap.LDAPError as e: logger.warn("updateUserByAdmin exception: %s" % e) return { "error": e } - def updateUserByUser(self, + def updateUserByHimself(self, USER_NAME, USER_PASSWORD, - USER_DETAIL): + USER_DATA): try: conn = self.bindUser(USER_NAME, USER_PASSWORD) dn = "uid=" + USER_NAME + ",ou=users," + self.LDAP_BASEDN @@ -311,10 +315,10 @@ def updateUserByUser(self, new_value[attr] = USER_DATA[userattr] mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update user by user %s" % json.dumps(result)) + logger.debug("ldap update user by user %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } - except ldap.LDAPError, e: + return { "details": result[0] } + except ldap.LDAPError as e: logger.warn("updateUserByUser exception: %s" % e) return { "error": e } @@ -327,17 +331,18 @@ def createGroup(self, conn = self.bindAdmin(LDAP_ADMIN_USER, LDAP_ADMIN_PASSWORD) dn = "cn=" + NEW_GROUP_NAME + ",ou=groups," + self.LDAP_BASEDN mymodlist = { - "objectClass": ["top", "groupofnames"], - "cn": [ str(NEW_GROUP_NAME) ], - 'member' : [ 'ou=groups,dc=openstack,dc=org' ], - "description": str(NEW_GROUP_DESCRIPTION) + "objectClass": [b"top", + b"groupofnames"], + "cn": [ str(NEW_GROUP_NAME).encode('utf-8') ], + 'member' : [ b'ou=groups,dc=openstack,dc=org' ], + "description": str(NEW_GROUP_DESCRIPTION).encode('utf-8') } logger.debug("create group mymodlist: %s" % mymodlist) result = conn.add_s(dn, ldap.modlist.addModlist(mymodlist)) logger.debug("ldap create group %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("createGroup exception: %s" % e) return { "error": e } @@ -352,7 +357,7 @@ def deleteGroupByAdmin(self, logger.debug("ldap delete group by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteGroupByAdmin exception: %s" % e) return { "error": e } @@ -367,7 +372,7 @@ def updateGroupByAdmin(self, old_value = {} new_value = {} results = conn.search_s(dn, ldap.SCOPE_BASE) - logger.debug("ldap update group search results %s" % json.dumps(results)) + logger.debug("ldap update group search results %s" % results) for result in results: result_dn = result[0] result_attrs = result[1] @@ -375,14 +380,14 @@ def updateGroupByAdmin(self, for userattr in ['description']: if attr == userattr: old_value[attr] = result_attrs[userattr] - new_value[attr] = [str(GROUP_DESCRIPTION)] - logger.debug("ldap update group old_value %s new_value %s " % (json.dumps(old_value), json.dumps(new_value))) + new_value[attr] = [str(GROUP_DESCRIPTION).encode('utf-8')] + logger.debug("ldap update group old_value %s new_value %s " % (old_value, new_value)) mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update group by admin %s" % json.dumps(result)) + logger.debug("ldap update group by admin %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } - except ldap.LDAPError, e: + return { "details": result[0] } + except ldap.LDAPError as e: logger.warn("updateGroupByAdmin exception: %s" % e) return { "error": e } @@ -415,6 +420,6 @@ def listGroups(self, else: res = { "error": FILTER + " not found" } return res - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("listGroups exception: %s" % e) return { "error": e } diff --git a/src/orchestrator/core/orion.py b/src/orchestrator/core/orion.py index 6cc34363..f1cc05e4 100644 --- a/src/orchestrator/core/orion.py +++ b/src/orchestrator/core/orion.py @@ -209,7 +209,7 @@ def deleteAllSubscriptions(self, SUBSERVICE_NAME) logger.debug("subscriptions: %s" % json.dumps(subscriptions, indent=3)) - except Exception, ex: + except Exception as ex: logger.warn("%s trying getListSubscriptions from CB: %s/%s" % (ex, SERVICE_NAME, SUBSERVICE_NAME)) @@ -222,7 +222,7 @@ def deleteAllSubscriptions(self, SUBSERVICE_NAME, subscription['id']) subscriptions_deleted.append(subscription['id']) - except Exception, ex: + except Exception as ex: logger.warn("%s trying to unsubscribe context: %s" % (ex, subscription['id'])) return subscriptions_deleted diff --git a/src/orchestrator/core/perseo.py b/src/orchestrator/core/perseo.py index a5b7546e..064b5661 100644 --- a/src/orchestrator/core/perseo.py +++ b/src/orchestrator/core/perseo.py @@ -90,7 +90,7 @@ def deleteAllRules(self, rules = json.loads(data) logger.debug("rules: %s" % json.dumps(rules, indent=3)) - except Exception, ex: + except Exception as ex: logger.warn("%s trying getRules from PERSEO: %s/%s" % (ex, SERVICE_NAME, SUBSERVICE_NAME)) @@ -113,7 +113,7 @@ def deleteAllRules(self, fiware_service_path='/'+SUBSERVICE_NAME) assert res.code == 204, (res.code, res.msg) rules_deleted.append(rule['name']) - except Exception, ex: + except Exception as ex: logger.warn("%s trying to remove rule: %s" % (ex, rule['name'])) diff --git a/src/orchestrator/core/startup.py b/src/orchestrator/core/startup.py index 57026039..bad21f7c 100644 --- a/src/orchestrator/core/startup.py +++ b/src/orchestrator/core/startup.py @@ -91,7 +91,7 @@ def check_endpoints(): try: idm.checkIdM() logger.info("Keystone endpoint OK") - except Exception, ex: + except Exception as ex: logger.error("keystone endpoint not found: %s" % ex) return "ERROR keystone endpoint not found: %s" % ex @@ -99,7 +99,7 @@ def check_endpoints(): try: ac.checkAccC() logger.info("Keypass endpoint OK") - except Exception, ex: + except Exception as ex: logger.error("keypass endpoint not found: %s" % ex) return "ERROR keypass endpoint not found: %s" % ex @@ -109,7 +109,7 @@ def check_endpoints(): try: mongo.checkMongo() logger.info("MongoDB endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("MongoDB endpoint not found: %s" % ex) # ContextBroker: optional @@ -120,7 +120,7 @@ def check_endpoints(): try: orion.checkCB() logger.info("Orion endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("Orion endpoint not found: %s" % ex) # Perseo: optional @@ -131,7 +131,7 @@ def check_endpoints(): try: orion.checkPERSEO() logger.info("PERSEO endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("PERSEO endpoint not found: %s" % ex) # OpenLDAP: optional @@ -142,7 +142,7 @@ def check_endpoints(): try: openldap.checkLdap() logger.info("LDAP endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("LDAP endpoint not found: %s" % ex) # Mailer: optional @@ -152,7 +152,7 @@ def check_endpoints(): try: mailer.checkMailer() logger.info("MAILER endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("MAILER endpoint not found: %s" % ex) return "OK" diff --git a/src/settings/dev.py b/src/settings/dev.py index 2b7d00ed..bcce2126 100644 --- a/src/settings/dev.py +++ b/src/settings/dev.py @@ -8,7 +8,7 @@ stipulated in the agreement/contract under which the program(s) have been supplied. """ -from common import * # noqa +from settings.common import * # noqa DEBUG = False TEMPLATE_DEBUG = DEBUG diff --git a/src/tests/api/test_ldap_api.py b/src/tests/api/test_ldap_api.py index bdbdc4f9..254c663b 100644 --- a/src/tests/api/test_ldap_api.py +++ b/src/tests/api/test_ldap_api.py @@ -32,6 +32,7 @@ USER_NAME="adm" USER_PASSWORD="4pass1w0rd" +GROUP_NAME="group" ORC_PROTOCOL="http" ORC_HOST="localhost" @@ -90,6 +91,15 @@ def __init__(self): "SubServiceAdminGroup"] } } + self.payload_data5b_ok = { + "USER_NAME": USER_NAME+"_%s" % self.suffix, + "USER_PASSWORD": USER_PASSWORD, + "USER_DATA": {"USER_EMAIL": "pepe33@acme.es", + "USER_DESCRIPTION": "Pepe perez", + "GROUP_NAMES": ["ServiceCustomerGroup", + "SubServiceAdminGroup"] + } + } self.suffix = str(uuid.uuid4())[:8] self.payload_data4_ok = { "NEW_USER_NAME": USER_NAME+"_%s" % self.suffix, @@ -169,6 +179,14 @@ def test_put_ok(self): data=self.payload_data5_ok) assert res.code == 200, (res.code, res.msg, res.raw_json) + def test_put2_ok(self): + res = self.TestRestOps.rest_request( + method="PUT", + url="/v1.0/ldap/user", + json_data=True, + data=self.payload_data5b_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + def test_delete_ok(self): res = self.TestRestOps.rest_request( method="DELETE", @@ -244,6 +262,69 @@ def test_post_and_delete_ok(self): assert res.code == 204, (res.code, res.msg, res.raw_json) +class Test_LDAPGroup_RestView(object): + + def __init__(self): + self.suffix = str(uuid.uuid4())[:8] + self.payload_data_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "NEW_GROUP_NAME": GROUP_NAME+"_%s" % self.suffix, + "NEW_USER_DESCRIPTION": GROUP_NAME+"_%s description" % self.suffix, + } + self.payload_data1b_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "GROUP_NAME": GROUP_NAME+"_%s" % self.suffix + } + self.payload_data1_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "FILTER": "*"+GROUP_NAME+"_%s*" % self.suffix + } + self.payload_data5_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "GROUP_NAME": GROUP_NAME+"_%s" % self.suffix, + "GROUP_DESCRIPTION": GROUP_NAME+"_%s description alt" % self.suffix, + } + self.TestRestOps = TestRestOperations(PROTOCOL=ORC_PROTOCOL, + HOST=ORC_HOST, + PORT=ORC_PORT) + + def test_post_ok(self): + res = self.TestRestOps.rest_request( + method="POST", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data_ok) + assert res.code == 201, (res.code, res.msg, res.raw_json) + + def test_delete_ok(self): + res = self.TestRestOps.rest_request( + method="DELETE", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data1b_ok) + assert res.code == 204, (res.code, res.msg, res.raw_json) + + def test_get_ok(self): + res = self.TestRestOps.rest_request( + method="GET", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data1_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + + def test_put_ok(self): + res = self.TestRestOps.rest_request( + method="PUT", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data5_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + + if __name__ == '__main__': # Tests @@ -255,6 +336,7 @@ def test_post_and_delete_ok(self): test_LdapUser.test_get2_ok() test_LdapUser.test_get3_ok() test_LdapUser.test_put_ok() + test_LdapUser.test_put2_ok() test_LdapUser.test_delete_ok() test_LdapUser.test_delete_bad() test_LdapUser.test_post2_ok() @@ -262,3 +344,10 @@ def test_post_and_delete_ok(self): test_LdapAuth = Test_LDAPAuth_RestView() test_LdapAuth.test_post_and_delete_ok() + + + test_LdapGroup = Test_LDAPGroup_RestView() + test_LdapGroup.test_post_ok() + test_LdapGroup.test_get_ok() + test_LdapGroup.test_put_ok() + test_LdapGroup.test_delete_ok()