Merge pull request #313 from telefonicaid/task/upgrade_python_3b

upgrade to python3 (& Django & DjangoRestFramework)

ChangeLog

@@ -1,6 +1,11 @@
 CHANGES
 =======
-UPDATE: Update jsonschema dep 2.5.0 to 2.5.1
+Upgrade Python (2 -> 3), #242
+        Django (1.11 -> 2.2.24) #310
+        DjangoRestFramework (3.9 -> 3.11) #309
+        requests (2.22.0 -> 2.25.1)
+        python-ldap (2.5.2 -> 3.3.1)
+UPDATE: Update jsonschema dep 2.5.0 to 2.6.0
 Upgrade STH xacml policies with orion frns
 
 3.8.0

DOCKER.md

@@ -104,7 +104,7 @@ Additionally, the following environment variables are available for orchestrator
 | PROCESSES                   |                           | 6                         |
 | THREADS                     |                           | 8                         |
 | HARAKIRI                    |                           | 80                        |
-| HTTP_TIMEOUT                |                           | 200                        |
+| HTTP_TIMEOUT                |                           | 200                       |
 | MAX_REQUESTS                |                           | 250                       |
 | UWSGI_BUFFER_SIZE           |                           | 4096                      |
 | KEYSTONE_HOST               | KEYSTONE.host             | localhost                 |

Dockerfile

@@ -8,7 +8,7 @@ ENV ORCHESTRATOR_USER_UID 10001
 
 ENV ORCHESTRATOR_VERSION 3.8.0
 
-ENV python_lib /var/env-orchestrator/lib/python2.7/site-packages
+ENV python_lib /var/env-orchestrator/lib/python3.6/site-packages
 ENV DJANGO_SETTINGS_MODULE settings
 ENV PYTHONPATH "${PYTHONPATH}:/opt/orchestrator"
 
@@ -20,15 +20,15 @@ RUN \
     adduser --comment "${ORCHESTRATOR_USER}" -u ${ORCHESTRATOR_USER_UID} ${ORCHESTRATOR_USER} && \
     # Install dependencies
     yum install -y epel-release && yum update -y epel-release && \
-    yum install -y yum-plugin-remove-with-leaves python python-pip python-devel openldap-devel python-virtualenv gcc ssh && \
+    yum install -y yum-plugin-remove-with-leaves python3 python3-pip python3-devel openldap-devel python3-virtualenv gcc ssh && \
     yum install -y tcping findutils sed && \
     mkdir -p $python_lib/iotp-orchestrator && \
     mkdir -p $python_lib/iotp-orchestrator/bin && \
     cp -rp /opt/sworchestrator/src/* $python_lib/iotp-orchestrator && cp -p /opt/sworchestrator/requirements.txt $python_lib/iotp-orchestrator && \
     cp -rp /opt/sworchestrator/bin $python_lib/iotp-orchestrator && \
     chmod 755 $python_lib/iotp-orchestrator/bin/orchestrator-entrypoint.sh && \
     chown -R ${ORCHESTRATOR_USER}:${ORCHESTRATOR_USER} $python_lib/iotp-orchestrator && \
-    pip install -r $python_lib/iotp-orchestrator/requirements.txt && \
+    pip3 install -r $python_lib/iotp-orchestrator/requirements.txt && \
     find $python_lib/iotp-orchestrator -name "*.pyc" -delete && \
     ln -s $python_lib/iotp-orchestrator /opt/orchestrator && \
     ln -s /opt/orchestrator/orchestrator/commands /opt/orchestrator/bin/ && \

INSTALL.md

@@ -12,7 +12,7 @@ Due to its stateless behavior, iotp-orchestrator does not require any backup pro
  * Internet access
  * Connectivity to http://github.com
  * [Git](http://git-scm.com/)
- * Python 2.7 (or upper) and pip
+ * Python 3 and pip3
  * C and C++ compilation platform: gcc, g++, make and python headers
  * UNIX platforms but not mandatory
  * WSGI Web server (Apache2/Nginx) but not mandatory
@@ -42,13 +42,13 @@ cd src
 
 Then install all dependencies by running:
 ```
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 ```
 
 
 Start server in 8084 port using django web server:
 ```
-python manage.py runserver 8084 --settings=settings.dev
+python3 manage.py runserver 8084 --settings=settings.dev
 ```
 
 or using another web server like uWSGI

README.md

@@ -29,9 +29,8 @@ Orchestrator is used to:
 - Create, List, Modify LDAP Groups
 
 Orchestrator is based mainly on:
-- Python 2.7 needed
+- Python 3 needed
 - Django / DjangoRestFramework
-- httplib
 
 Orchestrator relies on these other IoT parts:
 - Identity Manager: Keystone (mandatory)

TESTS.md

@@ -4,29 +4,29 @@
 - [Unit Tests](./src/tests/core/test_idm.py)
 Run unit tests with:
 ```
-        $ python ./test_idm.py
+        $ python3 ./test_idm.py
 ```
 
 
 - [End2End Tests](./src/tests/api/test_api.py)
 Run e2e tests with:
 
 ```
-        $ python ./test_api.py
+        $ python3 ./test_api.py
 ```
 
 - [End2End LDAP Tests](./src/tests/api/test_ldap_api.py)
 Run e2e LDAP tests with:
 
 ```
-        $ python ./test_ldap_api.py
+        $ python3 ./test_ldap_api.py
 ```
 
 - [Comamnd Tests](./src/tests/api/test_commands.py)
 Run scripts tests with:
 
 ```
-        $ python ./test_commands.py
+        $ python3 ./test_commands.py
 ```
 
 There is a config file to run Orchestrator build into Travis CI service:

orchestrator.spec

@@ -18,11 +18,8 @@ Prefix: /opt
 BuildArch: noarch
 
 %define _target_os Linux
-%define python_lib /var/env-orchestrator/lib/python2.6/site-packages
-%if 0%{?with_python27}
-%define python_lib /var/env-orchestrator/lib/python2.7/site-packages
-%define __python /usr/bin/python2.7
-%endif # if with_python27
+%define python_lib /var/env-orchestrator/lib/python3.6/site-packages
+%define __python /usr/bin/python3.6
 
 %description
 IoT Platform Orchestrator

package-orchestrator.sh

@@ -35,7 +35,6 @@ source $BASE/get_version_string.sh
 string=$(get_rpm_version_string)
 VERSION_VALUE=${string% *}
 RELEASE_VALUE=${string#* }
-PYTHON27_VALUE=0
 
 args=("$@")
 ELEMENTS=${#args[@]}
@@ -48,9 +47,6 @@ for (( i=0;i<$ELEMENTS;i++)); do
     if [ "$arg" == "--with-release" ]; then
         RELEASE_VALUE=${args[${i}+1]}
     fi
-    if [ "$arg" == "--with-python27" ]; then
-        PYTHON27_VALUE=1
-    fi
 done
 
 
@@ -63,6 +59,5 @@ rpmbuild -bb orchestrator.spec \
   --define "_topdir $RPM_DIR" \
   --define "_root $BASE"\
   --define "_project_user $ORCHESTRATOR_USER"\
-  --define "with_python27 $PYTHON27_VALUE"\
   --define "_version $VERSION_VALUE"\
   --define "_release $RELEASE_VALUE"

requirements.txt

@@ -1,7 +1,7 @@
-Django==1.11.29
-djangorestframework==3.9.4
-jsonschema==2.5.1
+Django==2.2.24
+djangorestframework==3.11.2
+jsonschema==2.6.0
 uwsgi==2.0.19.1
-requests==2.22.0
-python-ldap==2.5.2
+requests==2.25.1
+python-ldap==3.3.1
 pymongo==3.7.2

src/orchestrator/api/ldap_view.py

@@ -54,6 +54,9 @@ def post(self, request):
         try:
             request.data  # json validation
             flow = LdapUserHelper(
+                           None, None, None,
+                           None, None, None,
+                           None, None, None,
                            None, None, None,
                            LDAP_HOST=self.LDAP_HOST,
                            LDAP_PORT=self.LDAP_PORT,
@@ -98,8 +101,7 @@ def post(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -111,6 +113,9 @@ def get(self, request):
         try:
             request.data  # json validation
             flow = LdapUserHelper(
+                           None, None, None,
+                           None, None, None,
+                           None, None, None,
                            None, None, None,
                            LDAP_HOST=self.LDAP_HOST,
                            LDAP_PORT=self.LDAP_PORT,
@@ -151,8 +156,7 @@ def get(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -164,6 +168,9 @@ def put(self, request):
         try:
             request.data  # json validation
             flow = LdapUserHelper(
+                         None, None, None,
+                         None, None, None,
+                         None, None, None,
                          None, None, None,
                          LDAP_HOST=self.LDAP_HOST,
                          LDAP_PORT=self.LDAP_PORT,
@@ -188,8 +195,7 @@ def put(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -202,6 +208,9 @@ def delete(self, request):
         try:
             request.data  # json validation
             flow = LdapUserHelper(
+                         None, None, None,
+                         None, None, None,
+                         None, None, None,
                          None, None, None,
                          LDAP_HOST=self.LDAP_HOST,
                          LDAP_PORT=self.LDAP_PORT,
@@ -225,8 +234,7 @@ def delete(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -250,6 +258,9 @@ def post(self, request):
         try:
             request.data  # json validation
             flow = LdapUserHelper(
+                           None, None, None,
+                           None, None, None,
+                           None, None, None,
                            None, None, None,
                            LDAP_HOST=self.LDAP_HOST,
                            LDAP_PORT=self.LDAP_PORT,
@@ -272,8 +283,7 @@ def post(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -298,6 +308,9 @@ def post(self, request):
         try:
             request.data  # json validation
             flow = LdapGroupHelper(
+                           None, None, None,
+                           None, None, None,
+                           None, None, None,
                            None, None, None,
                            LDAP_HOST=self.LDAP_HOST,
                            LDAP_PORT=self.LDAP_PORT,
@@ -330,8 +343,7 @@ def post(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -343,6 +355,9 @@ def get(self, request):
         try:
             request.data  # json validation
             flow = LdapGroupHelper(
+                           None, None, None,
+                           None, None, None,
+                           None, None, None,
                            None, None, None,
                            LDAP_HOST=self.LDAP_HOST,
                            LDAP_PORT=self.LDAP_PORT,
@@ -378,8 +393,7 @@ def get(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -392,6 +406,9 @@ def put(self, request):
         try:
             request.data  # json validation
             flow = LdapGroupHelper(
+                         None, None, None,
+                         None, None, None,
+                         None, None, None,
                          None, None, None,
                          LDAP_HOST=self.LDAP_HOST,
                          LDAP_PORT=self.LDAP_PORT,
@@ -415,8 +432,7 @@ def put(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )
@@ -429,6 +445,9 @@ def delete(self, request):
         try:
             request.data  # json validation
             flow = LdapGroupHelper(
+                         None, None, None,
+                         None, None, None,
+                         None, None, None,
                          None, None, None,
                          LDAP_HOST=self.LDAP_HOST,
                          LDAP_PORT=self.LDAP_PORT,
@@ -451,8 +470,7 @@ def delete(self, request):
         except ParseError as error:
             Stats.num_api_errors += 1
             response = Response(
-                'Input validation error - {0} {1}'.format(error.message,
-                                                          error.detail),
+                'Input validation error - {0}'.format(error.detail),
                 status=status.HTTP_400_BAD_REQUEST,
                 headers={"Fiware-Correlator": CORRELATOR_ID}
             )

src/orchestrator/api/parsers.py

@@ -49,7 +49,7 @@ def parse(self, stream, media_type=None, parser_context=None):
                 data,
                 schemas.json[parser_context['view'].schema_name])
         except (ValueError, jsonschema.exceptions.ValidationError) as error:
-            logger.debug(error.message)
-            raise ParseError(detail=error.message)
+            logger.debug(str(error.args[0]))
+            raise ParseError(detail=str(error.args[0]))
         else:
             return data