From daa58065568ed475c3d4d0aee3199d3ff9301e04 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 11 Jun 2021 09:52:03 +0200 Subject: [PATCH 01/33] upgrade to python3 --- Dockerfile | 6 +++--- orchestrator.spec | 4 ++++ package-orchestrator.sh | 5 +++++ requirements.txt | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 137e7868..151809e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV ORCHESTRATOR_USER_UID 10001 ENV ORCHESTRATOR_VERSION 3.8.0 -ENV python_lib /var/env-orchestrator/lib/python2.7/site-packages +ENV python_lib /var/env-orchestrator/lib/python3.6/site-packages ENV DJANGO_SETTINGS_MODULE settings ENV PYTHONPATH "${PYTHONPATH}:/opt/orchestrator" @@ -20,7 +20,7 @@ RUN \ adduser --comment "${ORCHESTRATOR_USER}" -u ${ORCHESTRATOR_USER_UID} ${ORCHESTRATOR_USER} && \ # Install dependencies yum install -y epel-release && yum update -y epel-release && \ - yum install -y yum-plugin-remove-with-leaves python python-pip python-devel openldap-devel python-virtualenv gcc ssh && \ + yum install -y yum-plugin-remove-with-leaves python3 python3-pip python3-devel openldap-devel python3-virtualenv gcc ssh && \ yum install -y tcping findutils sed && \ mkdir -p $python_lib/iotp-orchestrator && \ mkdir -p $python_lib/iotp-orchestrator/bin && \ @@ -28,7 +28,7 @@ RUN \ cp -rp /opt/sworchestrator/bin $python_lib/iotp-orchestrator && \ chmod 755 $python_lib/iotp-orchestrator/bin/orchestrator-entrypoint.sh && \ chown -R ${ORCHESTRATOR_USER}:${ORCHESTRATOR_USER} $python_lib/iotp-orchestrator && \ - pip install -r $python_lib/iotp-orchestrator/requirements.txt && \ + pip3 install -r $python_lib/iotp-orchestrator/requirements.txt && \ find $python_lib/iotp-orchestrator -name "*.pyc" -delete && \ ln -s $python_lib/iotp-orchestrator /opt/orchestrator && \ ln -s /opt/orchestrator/orchestrator/commands /opt/orchestrator/bin/ && \ diff --git a/orchestrator.spec b/orchestrator.spec index ee18c7a0..e506b062 100644 --- a/orchestrator.spec +++ b/orchestrator.spec @@ -23,6 +23,10 @@ BuildArch: noarch %define python_lib /var/env-orchestrator/lib/python2.7/site-packages %define __python /usr/bin/python2.7 %endif # if with_python27 +%if 0%{?with_python3} +%define python_lib /var/env-orchestrator/lib/python3.6/site-packages +%define __python /usr/bin/python3.6 +%endif # if with_python3 %description IoT Platform Orchestrator diff --git a/package-orchestrator.sh b/package-orchestrator.sh index 2e7016fc..ff1ce04f 100755 --- a/package-orchestrator.sh +++ b/package-orchestrator.sh @@ -36,6 +36,7 @@ string=$(get_rpm_version_string) VERSION_VALUE=${string% *} RELEASE_VALUE=${string#* } PYTHON27_VALUE=0 +PYTHON3_VALUE=0 args=("$@") ELEMENTS=${#args[@]} @@ -51,6 +52,9 @@ for (( i=0;i<$ELEMENTS;i++)); do if [ "$arg" == "--with-python27" ]; then PYTHON27_VALUE=1 fi + if [ "$arg" == "--with-python3" ]; then + PYTHON3_VALUE=1 + fi done @@ -64,5 +68,6 @@ rpmbuild -bb orchestrator.spec \ --define "_root $BASE"\ --define "_project_user $ORCHESTRATOR_USER"\ --define "with_python27 $PYTHON27_VALUE"\ + --define "with_python3 $PYTHON3_VALUE"\ --define "_version $VERSION_VALUE"\ --define "_release $RELEASE_VALUE" diff --git a/requirements.txt b/requirements.txt index 9eec0c15..5880a5ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ djangorestframework==3.9.4 jsonschema==2.5.0 uwsgi==2.0.19.1 requests==2.22.0 -python-ldap==2.5.2 +python-ldap==3.3.1 pymongo==3.7.2 From a3e7f15d9a241e4fe388daba1bb1a4009a9b731b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 11 Jun 2021 11:21:59 +0200 Subject: [PATCH 02/33] upgrade deps to python3 --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5880a5ed..adf200ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Django==1.11.29 -djangorestframework==3.9.4 -jsonschema==2.5.0 +Django==2.2.24 +djangorestframework==3.11.2 +jsonschema==2.6.0 uwsgi==2.0.19.1 requests==2.22.0 python-ldap==3.3.1 From 65fefa2d34d0d84564c4f583f395e1ff0c66224f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 10:58:15 +0200 Subject: [PATCH 03/33] update code to python3 --- src/orchestrator/common/util.py | 28 ++++++++++++++-------------- src/orchestrator/core/mailer.py | 2 +- src/orchestrator/core/mongo.py | 6 +++--- src/orchestrator/core/openldap.py | 28 ++++++++++++++-------------- src/orchestrator/core/orion.py | 4 ++-- src/orchestrator/core/perseo.py | 4 ++-- src/orchestrator/core/startup.py | 14 +++++++------- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/orchestrator/common/util.py b/src/orchestrator/common/util.py index 8b33f993..2a81e21c 100644 --- a/src/orchestrator/common/util.py +++ b/src/orchestrator/common/util.py @@ -21,11 +21,11 @@ # # Author: IoT team # -import urllib2 +import urllib.request import base64 import json import csv -import StringIO +import io import requests import logging import time @@ -98,12 +98,12 @@ def rest_request(self, url, method, user=None, password=None, if data: if json_data: - request = urllib2.Request( + request = urllib.request.Request( url, data=json.dumps(data)) else: - request = urllib2.Request(url, data=data) + request = urllib.request.Request(url, data=data) else: - request = urllib2.Request(url) + request = urllib.request.Request(url) request.get_method = lambda: method if json_data: @@ -142,8 +142,8 @@ def rest_request(self, url, method, user=None, password=None, res = None try: - res = urllib2.urlopen(request) - except urllib2.HTTPError, e: + res = urllib.request.urlopen(request) + except urllib.request.HTTPError as e: res = e data = res.read() try: @@ -164,9 +164,9 @@ def rest_request(self, url, method, user=None, password=None, except ValueError: res.msg = data - except Exception, e: - print e - except urllib2.URLError, e: + except Exception as e: + print(e) + except urllib.request.URLError as e: data = None res = e res.code = 500 @@ -253,8 +253,8 @@ def rest_request2(self, url, method, user=None, password=None, data=rdata, verify=False) - except Exception, e: - print e + except Exception as e: + print(e) if settings.ORC_EXTENDED_METRICS: self.collectOutgoingMetrics(service_start, rdata, headers, res) @@ -276,7 +276,7 @@ def collectOutgoingMetrics(self, service_start, data_request, headers_request, r # Check headers self.sum["outgoingTransactionResponseSize"] += len(json.dumps(data_response)) + len(str(response.headers.headers)) if 'headers' in response and 'headers' in response.headers else 0 self.sum["serviceTimeTotal"] += (service_stop - service_start) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR collecting outgoing metrics %s", ex) def getOutgoingMetrics(self): @@ -294,7 +294,7 @@ def __init__(self): @staticmethod def read_devices(CSV): devices = {} - csvreader = csv.reader(StringIO.StringIO(CSV), + csvreader = csv.reader(io.StringIO(CSV), delimiter=',', #quotechar='"', skipinitialspace=True) diff --git a/src/orchestrator/core/mailer.py b/src/orchestrator/core/mailer.py index ab6f1ed2..540d076e 100644 --- a/src/orchestrator/core/mailer.py +++ b/src/orchestrator/core/mailer.py @@ -99,7 +99,7 @@ def sendMail(self, to=None, subject=None, text=None): try: server.sendmail(self.smtp_from, dest, msg) - except Exception, ex: # try to avoid catching Exception unless you have too + except Exception as ex: # try to avoid catching Exception unless you have too logger.error('MAILER003: SMTP sendmail error %s' % ex) return { "error": "SMTP sendmail error %s" % ex } finally: diff --git a/src/orchestrator/core/mongo.py b/src/orchestrator/core/mongo.py index 6c4bbb15..32834ba8 100644 --- a/src/orchestrator/core/mongo.py +++ b/src/orchestrator/core/mongo.py @@ -58,7 +58,7 @@ def createOrionIndexes(self, SERVICE_NAME): ("_id.id", pymongo.ASCENDING), ("_id.type", pymongo.ASCENDING)]) db.entities.create_index("creDate") - except Exception, e: + except Exception as e: logger.warn("createIndex database %s exception: %s" % (databaseName,e)) @@ -78,7 +78,7 @@ def createSTHIndexes(self, SERVICE_NAME, SUBSERVICE_NAME): ("entityType", pymongo.ASCENDING), ("attrName", pymongo.ASCENDING), ("recvTime", pymongo.ASCENDING)]) - except Exception, e: + except Exception as e: logger.warn("createIndex database %s exception: %s" % (databaseName,e)) def removeDatabases(self, SERVICE_NAME): @@ -87,5 +87,5 @@ def removeDatabases(self, SERVICE_NAME): self.client.drop_database(databaseName) databaseName = 'sth_' + SERVICE_NAME self.client.drop_database(databaseName) - except Exception, e: + except Exception as e: logger.warn("remove database %s exception: %s" % (databaseName,e)) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index b660b767..2a822b18 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -99,7 +99,7 @@ def createUser(self, logger.debug("ldap create user %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("createUser exception: %s" % e) return { "error": e } @@ -114,7 +114,7 @@ def deleteUserByAdmin(self, logger.debug("ldap delete user by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteUserByAdmin exception: %s" % e) return { "error": e } @@ -128,7 +128,7 @@ def deleteUserByHimself(self, logger.debug("ldap delete user by himself %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteUserByHimself exception: %s" % e) return { "error": e } @@ -139,7 +139,7 @@ def authUser(self, conn = self.bindUser(USER_NAME, USER_PASSWORD) self.unbind(conn) return { "details": "OK" } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("authUser exception: %s" % e) return { "error": e } @@ -172,7 +172,7 @@ def listUsers(self, else: res = { "error": FILTER + " not found" } return res - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("listUsers exception: %s" % e) return { "error": e } @@ -206,7 +206,7 @@ def assignGroupUser(self, logger.debug("ldap assing group user %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("assignGroupUser exception: %s" % e) return { "error": e } @@ -233,7 +233,7 @@ def getUserGroups(self, logger.debug("ldap groups of user: %s" % json.dumps(groups)) self.unbind(conn) return { "details": groups } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("getUserGroups exception: %s" % e) return { "error": e } @@ -259,7 +259,7 @@ def getUserDetail(self, logger.debug("ldap get user detail %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("getUserDetail exception: %s" % e) return { "error": e } @@ -287,7 +287,7 @@ def updateUserByAdmin(self, logger.debug("ldap update user by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("updateUserByAdmin exception: %s" % e) return { "error": e } @@ -314,7 +314,7 @@ def updateUserByUser(self, logger.debug("ldap update user by user %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("updateUserByUser exception: %s" % e) return { "error": e } @@ -337,7 +337,7 @@ def createGroup(self, logger.debug("ldap create group %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("createGroup exception: %s" % e) return { "error": e } @@ -352,7 +352,7 @@ def deleteGroupByAdmin(self, logger.debug("ldap delete group by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("deleteGroupByAdmin exception: %s" % e) return { "error": e } @@ -382,7 +382,7 @@ def updateGroupByAdmin(self, logger.debug("ldap update group by admin %s" % json.dumps(result)) self.unbind(conn) return { "details": result } - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("updateGroupByAdmin exception: %s" % e) return { "error": e } @@ -415,6 +415,6 @@ def listGroups(self, else: res = { "error": FILTER + " not found" } return res - except ldap.LDAPError, e: + except ldap.LDAPError as e: logger.warn("listGroups exception: %s" % e) return { "error": e } diff --git a/src/orchestrator/core/orion.py b/src/orchestrator/core/orion.py index 6cc34363..f1cc05e4 100644 --- a/src/orchestrator/core/orion.py +++ b/src/orchestrator/core/orion.py @@ -209,7 +209,7 @@ def deleteAllSubscriptions(self, SUBSERVICE_NAME) logger.debug("subscriptions: %s" % json.dumps(subscriptions, indent=3)) - except Exception, ex: + except Exception as ex: logger.warn("%s trying getListSubscriptions from CB: %s/%s" % (ex, SERVICE_NAME, SUBSERVICE_NAME)) @@ -222,7 +222,7 @@ def deleteAllSubscriptions(self, SUBSERVICE_NAME, subscription['id']) subscriptions_deleted.append(subscription['id']) - except Exception, ex: + except Exception as ex: logger.warn("%s trying to unsubscribe context: %s" % (ex, subscription['id'])) return subscriptions_deleted diff --git a/src/orchestrator/core/perseo.py b/src/orchestrator/core/perseo.py index a5b7546e..064b5661 100644 --- a/src/orchestrator/core/perseo.py +++ b/src/orchestrator/core/perseo.py @@ -90,7 +90,7 @@ def deleteAllRules(self, rules = json.loads(data) logger.debug("rules: %s" % json.dumps(rules, indent=3)) - except Exception, ex: + except Exception as ex: logger.warn("%s trying getRules from PERSEO: %s/%s" % (ex, SERVICE_NAME, SUBSERVICE_NAME)) @@ -113,7 +113,7 @@ def deleteAllRules(self, fiware_service_path='/'+SUBSERVICE_NAME) assert res.code == 204, (res.code, res.msg) rules_deleted.append(rule['name']) - except Exception, ex: + except Exception as ex: logger.warn("%s trying to remove rule: %s" % (ex, rule['name'])) diff --git a/src/orchestrator/core/startup.py b/src/orchestrator/core/startup.py index 57026039..bad21f7c 100644 --- a/src/orchestrator/core/startup.py +++ b/src/orchestrator/core/startup.py @@ -91,7 +91,7 @@ def check_endpoints(): try: idm.checkIdM() logger.info("Keystone endpoint OK") - except Exception, ex: + except Exception as ex: logger.error("keystone endpoint not found: %s" % ex) return "ERROR keystone endpoint not found: %s" % ex @@ -99,7 +99,7 @@ def check_endpoints(): try: ac.checkAccC() logger.info("Keypass endpoint OK") - except Exception, ex: + except Exception as ex: logger.error("keypass endpoint not found: %s" % ex) return "ERROR keypass endpoint not found: %s" % ex @@ -109,7 +109,7 @@ def check_endpoints(): try: mongo.checkMongo() logger.info("MongoDB endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("MongoDB endpoint not found: %s" % ex) # ContextBroker: optional @@ -120,7 +120,7 @@ def check_endpoints(): try: orion.checkCB() logger.info("Orion endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("Orion endpoint not found: %s" % ex) # Perseo: optional @@ -131,7 +131,7 @@ def check_endpoints(): try: orion.checkPERSEO() logger.info("PERSEO endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("PERSEO endpoint not found: %s" % ex) # OpenLDAP: optional @@ -142,7 +142,7 @@ def check_endpoints(): try: openldap.checkLdap() logger.info("LDAP endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("LDAP endpoint not found: %s" % ex) # Mailer: optional @@ -152,7 +152,7 @@ def check_endpoints(): try: mailer.checkMailer() logger.info("MAILER endpoint OK") - except Exception, ex: + except Exception as ex: logger.warn("MAILER endpoint not found: %s" % ex) return "OK" From 03defdcea98b788ffa0af39a9be14434d4503589 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 11:47:01 +0200 Subject: [PATCH 04/33] update exception handling to python3 --- src/orchestrator/core/flow/Domains.py | 16 ++++---- src/orchestrator/core/flow/Groups.py | 10 ++--- src/orchestrator/core/flow/LdapGroupHelper.py | 10 ++--- src/orchestrator/core/flow/LdapUserHelper.py | 16 ++++---- src/orchestrator/core/flow/Projects.py | 16 ++++---- src/orchestrator/core/flow/Roles.py | 38 +++++++++---------- src/orchestrator/core/flow/Users.py | 4 +- src/orchestrator/core/flow/base.py | 8 ++-- .../core/flow/createNewService.py | 16 ++++---- .../core/flow/createNewServiceRole.py | 2 +- .../core/flow/createNewServiceUser.py | 2 +- .../core/flow/createNewSubService.py | 4 +- .../core/flow/createTrustToken.py | 4 +- src/orchestrator/core/flow/removeUser.py | 2 +- src/orchestrator/core/flow/updateUser.py | 4 +- 15 files changed, 76 insertions(+), 76 deletions(-) diff --git a/src/orchestrator/core/flow/Domains.py b/src/orchestrator/core/flow/Domains.py index b298d3c0..471e2525 100644 --- a/src/orchestrator/core/flow/Domains.py +++ b/src/orchestrator/core/flow/Domains.py @@ -72,7 +72,7 @@ def domains(self, self.logger.debug("DOMAINS=%s" % json.dumps(DOMAINS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -140,7 +140,7 @@ def get_domain(self, self.logger.debug("DOMAIN=%s" % json.dumps(DOMAIN, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -213,7 +213,7 @@ def update_domain(self, self.logger.debug("DOMAIN=%s" % json.dumps(DOMAIN, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -345,7 +345,7 @@ def delete_domain(self, # try: self.idm.removeRoles(ADMIN_TOKEN, DOMAIN_ID) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("Deleting roles in a slow way") roles = self.idm.getDomainRoles(ADMIN_TOKEN, DOMAIN_ID) @@ -487,7 +487,7 @@ def getDomainRolePolicies(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -596,7 +596,7 @@ def activate_module(self, subscriptionid = cb_res['subscriptionId'] self.logger.debug("subscription id=%s" % subscriptionid) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -690,7 +690,7 @@ def deactivate_module(self, sub['id']) break - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -771,7 +771,7 @@ def list_activated_modules(self, modules = self.cb.extract_modules_from_subscriptions(self, IOTMODULES, cb_res) self.logger.debug("modules=%s" % json.dumps(modules, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Groups.py b/src/orchestrator/core/flow/Groups.py index 8d77c693..03a69088 100644 --- a/src/orchestrator/core/flow/Groups.py +++ b/src/orchestrator/core/flow/Groups.py @@ -94,7 +94,7 @@ def groups(self, self.logger.debug("SERVICE_GROUPS=%s" % json.dumps(SERVICE_GROUPS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -162,7 +162,7 @@ def group(self, GROUP_ID) self.logger.debug("DETAIL_GROUP=%s" % json.dumps(DETAIL_GROUP, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -254,7 +254,7 @@ def updateGroup(self, GROUP_ID, GROUP_DATA_VALUE) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -343,7 +343,7 @@ def removeGroup(self, self.idm.removeGroup(SERVICE_ADMIN_TOKEN, GROUP_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -428,7 +428,7 @@ def createNewServiceGroup(self, NEW_SERVICE_GROUP_DESCRIPTION) self.logger.debug("ID of group %s: %s" % (NEW_SERVICE_GROUP_NAME, ID_GROUP)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/LdapGroupHelper.py b/src/orchestrator/core/flow/LdapGroupHelper.py index 83e3b5b0..eb7f32a7 100644 --- a/src/orchestrator/core/flow/LdapGroupHelper.py +++ b/src/orchestrator/core/flow/LdapGroupHelper.py @@ -62,7 +62,7 @@ def createNewGroup(self, return {} else: raise Exception(400, "None group was created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating group %s: %s" % ( NEW_GROUP_NAME, ex)) @@ -93,7 +93,7 @@ def deleteGroup(self, return {} else: raise Exception(400, "None group deleted in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR deleting group %s: %s" % ( GROUP_NAME, ex)) @@ -123,7 +123,7 @@ def listGroups(self, return res else: raise Exception(404, "None groups were retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving groups %s: %s" % ( FILTER, ex)) @@ -155,7 +155,7 @@ def getGroupDetailByAdmin(self, return group else: raise Exception(400, "None group detail was retrieved from ldap: group %s" % (group)) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving group detail %s: %s" % ( GROUP_NAME, ex)) @@ -190,7 +190,7 @@ def updateGroup(self, return res else: raise Exception(400, "None group was updated in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR updating group %s: %s" % ( GROUP_NAME, ex)) diff --git a/src/orchestrator/core/flow/LdapUserHelper.py b/src/orchestrator/core/flow/LdapUserHelper.py index 7b76dcd9..d9019da7 100644 --- a/src/orchestrator/core/flow/LdapUserHelper.py +++ b/src/orchestrator/core/flow/LdapUserHelper.py @@ -85,7 +85,7 @@ def createNewUser(self, return {} else: raise Exception(400, "None user was created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_USER_NAME, ex)) @@ -117,7 +117,7 @@ def askForCreateNewUser(self, return res else: raise Exception(400, "None user was asked to be created in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR asking for create user %s: %s" % ( NEW_USER_NAME, ex)) @@ -154,7 +154,7 @@ def deleteUser(self, return {} else: raise Exception(400, "None user deleted in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR deleting user %s: %s" % ( USER_NAME, ex)) @@ -184,7 +184,7 @@ def listUsers(self, return res else: raise Exception(404, "None users were retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving users %s: %s" % ( FILTER, ex)) @@ -211,7 +211,7 @@ def getUserDetail(self, return res else: raise Exception(400, "None user detail was retrieved from ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving user detail %s: %s" % ( USER_NAME, ex)) @@ -252,7 +252,7 @@ def getUserDetailByAdmin(self, return user else: raise Exception(400, "None user detail was retrieved from ldap: user %s groups %s" % (user, groups)) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR retrieving user detail %s: %s" % ( USER_NAME, ex)) @@ -279,7 +279,7 @@ def authUser(self, return res else: raise Exception(401, "None user was auth by ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR authenticating user %s: %s" % ( USER_NAME, ex)) @@ -330,7 +330,7 @@ def updateUser(self, return res else: raise Exception(400, "None user was updated in ldap: %s" % res['error']) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR updating user %s: %s" % ( USER_NAME, ex)) diff --git a/src/orchestrator/core/flow/Projects.py b/src/orchestrator/core/flow/Projects.py index 43105945..6ed4c508 100644 --- a/src/orchestrator/core/flow/Projects.py +++ b/src/orchestrator/core/flow/Projects.py @@ -88,7 +88,7 @@ def projects(self, self.logger.debug("PROJECTS=%s" % json.dumps(PROJECTS, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -164,7 +164,7 @@ def get_project(self, try: PROJECT = self.idm.getProject(ADMIN_TOKEN, PROJECT_ID) - except Exception, ex: + except Exception as ex: PROJECT = { 'project': { 'description': PROJECT_NAME, @@ -177,7 +177,7 @@ def get_project(self, PROJECT['project']['name']=PROJECT_NAME self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -269,7 +269,7 @@ def update_project(self, self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -384,7 +384,7 @@ def delete_project(self, self.logger.debug("PROJECT=%s" % PROJECT) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -520,7 +520,7 @@ def activate_module(self, subscriptionid = cb_res['subscriptionId'] self.logger.debug("subscription id=%s" % subscriptionid) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -638,7 +638,7 @@ def deactivate_module(self, sub['id']) break - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -741,7 +741,7 @@ def list_activated_modules(self, modules = self.cb.extract_modules_from_subscriptions(self, IOTMODULES, cb_res) self.logger.debug("modules=%s" % json.dumps(modules, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Roles.py b/src/orchestrator/core/flow/Roles.py index 0c3d0c47..022a5669 100644 --- a/src/orchestrator/core/flow/Roles.py +++ b/src/orchestrator/core/flow/Roles.py @@ -102,7 +102,7 @@ def roles(self, self.logger.debug("ROLES=%s" % json.dumps(ROLES, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -292,7 +292,7 @@ def roles_assignments(self, self.logger.debug("ROLES=%s" % json.dumps(role_assignments_expanded, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -424,7 +424,7 @@ def assignRoleServiceUser(self, SERVICE_ID, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -561,7 +561,7 @@ def assignRoleSubServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -673,7 +673,7 @@ def assignInheritRoleServiceUser(self, SERVICE_USER_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -786,7 +786,7 @@ def revokeRoleServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -915,7 +915,7 @@ def revokeRoleSubServiceUser(self, SERVICE_USER_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1026,7 +1026,7 @@ def revokeInheritRoleServiceUser(self, SERVICE_ID, SERVICE_USER_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1219,7 +1219,7 @@ def roles_assignments_groups(self, self.logger.debug("ROLES=%s" % json.dumps(role_assignments_expanded, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1352,7 +1352,7 @@ def assignRoleServiceGroup(self, SERVICE_ID, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1490,7 +1490,7 @@ def assignRoleSubServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1603,7 +1603,7 @@ def assignInheritRoleServiceGroup(self, SERVICE_GROUP_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1717,7 +1717,7 @@ def revokeRoleServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1847,7 +1847,7 @@ def revokeRoleSubServiceGroup(self, SERVICE_GROUP_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -1958,7 +1958,7 @@ def revokeInheritRoleServiceGroup(self, SERVICE_ID, SERVICE_GROUP_ID, INHERIT_ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2047,7 +2047,7 @@ def removeRole(self, SERVICE_ID, ROLE_ID) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2161,7 +2161,7 @@ def setPolicyRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2273,7 +2273,7 @@ def removePolicyFromRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -2384,7 +2384,7 @@ def getPolicyFromRole(self, else: raise Exception("not admin role found to perform this action") - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/Users.py b/src/orchestrator/core/flow/Users.py index c17861fc..948e606f 100644 --- a/src/orchestrator/core/flow/Users.py +++ b/src/orchestrator/core/flow/Users.py @@ -107,7 +107,7 @@ def users(self, # Listar los usuarios de un Subservicio - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -175,7 +175,7 @@ def user(self, USER_ID) self.logger.debug("DETAIL_USER=%s" % json.dumps(DETAIL_USER, indent=3)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 562bd980..031653e2 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -218,7 +218,7 @@ def ensure_service_name(self, USER_TOKEN, SERVICE_ID, SERVICE_NAME): SERVICE_NAME = self.idm.getDomainNameFromToken( USER_TOKEN, SERVICE_ID) - except Exception, ex: + except Exception as ex: # This op could be executed by cloud_admin user SERVICE = self.idm.getDomain(USER_TOKEN, SERVICE_ID) @@ -235,7 +235,7 @@ def ensure_subservice_name(self, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID) - except Exception, ex: + except Exception as ex: # This op could be executed by cloud_admin user SUBSERVICE = self.idm.getProject(USER_TOKEN, SUBSERVICE_ID) @@ -263,7 +263,7 @@ def get_extended_token(self, USER_TOKEN): token_extended['project'] = \ token_detail['token']['project']['name'][1:] - except Exception, ex: + except Exception as ex: # Probably expired? token_extended = { "token": USER_TOKEN, @@ -283,7 +283,7 @@ def collectComponentMetrics(self): all.append(self.perseo.PerseoRestOperations.getOutgoingMetrics()) # TODO: Take care of the following operation takes too much time self.sum = reduce(lambda x, y: dict((k, v + y[k]) for k, v in x.iteritems()), all) - except Exception, ex: + except Exception as ex: self.logger.error("ERROR collecting component metrics %s", ex) def getFlowMetrics(self): diff --git a/src/orchestrator/core/flow/createNewService.py b/src/orchestrator/core/flow/createNewService.py index 57008579..9c5c915d 100644 --- a/src/orchestrator/core/flow/createNewService.py +++ b/src/orchestrator/core/flow/createNewService.py @@ -113,7 +113,7 @@ def createNewService(self, NEW_SERVICE_ADMIN_PASSWORD, NEW_SERVICE_ADMIN_EMAIL, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_SERVICE_ADMIN_USER, ex)) @@ -121,7 +121,7 @@ def createNewService(self, try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) return self.composeErrorCode(ex) @@ -160,7 +160,7 @@ def createNewService(self, ID_NEW_SERVICE_ROLE_SERVICECUSTOMER] = self.idm.createDomainRoles(NEW_SERVICE_ADMIN_TOKEN, ROLES, ID_DOM1) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("creating roles in a slow way") ID_NEW_SERVICE_ROLE_SUBSERVICEADMIN = self.idm.createDomainRole( @@ -210,7 +210,7 @@ def createNewService(self, ID_NEW_SERVICE_ROLE_ADMIN_T] = self.idm.createDomainRoles(NEW_SERVICE_ADMIN_TOKEN, ROLES, ID_DOM1) - except Exception, ex: + except Exception as ex: # 404 if old version of keystone-scim self.logger.info("creating roles in a slow way") ID_NEW_SERVICE_ROLE_SUBSERVICEADMIN_T = self.idm.createDomainRole( @@ -383,14 +383,14 @@ def createNewService(self, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating groups %s" % ( ex)) self.logger.info("removing uncomplete created domain %s" % ID_DOM1) try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) return self.composeErrorCode(ex) @@ -442,13 +442,13 @@ def createNewService(self, self.mongodb.createSTHIndexes(NEW_SERVICE_NAME, '') - except Exception, ex: + except Exception as ex: if ID_DOM1: self.logger.info("removing uncomplete created domain %s" % ID_DOM1) try: self.idm.disableDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) self.idm.deleteDomain(DOMAIN_ADMIN_TOKEN, ID_DOM1) - except Exception, ex: + except Exception as ex: self.logger.warn("%s trying to remove uncomplete created domain %s" % (ex, ID_DOM1)) error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) diff --git a/src/orchestrator/core/flow/createNewServiceRole.py b/src/orchestrator/core/flow/createNewServiceRole.py index 193969cd..a5d20177 100644 --- a/src/orchestrator/core/flow/createNewServiceRole.py +++ b/src/orchestrator/core/flow/createNewServiceRole.py @@ -127,7 +127,7 @@ def createNewServiceRole(self, ID_ROLE, POLICY_FILE_NAME='policy-keypass-customer2.xml') - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/createNewServiceUser.py b/src/orchestrator/core/flow/createNewServiceUser.py index ca202f71..ca4d5697 100644 --- a/src/orchestrator/core/flow/createNewServiceUser.py +++ b/src/orchestrator/core/flow/createNewServiceUser.py @@ -107,7 +107,7 @@ def createNewServiceUser(self, NEW_USER_DESCRIPTION) self.logger.debug("ID of user %s: %s" % (NEW_USER_NAME, ID_USER)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/createNewSubService.py b/src/orchestrator/core/flow/createNewSubService.py index 01f33807..f1ee3aa8 100644 --- a/src/orchestrator/core/flow/createNewSubService.py +++ b/src/orchestrator/core/flow/createNewSubService.py @@ -125,7 +125,7 @@ def createNewSubService(self, NEW_SUBSERVICE_ADMIN_PASSWORD, NEW_SUBSERVICE_ADMIN_EMAIL, None) - except Exception, ex: + except Exception as ex: self.logger.warn("ERROR creating user %s: %s" % ( NEW_SUBSERVICE_ADMIN_USER, ex)) @@ -151,7 +151,7 @@ def createNewSubService(self, self.mongodb.createSTHIndexes(SERVICE_NAME, NEW_SUBSERVICE_NAME) - except Exception, ex: + except Exception as ex: if ID_PRO1: self.logger.info("removing uncomplete created project %s" % ID_PRO1) self.idm.disableProject(SERVICE_ADMIN_TOKEN, SERVICE_ID, ID_PRO1) diff --git a/src/orchestrator/core/flow/createTrustToken.py b/src/orchestrator/core/flow/createTrustToken.py index f8e5a6b5..5da361ac 100644 --- a/src/orchestrator/core/flow/createTrustToken.py +++ b/src/orchestrator/core/flow/createTrustToken.py @@ -178,7 +178,7 @@ def createTrustToken(self, self.logger.debug("ID of Trust %s" % (ID_TRUST)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -275,7 +275,7 @@ def getTrustsUserTrustee(self, self.logger.debug("Trusts %s" % (TRUSTS)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/removeUser.py b/src/orchestrator/core/flow/removeUser.py index 342e49a0..ac95b67c 100644 --- a/src/orchestrator/core/flow/removeUser.py +++ b/src/orchestrator/core/flow/removeUser.py @@ -104,7 +104,7 @@ def removeUser(self, USER_ID) # self.logger.debug("ID of user %s: %s" % (USER_NAME, ID_USER)) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code diff --git a/src/orchestrator/core/flow/updateUser.py b/src/orchestrator/core/flow/updateUser.py index 4bcabdb1..8acb964f 100644 --- a/src/orchestrator/core/flow/updateUser.py +++ b/src/orchestrator/core/flow/updateUser.py @@ -105,7 +105,7 @@ def updateUser(self, USER_ID, USER_DATA_VALUE) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code @@ -197,7 +197,7 @@ def changeUserPassword(self, SERVICE_USER_PASSWORD, NEW_USER_PASSWORD) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code From 5499bfc430bb66ea32d905c2e7c4d2d2970ca9ef Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 12:31:57 +0200 Subject: [PATCH 05/33] update import --- src/settings/dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings/dev.py b/src/settings/dev.py index 2b7d00ed..bcce2126 100644 --- a/src/settings/dev.py +++ b/src/settings/dev.py @@ -8,7 +8,7 @@ stipulated in the agreement/contract under which the program(s) have been supplied. """ -from common import * # noqa +from settings.common import * # noqa DEBUG = False TEMPLATE_DEBUG = DEBUG From 7b91579eeccbabbcda2ba7d6fac6b25a5433028b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 12:33:50 +0200 Subject: [PATCH 06/33] update CNR --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 32214f78..6e5a425f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ CHANGES ======= +Upgrade Python (2 -> 3), + Django (1.11 -> 2.2.24) + DjangoRestFramework (3.9 -> 3.11) UPDATE: Update jsonschema dep 2.5.0 to 2.5.1 Upgrade STH xacml policies with orion frns From 785dda174227096bf69de2b16fa0f9774204a78d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 13:02:01 +0200 Subject: [PATCH 07/33] fix missed python2 exception --- src/orchestrator/core/flow/Domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/core/flow/Domains.py b/src/orchestrator/core/flow/Domains.py index 471e2525..b8f47977 100644 --- a/src/orchestrator/core/flow/Domains.py +++ b/src/orchestrator/core/flow/Domains.py @@ -368,7 +368,7 @@ def delete_domain(self, self.logger.debug("DOMAIN=%s" % DOMAIN) - except Exception, ex: + except Exception as ex: error_code = self.composeErrorCode(ex) self.logError(self.logger, error_code, ex) return error_code From 3c5f5375a09b0647518b65798a92a8ecf1598efd Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 15 Jun 2021 16:57:43 +0200 Subject: [PATCH 08/33] composeError for python3 --- src/orchestrator/core/flow/base.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 031653e2..940163bd 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -40,8 +40,6 @@ from settings.dev import IOTMODULES from settings import dev as settings - - class FlowBase(object): def __init__(self, KEYSTONE_PROTOCOL, @@ -155,12 +153,17 @@ def composeErrorCode(self, ex): # print(exc_type, fname, exc_tb.tb_lineno) res = {"error": str(ex), "code": 500} if isinstance(ex.args, tuple) and ( - (len(ex.args) > 0) and - not isinstance(ex.args[0], tuple)): # Python 2.6 - res['code'] = ex.args[0] - if res['code'] == 400 and len(ex.args) > 1 and \ - ex.args[1].startswith('SPASSWORD'): - res['error'] = ex.args[1] + (len(ex.args) > 0)): + if not isinstance(ex.args[0], tuple): # Python 2.6 + res['code'] = ex.args[0] + if res['code'] == 400 and len(ex.args) > 1 and \ + ex.args[1].startswith('SPASSWORD'): + res['error'] = ex.args[1] + else: # Python 3 + res['code'] = ex.args[0][0] + if res['code'] == 400 and len(ex.args[0]) > 1 and \ + ex.args[0][1].startswith('SPASSWORD'): + res['error'] = ex.args[0][1] elif isinstance(ex.message, tuple): # Python 2.7 res['code'] = ex.message[0] if res['code'] == 400 and len(ex.message) > 1 and \ From ffa09f5b63bffd2afc1b47150ce464399669d1a5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 16 Jun 2021 10:33:00 +0200 Subject: [PATCH 09/33] encode data json --- src/orchestrator/common/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/common/util.py b/src/orchestrator/common/util.py index 2a81e21c..6ccfb6de 100644 --- a/src/orchestrator/common/util.py +++ b/src/orchestrator/common/util.py @@ -99,7 +99,7 @@ def rest_request(self, url, method, user=None, password=None, if data: if json_data: request = urllib.request.Request( - url, data=json.dumps(data)) + url, data=json.dumps(data).encode()) else: request = urllib.request.Request(url, data=data) else: From 03f7f2a579b7666a49d2b69ebfffc358ecce4da4 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 16 Jun 2021 15:37:54 +0200 Subject: [PATCH 10/33] update rest_request2 --- src/orchestrator/common/util.py | 30 +++++++++++++++++------------- src/orchestrator/core/flow/base.py | 2 +- src/orchestrator/core/keypass.py | 14 +++++++------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/orchestrator/common/util.py b/src/orchestrator/common/util.py index 6ccfb6de..8524a0e8 100644 --- a/src/orchestrator/common/util.py +++ b/src/orchestrator/common/util.py @@ -99,9 +99,9 @@ def rest_request(self, url, method, user=None, password=None, if data: if json_data: request = urllib.request.Request( - url, data=json.dumps(data).encode()) + url=url, data=json.dumps(data).encode()) else: - request = urllib.request.Request(url, data=data) + request = urllib.request.Request(url=url, body=data) else: request = urllib.request.Request(url) request.get_method = lambda: method @@ -165,7 +165,7 @@ def rest_request(self, url, method, user=None, password=None, except ValueError: res.msg = data except Exception as e: - print(e) + self.logger.warn("exception %s", e) except urllib.request.URLError as e: data = None res = e @@ -242,19 +242,23 @@ def rest_request2(self, url, method, user=None, password=None, try: if not auth: - res = requests.post(url, - headers=headers, - data=rdata, - verify=False) + res = requests.request(method=method, + url=url, + headers=headers, + data=rdata, + verify=False) else: - res = requests.post(url, - auth=auth, - headers=headers, - data=rdata, - verify=False) + res = requests.request(method=method, + url=url, + auth=auth, + headers=headers, + data=rdata, + verify=False) + res.code = res.status_code + res.msg = res.reason except Exception as e: - print(e) + self.logger.warn("exception %s", e) if settings.ORC_EXTENDED_METRICS: self.collectOutgoingMetrics(service_start, rdata, headers, res) diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 940163bd..4f1a9c78 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -70,7 +70,7 @@ def __init__(self, FROM=None): # Generate Transaction ID - self.TRANSACTION_ID = uuid.uuid4() + self.TRANSACTION_ID = str(uuid.uuid4()) if not CORRELATOR_ID: self.CORRELATOR_ID = self.TRANSACTION_ID diff --git a/src/orchestrator/core/keypass.py b/src/orchestrator/core/keypass.py index 717a85ae..6720ca54 100644 --- a/src/orchestrator/core/keypass.py +++ b/src/orchestrator/core/keypass.py @@ -55,7 +55,7 @@ def __init__(self, self.policy_dir = os.path.dirname(policies.__file__) def checkAccC(self): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/', method='GET', data=None) @@ -82,7 +82,7 @@ def provisionPolicyByContent(self, SERVICE_ROLE_ID, POLICY_CONTENT): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='POST', json_data=False, @@ -97,7 +97,7 @@ def deleteTenantPolicies(self, SERVICE_NAME, SERVICE_ADMIN_TOKEN): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1', method='DELETE', json_data=False, @@ -111,7 +111,7 @@ def getRolePolicies(self, SERVICE_ADMIN_TOKEN, SERVICE_ROLE_ID): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='GET', json_data=False, @@ -129,7 +129,7 @@ def getRolePolicy(self, SERVICE_ROLE_ID, POLICY_NAME): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+ SERVICE_ROLE_ID + '/policy/' + POLICY_NAME, method='GET', json_data=False, @@ -146,7 +146,7 @@ def deleteRolePolicies(self, SERVICE_ADMIN_TOKEN, SERVICE_ROLE_ID): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+SERVICE_ROLE_ID, method='DELETE', json_data=False, @@ -161,7 +161,7 @@ def deleteRolePolicy(self, SERVICE_ROLE_ID, POLICY_NAME): - res = self.AccessControlRestOperations.rest_request( + res = self.AccessControlRestOperations.rest_request2( url='/pap/v1/subject/'+ SERVICE_ROLE_ID + '/policy/' + POLICY_NAME, method='DELETE', json_data=False, From cb399c8f64ee58e276a2d3bed87cf4e7028cf739 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 16 Jun 2021 16:00:40 +0200 Subject: [PATCH 11/33] Update ChangeLog --- ChangeLog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6e5a425f..8646e84d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,8 @@ CHANGES ======= -Upgrade Python (2 -> 3), - Django (1.11 -> 2.2.24) - DjangoRestFramework (3.9 -> 3.11) +Upgrade Python (2 -> 3), #242 + Django (1.11 -> 2.2.24) #310 + DjangoRestFramework (3.9 -> 3.11) #309 UPDATE: Update jsonschema dep 2.5.0 to 2.5.1 Upgrade STH xacml policies with orion frns From b9a2b5ad958d7c9cae24ef162fa8f1ddb8901289 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 16 Jun 2021 16:57:07 +0200 Subject: [PATCH 12/33] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index adf200ce..1e3a810e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ Django==2.2.24 djangorestframework==3.11.2 jsonschema==2.6.0 uwsgi==2.0.19.1 -requests==2.22.0 +requests==2.25.1 python-ldap==3.3.1 pymongo==3.7.2 From 4a96eb1e898c797736d540851980c6712d4f1458 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 17 Jun 2021 10:58:30 +0200 Subject: [PATCH 13/33] Update ChangeLog --- ChangeLog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 8646e84d..dbb0c8c0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,7 +3,9 @@ CHANGES Upgrade Python (2 -> 3), #242 Django (1.11 -> 2.2.24) #310 DjangoRestFramework (3.9 -> 3.11) #309 -UPDATE: Update jsonschema dep 2.5.0 to 2.5.1 + requests (2.22.0 -> 2.25.1) + python-ldap (2.5.2 > 3.3.1) +UPDATE: Update jsonschema dep 2.5.0 to 2.6.0 Upgrade STH xacml policies with orion frns 3.8.0 From 9bbee220b2e07dc0335cb3d877f291d828beb8b5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 17 Jun 2021 16:10:23 +0200 Subject: [PATCH 14/33] Update base.py --- src/orchestrator/core/flow/base.py | 40 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 4f1a9c78..f4749d85 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -152,31 +152,33 @@ def composeErrorCode(self, ex): # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] # print(exc_type, fname, exc_tb.tb_lineno) res = {"error": str(ex), "code": 500} - if isinstance(ex.args, tuple) and ( - (len(ex.args) > 0)): - if not isinstance(ex.args[0], tuple): # Python 2.6 - res['code'] = ex.args[0] - if res['code'] == 400 and len(ex.args) > 1 and \ - ex.args[1].startswith('SPASSWORD'): - res['error'] = ex.args[1] - else: # Python 3 - res['code'] = ex.args[0][0] - if res['code'] == 400 and len(ex.args[0]) > 1 and \ - ex.args[0][1].startswith('SPASSWORD'): - res['error'] = ex.args[0][1] - elif isinstance(ex.message, tuple): # Python 2.7 - res['code'] = ex.message[0] - if res['code'] == 400 and len(ex.message) > 1 and \ - ex.message[1].startswith('SPASSWORD'): - res['error'] = ex.message[1] - return res, None, None + try: + if isinstance(ex.args, tuple) and ( + (len(ex.args) > 0)): + if not isinstance(ex.args[0], tuple): # Python 2.6 + res['code'] = ex.args[0] + if res['code'] == 400 and len(ex.args) > 1 and \ + ex.args[1].startswith('SPASSWORD'): + res['error'] = ex.args[1] + else: # Python 3 + res['code'] = ex.args[0][0] + if res['code'] == 400 and len(ex.args[0]) > 1 and \ + ex.args[0][1].startswith('SPASSWORD'): + res['error'] = ex.args[0][1] + elif isinstance(ex.message, tuple): # Python 2.7 + res['code'] = ex.message[0] + if res['code'] == 400 and len(ex.message) > 1 and \ + ex.message[1].startswith('SPASSWORD'): + res['error'] = ex.message[1] + finally: + return res, None, None def logError(self, logger, error_code, ex): ''' Log as error level error_code if is < 400 or > 500 ''' - if (error_code[0]['code'] < 400 or error_code[0]['code'] > 500): + if (error_code[0]['code'] < 400 or error_code[0]['code'] >= 500): logger.error(ex) else: logger.debug(ex) From 22c4d6475ddde8736df3429acce1a72848a20740 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 18 Jun 2021 12:31:39 +0200 Subject: [PATCH 15/33] replace python by python3 --- DOCKER.md | 2 +- INSTALL.md | 6 +++--- README.md | 3 +-- TESTS.md | 8 ++++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 8c9496c6..5af7546b 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -104,7 +104,7 @@ Additionally, the following environment variables are available for orchestrator | PROCESSES | | 6 | | THREADS | | 8 | | HARAKIRI | | 80 | -| HTTP_TIMEOUT | | 200 | +| HTTP_TIMEOUT | | 200 | | MAX_REQUESTS | | 250 | | UWSGI_BUFFER_SIZE | | 4096 | | KEYSTONE_HOST | KEYSTONE.host | localhost | diff --git a/INSTALL.md b/INSTALL.md index 3759676e..f8cb9204 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -12,7 +12,7 @@ Due to its stateless behavior, iotp-orchestrator does not require any backup pro * Internet access * Connectivity to http://github.com * [Git](http://git-scm.com/) - * Python 2.7 (or upper) and pip + * Python 3 and pip3 * C and C++ compilation platform: gcc, g++, make and python headers * UNIX platforms but not mandatory * WSGI Web server (Apache2/Nginx) but not mandatory @@ -42,13 +42,13 @@ cd src Then install all dependencies by running: ``` -pip install -r requirements.txt +pip3 install -r requirements.txt ``` Start server in 8084 port using django web server: ``` -python manage.py runserver 8084 --settings=settings.dev +python3 manage.py runserver 8084 --settings=settings.dev ``` or using another web server like uWSGI diff --git a/README.md b/README.md index 7e4ee0b0..3983a22e 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,8 @@ Orchestrator is used to: - Create, List, Modify LDAP Groups Orchestrator is based mainly on: -- Python 2.7 needed +- Python 3 needed - Django / DjangoRestFramework -- httplib Orchestrator relies on these other IoT parts: - Identity Manager: Keystone (mandatory) diff --git a/TESTS.md b/TESTS.md index 4b0904fa..1b00b3f3 100644 --- a/TESTS.md +++ b/TESTS.md @@ -4,7 +4,7 @@ - [Unit Tests](./src/tests/core/test_idm.py) Run unit tests with: ``` - $ python ./test_idm.py + $ python3 ./test_idm.py ``` @@ -12,21 +12,21 @@ Run unit tests with: Run e2e tests with: ``` - $ python ./test_api.py + $ python3 ./test_api.py ``` - [End2End LDAP Tests](./src/tests/api/test_ldap_api.py) Run e2e LDAP tests with: ``` - $ python ./test_ldap_api.py + $ python3 ./test_ldap_api.py ``` - [Comamnd Tests](./src/tests/api/test_commands.py) Run scripts tests with: ``` - $ python ./test_commands.py + $ python3 ./test_commands.py ``` There is a config file to run Orchestrator build into Travis CI service: From c536a07c9d9b024a5917829d5bfb07c46f2ee70c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 18 Jun 2021 13:47:01 +0200 Subject: [PATCH 16/33] fix parse error --- src/orchestrator/api/ldap_view.py | 27 +++----- src/orchestrator/api/parsers.py | 4 +- src/orchestrator/api/views.py | 109 ++++++++++-------------------- 3 files changed, 48 insertions(+), 92 deletions(-) diff --git a/src/orchestrator/api/ldap_view.py b/src/orchestrator/api/ldap_view.py index ead9b6ca..2cc09bcb 100644 --- a/src/orchestrator/api/ldap_view.py +++ b/src/orchestrator/api/ldap_view.py @@ -98,8 +98,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -151,8 +150,7 @@ def get(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -188,8 +186,7 @@ def put(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -225,8 +222,7 @@ def delete(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -272,8 +268,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -330,8 +325,7 @@ def post(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -378,8 +372,7 @@ def get(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -415,8 +408,7 @@ def put(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -451,8 +443,7 @@ def delete(self, request): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) diff --git a/src/orchestrator/api/parsers.py b/src/orchestrator/api/parsers.py index ad1a803f..6014aa00 100644 --- a/src/orchestrator/api/parsers.py +++ b/src/orchestrator/api/parsers.py @@ -49,7 +49,7 @@ def parse(self, stream, media_type=None, parser_context=None): data, schemas.json[parser_context['view'].schema_name]) except (ValueError, jsonschema.exceptions.ValidationError) as error: - logger.debug(error.message) - raise ParseError(detail=error.message) + logger.debug(str(error.args[0])) + raise ParseError(detail=str(error.args[0])) else: return data diff --git a/src/orchestrator/api/views.py b/src/orchestrator/api/views.py index a6ae054e..cab638b9 100644 --- a/src/orchestrator/api/views.py +++ b/src/orchestrator/api/views.py @@ -128,8 +128,7 @@ def get(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -171,8 +170,7 @@ def put(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -223,8 +221,7 @@ def delete(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -287,8 +284,7 @@ def post(self, request, *args, **kw): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -359,8 +355,7 @@ def get(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -411,8 +406,7 @@ def put(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -469,8 +463,7 @@ def delete(self, request, service_id=None, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -536,8 +529,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -589,8 +581,7 @@ def delete(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -632,7 +623,7 @@ def put(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0}'.format(error.message), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -673,8 +664,7 @@ def get(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -717,8 +707,7 @@ def post(self, request, service_id, user_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -775,8 +764,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -820,8 +808,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -874,8 +861,7 @@ def delete(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -917,7 +903,7 @@ def put(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0}'.format(error.message), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -958,8 +944,7 @@ def get(self, request, service_id, group_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1016,8 +1001,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1059,8 +1043,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1119,8 +1102,7 @@ def get(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1171,8 +1153,7 @@ def post(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1218,8 +1199,7 @@ def delete(self, request, service_id, role_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1279,8 +1259,7 @@ def get(self, request, service_id, role_id, policy_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1327,8 +1306,7 @@ def delete(self, request, service_id, role_id, policy_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1383,8 +1361,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1429,8 +1406,7 @@ def get(self, request, service_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1491,8 +1467,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1570,8 +1545,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1650,8 +1624,7 @@ def delete(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1712,8 +1685,7 @@ def get(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1791,8 +1763,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1871,8 +1842,7 @@ def delete(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -1933,8 +1903,7 @@ def post(self, request, service_id): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2025,8 +1994,7 @@ def get(self, request, service_id, subservice_id=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2106,8 +2074,7 @@ def post(self, request, service_id, subservice_id=None, iot_module=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2187,8 +2154,7 @@ def delete(self, request, service_id, subservice_id=None, iot_module=None): except ParseError as error: Stats.num_api_errors += 1 response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST, headers={"Fiware-Correlator": CORRELATOR_ID} ) @@ -2281,8 +2247,7 @@ def get(self, request): except ParseError as error: response = Response( - 'Input validation error - {0} {1}'.format(error.message, - error.detail), + 'Input validation error - {0}'.format(error.detail), status=status.HTTP_400_BAD_REQUEST ) self.collectMetrics(service_start, service_name, subservice_name, request, response, flow) From b1558fa649f1b46e9efbb70018f92413dee70fd2 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 10:17:41 +0200 Subject: [PATCH 17/33] check error_code type --- src/orchestrator/core/flow/base.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index f4749d85..afeac530 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -178,12 +178,15 @@ def logError(self, logger, error_code, ex): ''' Log as error level error_code if is < 400 or > 500 ''' - if (error_code[0]['code'] < 400 or error_code[0]['code'] >= 500): - logger.error(ex) - else: + try: + if (isinstance(error_code[0]['code'], int) and + (error_code[0]['code'] < 400 or error_code[0]['code'] >= 500)): + logger.error(ex) + else: + logger.debug(ex) + except: logger.debug(ex) - def get_endpoint_iot_module(self, iot_module): assert iot_module in IOTMODULES if iot_module in self.endpoints: From 726b10d796f7fa36d38f8455a6dd121980cfaa52 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 12:19:36 +0200 Subject: [PATCH 18/33] decode result --- src/orchestrator/core/keypass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/orchestrator/core/keypass.py b/src/orchestrator/core/keypass.py index 6720ca54..1480dbd9 100644 --- a/src/orchestrator/core/keypass.py +++ b/src/orchestrator/core/keypass.py @@ -119,7 +119,7 @@ def getRolePolicies(self, fiware_service=SERVICE_NAME) assert res.code == 200, (res.code, res.msg) - body_data = res.read() + body_data = res.content.decode() logger.debug("data response: %s" % body_data) return body_data @@ -137,7 +137,7 @@ def getRolePolicy(self, fiware_service=SERVICE_NAME) assert res.code == 200, (res.code, res.msg) - body_data = res.read() + body_data = res.content.decode() logger.debug("data response: %s" % body_data) return body_data From c068f4b7a8a622180d975063abdbac281198442b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 13:06:35 +0200 Subject: [PATCH 19/33] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1e3a810e..b5caae23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ djangorestframework==3.11.2 jsonschema==2.6.0 uwsgi==2.0.19.1 requests==2.25.1 -python-ldap==3.3.1 +python-ldap==3.0.0 pymongo==3.7.2 From a9d9cb844caa0e2e99cbe61947c66c550f6b8c33 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 13:06:53 +0200 Subject: [PATCH 20/33] Update ChangeLog --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index dbb0c8c0..4476e4d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ Upgrade Python (2 -> 3), #242 Django (1.11 -> 2.2.24) #310 DjangoRestFramework (3.9 -> 3.11) #309 requests (2.22.0 -> 2.25.1) - python-ldap (2.5.2 > 3.3.1) + python-ldap (2.5.2 > 3.0.0) UPDATE: Update jsonschema dep 2.5.0 to 2.6.0 Upgrade STH xacml policies with orion frns From d518690277545915f50f664f751aad0c5ce03791 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 17:13:09 +0200 Subject: [PATCH 21/33] update to openldap3 --- src/orchestrator/core/openldap.py | 53 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index 2a822b18..c562f5a0 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -47,12 +47,12 @@ def __init__(self, self.LDAP_BASEDN = LDAP_BASEDN def checkLdap(self): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+self.LDAP_HOST+':'+self.LDAP_PORT) # Just for check connection conn.simple_bind_s("","") def bindAdmin(self, USERNAME, PASSWORD): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) conn.protocol_version = ldap.VERSION3 username = "cn=" + USERNAME + "," + self.LDAP_BASEDN logger.debug("bind admin %s" % username) @@ -60,7 +60,7 @@ def bindAdmin(self, USERNAME, PASSWORD): return conn def bindUser(self, USERNAME, PASSWORD): - conn = ldap.open(self.LDAP_HOST, self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) conn.protocol_version = ldap.VERSION3 username = "uid=" + USERNAME + ", ou=users," + self.LDAP_BASEDN logger.debug("bind user %s" % username) @@ -82,17 +82,20 @@ def createUser(self, conn = self.bindAdmin(LDAP_ADMIN_USER, LDAP_ADMIN_PASSWORD) dn = "uid=" + NEW_USER_NAME + ",ou=users," + self.LDAP_BASEDN mymodlist = { - "objectClass": ["top", "posixAccount", "shadowAccount", - "organizationalPerson", "inetOrgPerson"], - "uid": [ str(NEW_USER_NAME) ], - "cn": [ str(NEW_USER_NAME) ], - "uidNumber": ["5000"], - "gidNumber": ["10000"], - "loginShell": ["/bin/bash"], - "homeDirectory": ["/home/"+ str(NEW_USER_NAME)], - "mail": str(NEW_USER_EMAIL), - "sn": str(NEW_USER_NAME), - "userPassword": str(NEW_USER_PASSWORD) + "objectClass": [ b"top", + b"posixAccount", + b"shadowAccount", + b"organizationalPerson", + b"inetOrgPerson"], + "uid": [ NEW_USER_NAME.encode('utf-8') ], + "cn": [ NEW_USER_NAME.encode('utf-8') ], + "uidNumber": [b"5000"], + "gidNumber": [b"10000"], + "loginShell": [b"/bin/bash"], + "homeDirectory": [("/home/" + NEW_USER_NAME).encode('utf-8')], + "mail": NEW_USER_EMAIL.encode('utf-8'), + "sn": NEW_USER_NAME.encode('utf-8'), + "userPassword": NEW_USER_PASSWORD.encode('utf-8') } logger.debug("create user mymodlist: %s" % mymodlist) result = conn.add_s(dn, ldap.modlist.addModlist(mymodlist)) @@ -200,7 +203,7 @@ def assignGroupUser(self, old_value['member'] = oldgroupMembers new_value['member'] = groupMembers # Add new group member - new_value['member'].append('uid=' + str(USER_NAME) +',ou=users,' + self.LDAP_BASEDN) + new_value['member'].append(str('uid=' + str(USER_NAME) +',ou=users,' + self.LDAP_BASEDN).encode('utf-8')) mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) logger.debug("ldap assing group user %s" % json.dumps(result)) @@ -228,7 +231,7 @@ def getUserGroups(self, result_dn = result[0] result_attrs = result[1] if ('cn' in result_attrs and len(result_attrs['cn']) > 0): - groups.append(result_attrs['cn'][0]) + groups.append(result_attrs['cn'][0].decode('utf-8')) logger.debug("ldap groups of user: %s" % json.dumps(groups)) self.unbind(conn) @@ -256,9 +259,10 @@ def getUserDetail(self, else: if result_type == ldap.RES_SEARCH_ENTRY: result = result_data[0] - logger.debug("ldap get user detail %s" % json.dumps(result)) + details = {key:[v.decode('utf-8') for v in values] for key, values in result[1].items()} + logger.debug("ldap get user detail %s" % json.dumps(details)) self.unbind(conn) - return { "details": result } + return { "details": details } except ldap.LDAPError as e: logger.warn("getUserDetail exception: %s" % e) return { "error": e } @@ -284,9 +288,9 @@ def updateUserByAdmin(self, new_value[attr] = USER_DATA[userattr] mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update user by admin %s" % json.dumps(result)) + logger.debug("ldap update user by admin %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } + return { "details": result[0] } except ldap.LDAPError as e: logger.warn("updateUserByAdmin exception: %s" % e) return { "error": e } @@ -327,10 +331,11 @@ def createGroup(self, conn = self.bindAdmin(LDAP_ADMIN_USER, LDAP_ADMIN_PASSWORD) dn = "cn=" + NEW_GROUP_NAME + ",ou=groups," + self.LDAP_BASEDN mymodlist = { - "objectClass": ["top", "groupofnames"], - "cn": [ str(NEW_GROUP_NAME) ], - 'member' : [ 'ou=groups,dc=openstack,dc=org' ], - "description": str(NEW_GROUP_DESCRIPTION) + "objectClass": [b"top", + b"groupofnames"], + "cn": [ str(NEW_GROUP_NAME).encode('utf-8') ], + 'member' : [ b'ou=groups,dc=openstack,dc=org' ], + "description": str(NEW_GROUP_DESCRIPTION).encode('utf-8') } logger.debug("create group mymodlist: %s" % mymodlist) result = conn.add_s(dn, ldap.modlist.addModlist(mymodlist)) From d78a5e8255356225548e0402cce11325784200bc Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 21 Jun 2021 17:16:56 +0200 Subject: [PATCH 22/33] Update CNR --- ChangeLog | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4476e4d0..bcb436a5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ Upgrade Python (2 -> 3), #242 Django (1.11 -> 2.2.24) #310 DjangoRestFramework (3.9 -> 3.11) #309 requests (2.22.0 -> 2.25.1) - python-ldap (2.5.2 > 3.0.0) + python-ldap (2.5.2 > 3.3.1) UPDATE: Update jsonschema dep 2.5.0 to 2.6.0 Upgrade STH xacml policies with orion frns diff --git a/requirements.txt b/requirements.txt index b5caae23..1e3a810e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ djangorestframework==3.11.2 jsonschema==2.6.0 uwsgi==2.0.19.1 requests==2.25.1 -python-ldap==3.0.0 +python-ldap==3.3.1 pymongo==3.7.2 From ef0bed80e964277985e15a40bcebb93e8d7cc9eb Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 23 Jun 2021 12:23:19 +0200 Subject: [PATCH 23/33] fix update user by himself and group --- src/orchestrator/core/flow/LdapUserHelper.py | 2 +- src/orchestrator/core/openldap.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/orchestrator/core/flow/LdapUserHelper.py b/src/orchestrator/core/flow/LdapUserHelper.py index d9019da7..dcb56135 100644 --- a/src/orchestrator/core/flow/LdapUserHelper.py +++ b/src/orchestrator/core/flow/LdapUserHelper.py @@ -318,7 +318,7 @@ def updateUser(self, NEW_USER_NAME, GROUP_NAME) self.logger.debug("res=%s" % res) - elif USERNAME and USER_PASSWORD: + elif USER_NAME and USER_PASSWORD: res = self.ldap.updateUserByHimself( USER_NAME, USER_PASSWORD, diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index c562f5a0..87fc0e1e 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -315,9 +315,9 @@ def updateUserByUser(self, new_value[attr] = USER_DATA[userattr] mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update user by user %s" % json.dumps(result)) + logger.debug("ldap update user by user %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } + return { "details": result[0] } except ldap.LDAPError as e: logger.warn("updateUserByUser exception: %s" % e) return { "error": e } @@ -384,9 +384,9 @@ def updateGroupByAdmin(self, logger.debug("ldap update group old_value %s new_value %s " % (json.dumps(old_value), json.dumps(new_value))) mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) - logger.debug("ldap update group by admin %s" % json.dumps(result)) + logger.debug("ldap update group by admin %s" % json.dumps(result[0])) self.unbind(conn) - return { "details": result } + return { "details": result[0] } except ldap.LDAPError as e: logger.warn("updateGroupByAdmin exception: %s" % e) return { "error": e } From 73346942258755b3660be4d97054ee014b4a64aa Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 23 Jun 2021 13:10:16 +0200 Subject: [PATCH 24/33] fix updateUserByHimself --- src/orchestrator/core/openldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index 87fc0e1e..9a092dfc 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -295,7 +295,7 @@ def updateUserByAdmin(self, logger.warn("updateUserByAdmin exception: %s" % e) return { "error": e } - def updateUserByUser(self, + def updateUserByHimself(self, USER_NAME, USER_PASSWORD, USER_DETAIL): From 24b7451f69e6299e0e28cbdf215b2ac117d0d4f8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 23 Jun 2021 14:53:31 +0200 Subject: [PATCH 25/33] fix typo USER_DETAIL -> USER_DATA --- src/orchestrator/core/openldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index 9a092dfc..0c806b47 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -298,7 +298,7 @@ def updateUserByAdmin(self, def updateUserByHimself(self, USER_NAME, USER_PASSWORD, - USER_DETAIL): + USER_DATA): try: conn = self.bindUser(USER_NAME, USER_PASSWORD) dn = "uid=" + USER_NAME + ",ou=users," + self.LDAP_BASEDN From ee4f23fdd852abc3d39a83c62a417552588a854d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 23 Jun 2021 15:40:27 +0200 Subject: [PATCH 26/33] fix constructor args --- src/orchestrator/api/ldap_view.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/orchestrator/api/ldap_view.py b/src/orchestrator/api/ldap_view.py index 2cc09bcb..0bb3b38a 100644 --- a/src/orchestrator/api/ldap_view.py +++ b/src/orchestrator/api/ldap_view.py @@ -54,6 +54,9 @@ def post(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -110,6 +113,9 @@ def get(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -162,7 +168,10 @@ def put(self, request): try: request.data # json validation flow = LdapUserHelper( - None, None, None, + None, None, None, + None, None, None, + None, None, None, + None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, LDAP_BASEDN=self.LDAP_BASEDN, @@ -199,6 +208,9 @@ def delete(self, request): try: request.data # json validation flow = LdapUserHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -246,7 +258,10 @@ def post(self, request): try: request.data # json validation flow = LdapUserHelper( - None, None, None, + None, None, None, + None, None, None, + None, None, None, + None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, LDAP_BASEDN=self.LDAP_BASEDN, @@ -293,6 +308,9 @@ def post(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -337,6 +355,9 @@ def get(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -385,6 +406,9 @@ def put(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, @@ -421,6 +445,9 @@ def delete(self, request): try: request.data # json validation flow = LdapGroupHelper( + None, None, None, + None, None, None, + None, None, None, None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, From 3fa5b90233f07390bb525728ffdd89f0f191d7d8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 23 Jun 2021 16:11:07 +0200 Subject: [PATCH 27/33] add tests group ldap groups --- src/orchestrator/core/openldap.py | 6 +- src/tests/api/test_ldap_api.py | 91 ++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index 0c806b47..64909933 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -372,7 +372,7 @@ def updateGroupByAdmin(self, old_value = {} new_value = {} results = conn.search_s(dn, ldap.SCOPE_BASE) - logger.debug("ldap update group search results %s" % json.dumps(results)) + logger.debug("ldap update group search results %s" % results) for result in results: result_dn = result[0] result_attrs = result[1] @@ -380,8 +380,8 @@ def updateGroupByAdmin(self, for userattr in ['description']: if attr == userattr: old_value[attr] = result_attrs[userattr] - new_value[attr] = [str(GROUP_DESCRIPTION)] - logger.debug("ldap update group old_value %s new_value %s " % (json.dumps(old_value), json.dumps(new_value))) + new_value[attr] = [str(GROUP_DESCRIPTION).encode('utf-8')] + logger.debug("ldap update group old_value %s new_value %s " % (old_value, new_value)) mymodlist = ldap.modlist.modifyModlist(old_value, new_value) result = conn.modify_s(dn, mymodlist) logger.debug("ldap update group by admin %s" % json.dumps(result[0])) diff --git a/src/tests/api/test_ldap_api.py b/src/tests/api/test_ldap_api.py index bdbdc4f9..f9806665 100644 --- a/src/tests/api/test_ldap_api.py +++ b/src/tests/api/test_ldap_api.py @@ -5,7 +5,7 @@ # # IoT orchestrator is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the +# published by the Free Software Founda, either version 3 of the # License, or (at your option) any later version. # # IoT orchestrator is distributed in the hope that it will be useful, @@ -32,6 +32,7 @@ USER_NAME="adm" USER_PASSWORD="4pass1w0rd" +GROUP_NAME="group" ORC_PROTOCOL="http" ORC_HOST="localhost" @@ -90,6 +91,15 @@ def __init__(self): "SubServiceAdminGroup"] } } + self.payload_data5b_ok = { + "USER_NAME": USER_NAME+"_%s" % self.suffix, + "USER_PASSWORD": USER_PASSWORD, + "USER_DATA": {"USER_EMAIL": "pepe33@acme.es", + "USER_DESCRIPTION": "Pepe perez", + "GROUP_NAMES": ["ServiceCustomerGroup", + "SubServiceAdminGroup"] + } + } self.suffix = str(uuid.uuid4())[:8] self.payload_data4_ok = { "NEW_USER_NAME": USER_NAME+"_%s" % self.suffix, @@ -169,6 +179,14 @@ def test_put_ok(self): data=self.payload_data5_ok) assert res.code == 200, (res.code, res.msg, res.raw_json) + def test_put2_ok(self): + res = self.TestRestOps.rest_request( + method="PUT", + url="/v1.0/ldap/user", + json_data=True, + data=self.payload_data5b_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + def test_delete_ok(self): res = self.TestRestOps.rest_request( method="DELETE", @@ -244,6 +262,69 @@ def test_post_and_delete_ok(self): assert res.code == 204, (res.code, res.msg, res.raw_json) +class Test_LDAPGroup_RestView(object): + + def __init__(self): + self.suffix = str(uuid.uuid4())[:8] + self.payload_data_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "NEW_GROUP_NAME": GROUP_NAME+"_%s" % self.suffix, + "NEW_USER_DESCRIPTION": GROUP_NAME+"_%s description" % self.suffix, + } + self.payload_data1b_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "GROUP_NAME": GROUP_NAME+"_%s" % self.suffix + } + self.payload_data1_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "FILTER": "*"+GROUP_NAME+"_%s*" % self.suffix + } + self.payload_data5_ok = { + "LDAP_ADMIN_USER": LDAP_ADMIN_USER, + "LDAP_ADMIN_PASSWORD": LDAP_ADMIN_PASSWORD, + "GROUP_NAME": GROUP_NAME+"_%s" % self.suffix, + "GROUP_DESCRIPTION": GROUP_NAME+"_%s description alt" % self.suffix, + } + self.TestRestOps = TestRestOperations(PROTOCOL=ORC_PROTOCOL, + HOST=ORC_HOST, + PORT=ORC_PORT) + + def test_post_ok(self): + res = self.TestRestOps.rest_request( + method="POST", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data_ok) + assert res.code == 201, (res.code, res.msg, res.raw_json) + + def test_delete_ok(self): + res = self.TestRestOps.rest_request( + method="DELETE", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data1b_ok) + assert res.code == 204, (res.code, res.msg, res.raw_json) + + def test_get_ok(self): + res = self.TestRestOps.rest_request( + method="GET", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data1_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + + def test_put_ok(self): + res = self.TestRestOps.rest_request( + method="PUT", + url="/v1.0/ldap/group", + json_data=True, + data=self.payload_data5_ok) + assert res.code == 200, (res.code, res.msg, res.raw_json) + + if __name__ == '__main__': # Tests @@ -255,6 +336,7 @@ def test_post_and_delete_ok(self): test_LdapUser.test_get2_ok() test_LdapUser.test_get3_ok() test_LdapUser.test_put_ok() + test_LdapUser.test_put2_ok() test_LdapUser.test_delete_ok() test_LdapUser.test_delete_bad() test_LdapUser.test_post2_ok() @@ -262,3 +344,10 @@ def test_post_and_delete_ok(self): test_LdapAuth = Test_LDAPAuth_RestView() test_LdapAuth.test_post_and_delete_ok() + + + test_LdapGroup = Test_LDAPGroup_RestView() + test_LdapGroup.test_post_ok() + test_LdapGroup.test_get_ok() + test_LdapGroup.test_put_ok() + test_LdapGroup.test_delete_ok() From 9d9daaa7a6059e220000a08d4caa5b65e8cc012a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 29 Jun 2021 11:39:15 +0200 Subject: [PATCH 28/33] fix url for check openldap endpoint~ --- src/orchestrator/core/openldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orchestrator/core/openldap.py b/src/orchestrator/core/openldap.py index 64909933..aaa39134 100644 --- a/src/orchestrator/core/openldap.py +++ b/src/orchestrator/core/openldap.py @@ -47,7 +47,7 @@ def __init__(self, self.LDAP_BASEDN = LDAP_BASEDN def checkLdap(self): - conn = ldap.initialize('ldap://'+self.LDAP_HOST+':'+self.LDAP_PORT) + conn = ldap.initialize('ldap://'+str(self.LDAP_HOST)+':'+str(self.LDAP_PORT)) # Just for check connection conn.simple_bind_s("","") From 065948cdfa26cd5f8f9153281361ad3bb193bada Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 12 Jul 2021 10:43:30 +0200 Subject: [PATCH 29/33] Update src/tests/api/test_ldap_api.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- src/tests/api/test_ldap_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/api/test_ldap_api.py b/src/tests/api/test_ldap_api.py index f9806665..254c663b 100644 --- a/src/tests/api/test_ldap_api.py +++ b/src/tests/api/test_ldap_api.py @@ -5,7 +5,7 @@ # # IoT orchestrator is free software: you can redistribute it and/or # modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Founda, either version 3 of the +# published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # IoT orchestrator is distributed in the hope that it will be useful, From 844f5dc77d99e46050414a95e68e002faff36acd Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 12 Jul 2021 10:56:44 +0200 Subject: [PATCH 30/33] Update ChangeLog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index bcb436a5..5fe14eed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ Upgrade Python (2 -> 3), #242 Django (1.11 -> 2.2.24) #310 DjangoRestFramework (3.9 -> 3.11) #309 requests (2.22.0 -> 2.25.1) - python-ldap (2.5.2 > 3.3.1) + python-ldap (2.5.2 -> 3.3.1) UPDATE: Update jsonschema dep 2.5.0 to 2.6.0 Upgrade STH xacml policies with orion frns From a7980952ef3fafb703d248439f7585a99411bf52 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 12 Jul 2021 11:14:00 +0200 Subject: [PATCH 31/33] Update ldap_view.py --- src/orchestrator/api/ldap_view.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/orchestrator/api/ldap_view.py b/src/orchestrator/api/ldap_view.py index 0bb3b38a..6e1e4561 100644 --- a/src/orchestrator/api/ldap_view.py +++ b/src/orchestrator/api/ldap_view.py @@ -168,10 +168,10 @@ def put(self, request): try: request.data # json validation flow = LdapUserHelper( - None, None, None, - None, None, None, - None, None, None, - None, None, None, + None, None, None, + None, None, None, + None, None, None, + None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, LDAP_BASEDN=self.LDAP_BASEDN, @@ -258,10 +258,10 @@ def post(self, request): try: request.data # json validation flow = LdapUserHelper( - None, None, None, - None, None, None, - None, None, None, - None, None, None, + None, None, None, + None, None, None, + None, None, None, + None, None, None, LDAP_HOST=self.LDAP_HOST, LDAP_PORT=self.LDAP_PORT, LDAP_BASEDN=self.LDAP_BASEDN, From 16b2944a7c3b4264ba5547793ce7accf11f8fea1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 12 Jul 2021 11:18:00 +0200 Subject: [PATCH 32/33] Update orchestrator.spec --- orchestrator.spec | 7 ------- 1 file changed, 7 deletions(-) diff --git a/orchestrator.spec b/orchestrator.spec index e506b062..7789a140 100644 --- a/orchestrator.spec +++ b/orchestrator.spec @@ -18,15 +18,8 @@ Prefix: /opt BuildArch: noarch %define _target_os Linux -%define python_lib /var/env-orchestrator/lib/python2.6/site-packages -%if 0%{?with_python27} -%define python_lib /var/env-orchestrator/lib/python2.7/site-packages -%define __python /usr/bin/python2.7 -%endif # if with_python27 -%if 0%{?with_python3} %define python_lib /var/env-orchestrator/lib/python3.6/site-packages %define __python /usr/bin/python3.6 -%endif # if with_python3 %description IoT Platform Orchestrator From 8b9b4e35038d9e0f58ea469c434e9ddbe01efd5d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 12 Jul 2021 11:19:29 +0200 Subject: [PATCH 33/33] Update package-orchestrator.sh --- package-orchestrator.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/package-orchestrator.sh b/package-orchestrator.sh index ff1ce04f..21a39ab9 100755 --- a/package-orchestrator.sh +++ b/package-orchestrator.sh @@ -35,8 +35,6 @@ source $BASE/get_version_string.sh string=$(get_rpm_version_string) VERSION_VALUE=${string% *} RELEASE_VALUE=${string#* } -PYTHON27_VALUE=0 -PYTHON3_VALUE=0 args=("$@") ELEMENTS=${#args[@]} @@ -49,12 +47,6 @@ for (( i=0;i<$ELEMENTS;i++)); do if [ "$arg" == "--with-release" ]; then RELEASE_VALUE=${args[${i}+1]} fi - if [ "$arg" == "--with-python27" ]; then - PYTHON27_VALUE=1 - fi - if [ "$arg" == "--with-python3" ]; then - PYTHON3_VALUE=1 - fi done @@ -67,7 +59,5 @@ rpmbuild -bb orchestrator.spec \ --define "_topdir $RPM_DIR" \ --define "_root $BASE"\ --define "_project_user $ORCHESTRATOR_USER"\ - --define "with_python27 $PYTHON27_VALUE"\ - --define "with_python3 $PYTHON3_VALUE"\ --define "_version $VERSION_VALUE"\ --define "_release $RELEASE_VALUE"