From 2a9c04c3f084775560c7c2b93429a07d63bb8983 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Wed, 10 Apr 2019 22:10:29 +0200 Subject: [PATCH 1/3] Implemented forwarded updates in NGSIv2 --- CHANGES_NEXT_RELEASE | 3 +- src/lib/mongoBackend/MongoCommonUpdate.cpp | 18 + src/lib/mongoBackend/MongoGlobal.cpp | 207 ++++++--- src/lib/mongoBackend/MongoGlobal.h | 3 +- src/lib/ngsi10/UpdateContextRequest.cpp | 13 +- src/lib/ngsi10/UpdateContextRequest.h | 3 +- src/lib/rest/httpRequestSend.cpp | 1 + src/lib/serviceRoutines/postUpdateContext.cpp | 234 ++++++---- src/lib/serviceRoutinesV2/patchEntity.cpp | 8 +- .../update_forwards_with_accumulator.test | 132 ++++++ .../update_forwards_with_one_provider.test | 152 +++++++ .../update_forwards_with_providers.test | 415 ++++++++++++++++++ test/functionalTest/harnessFunctions.sh | 10 +- .../mongoDiscoverContextAvailability_test.cpp | 13 +- 14 files changed, 1062 insertions(+), 150 deletions(-) create mode 100644 test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_accumulator.test create mode 100644 test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_one_provider.test create mode 100644 test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_providers.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1cc59753ef..69f6c83343 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ - Fix: idPattern '.*' is now allowed in NGSIv2 registrations (#3458) - Add: Forwarded queries work for registrations using idPattern == '.*', but only for simple cases (#3458) -- Add: NGSIv2 forwards now works for queries - for simple cases (#3068) +- Add: Forwarded updates work for registrations using idPattern == '.*', but only for simple cases (#3458) +- Add: NGSIv2 forwards now works for queries and updates - for simple cases (#3068) diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp index 21946ae334..a8780d49ae 100644 --- a/src/lib/mongoBackend/MongoCommonUpdate.cpp +++ b/src/lib/mongoBackend/MongoCommonUpdate.cpp @@ -2703,17 +2703,24 @@ static void searchContextProviders std::string err; /* Fill input data for registrationsQuery() */ + LM_T(LmtForward, ("Entity Id: %s", en.id.c_str())); + LM_T(LmtForward, ("Entity isPattern: %s", en.isPattern.c_str())); + LM_T(LmtForward, ("Entity Type: %s", en.type.c_str())); + enV.push_back(&en); for (unsigned int ix = 0; ix < caV.size(); ++ix) { attrL.push_back(caV[ix]->name); + LM_T(LmtForward, ("Attr %d: %s", ix, caV[ix]->name.c_str())); } /* First CPr lookup (in the case some CER is not found): looking in E-A registrations */ if (someContextElementNotFound(*cerP)) { + LM_T(LmtForward, ("Calling registrationsQuery I")); if (registrationsQuery(enV, attrL, &crrV, &err, tenant, servicePathV, 0, 0, false)) { + LM_T(LmtForward, ("registrationsQuery I returned %d items in crrV", crrV.size())); if (crrV.size() > 0) { fillContextProviders(cerP, crrV); @@ -2734,8 +2741,10 @@ static void searchContextProviders StringList attrNullList; if (someContextElementNotFound(*cerP)) { + LM_T(LmtForward, ("Calling registrationsQuery II")); if (registrationsQuery(enV, attrNullList, &crrV, &err, tenant, servicePathV, 0, 0, false)) { + LM_T(LmtForward, ("registrationsQuery II returned %d items in crrV", crrV.size())); if (crrV.size() > 0) { fillContextProviders(cerP, crrV); @@ -2761,21 +2770,26 @@ static void searchContextProviders */ static bool forwardsPending(UpdateContextResponse* upcrsP) { + LM_T(LmtForward, ("Looping over %d context element responses", upcrsP->contextElementResponseVector.size())); for (unsigned int cerIx = 0; cerIx < upcrsP->contextElementResponseVector.size(); ++cerIx) { ContextElementResponse* cerP = upcrsP->contextElementResponseVector[cerIx]; + LM_T(LmtForward, ("Looping over %d attributes", cerP->entity.attributeVector.size())); for (unsigned int aIx = 0 ; aIx < cerP->entity.attributeVector.size(); ++aIx) { ContextAttribute* aP = cerP->entity.attributeVector[aIx]; + LM_T(LmtForward, ("Attribute: '%s'", aP->name.c_str())); if (aP->providingApplication.get() != "") { + LM_T(LmtForward, ("Found a providingApplication: %s", aP->providingApplication.get().c_str())); return true; } } } + LM_T(LmtForward, ("No providingApplication found")); return false; } @@ -2952,6 +2966,7 @@ static void updateEntity // FIXME P8: the same three statements are at the end of the while loop. Refactor the code to have this // in only one place // + LM_T(LmtForward, ("Calling searchContextProviders")); searchContextProviders(tenant, servicePathV, en, eP->attributeVector, cerP); if (!(attributeAlreadyExistsError && (action == ActionTypeAppendStrict))) @@ -3488,8 +3503,11 @@ void processContextElement // // If no context providers found, then the UPDATE was simply for a non-found entity and an error should be returned // + LM_T(LmtForward, ("Calling forwardsPending")); if (forwardsPending(responseP) == false) { + LM_T(LmtForward, ("No forwards pending")); + cerP->statusCode.fill(SccContextElementNotFound); if (apiVersion == V1) diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index 23e46ea54f..c50c91f214 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -753,6 +753,22 @@ bool matchEntity(const EntityId* en1, const EntityId* en2) regfree(®ex); // If regcomp fails it frees up itself (see glibc sources for details) } } + else if (isTrue(en1->isPattern)) + { + regex_t regex; + + idMatch = false; + if (regcomp(®ex, en1->id.c_str(), REG_EXTENDED) != 0) + { + std::string details = std::string("error compiling regex for id: '") + en2->id + "'"; + alarmMgr.badInput(clientIp, details); + } + else + { + idMatch = (regexec(®ex, en2->id.c_str(), 0, NULL, 0) == 0); + regfree(®ex); // If regcomp fails it frees up itself (see glibc sources for details) + } + } else /* isPattern == false */ { idMatch = (en2->id == en1->id); @@ -1886,10 +1902,12 @@ static void processEntity(ContextRegistrationResponse* crr, const EntityIdVector en.type = entity.hasField(REG_ENTITY_TYPE)? getStringFieldF(entity, REG_ENTITY_TYPE) : ""; en.isPattern = entity.hasField(REG_ENTITY_ISPATTERN)? getStringFieldF(entity, REG_ENTITY_ISPATTERN) : "false"; + LM_T(LmtForward, ("Entity: id:'%s'/isPattern:'%s'/type:'%s'", en.id.c_str(), en.isPattern.c_str(), en.type.c_str())); if (includedEntity(en, enV)) { EntityId* enP = new EntityId(en.id, en.type, en.isPattern); + LM_T(LmtForward, ("Included - adding entity to crr->contextRegistration.entityIdVector")); crr->contextRegistration.entityIdVector.push_back(enP); } } @@ -1931,18 +1949,23 @@ static void processContextRegistrationElement { ContextRegistrationResponse crr; + LM_T(LmtForward, ("IN: enV of %d elements, attrL of %d elements", enV.size(), attrL.size())); + crr.contextRegistration.providingApplication.set(getStringFieldF(cr, REG_PROVIDING_APPLICATION)); crr.contextRegistration.providingApplication.setMimeType(mimeType); crr.contextRegistration.providingApplication.setProviderFormat(providerFormat); + LM_T(LmtForward, ("Set providerFormat to %d for CRR", providerFormat)); std::vector queryEntityV = getFieldF(cr, REG_ENTITIES).Array(); + LM_T(LmtForward, ("queryEntityV.size() == %d", queryEntityV.size())); for (unsigned int ix = 0; ix < queryEntityV.size(); ++ix) { processEntity(&crr, enV, queryEntityV[ix].embeddedObject()); } /* Note that attributes can be included only if at least one entity has been found */ + LM_T(LmtForward, ("crr.contextRegistration.entityIdVector.size() == %d", crr.contextRegistration.entityIdVector.size())); if (crr.contextRegistration.entityIdVector.size() > 0) { if (cr.hasField(REG_ATTRS)) /* To prevent registration in the E- style */ @@ -1966,6 +1989,7 @@ static void processContextRegistrationElement */ if (crr.contextRegistration.entityIdVector.size() == 0) { + LM_T(LmtForward, ("crr.contextRegistration.entityIdVector.size is ZERO - we are done. NO OUTPUT")); return; } @@ -1977,8 +2001,11 @@ static void processContextRegistrationElement crrP->contextRegistration = crr.contextRegistration; crrP->providerFormat = providerFormat; + LM_T(LmtForward, ("Adding an OUTPUT crr (providingApplication: '%s')", crrP->contextRegistration.providingApplication.get().c_str())); crrV->push_back(crrP); } + else + LM_T(LmtForward, ("No OUTPUT crr")); } @@ -2005,51 +2032,113 @@ bool registrationsQuery long long* countP ) { - /* Build query based on arguments */ - // FIXME P2: this implementation needs to be refactored for cleanup - std::string contextRegistrationEntities = REG_CONTEXT_REGISTRATION "." REG_ENTITIES; - std::string contextRegistrationEntitiesId = REG_CONTEXT_REGISTRATION "." REG_ENTITIES "." REG_ENTITY_ID; - std::string contextRegistrationEntitiesType = REG_CONTEXT_REGISTRATION "." REG_ENTITIES "." REG_ENTITY_TYPE; - std::string contextRegistrationAttrsNames = REG_CONTEXT_REGISTRATION "." REG_ATTRS "." REG_ATTRS_NAME; - BSONArrayBuilder entityOr; - BSONArrayBuilder entitiesWithType; - BSONArrayBuilder entitiesWithoutType; + mongo::BSONObjBuilder queryBuilder; + mongo::BSONArrayBuilder entityOr; - for (unsigned int ix = 0; ix < enV.size(); ++ix) + // + // If only one entity in the REST request, the query to mongo can be simplified + // NGSIv2 requests like PATCH /v2/entities//attrs can have only ONE entity + // and will ALWAYS enter the "then part" of the if-then-else. + // + // Note than registrations with entity id as a pattern is treated by the "then part" but not + // by the older "else part". + // + // One NGSIv1 is removed from the source code, the "else part" can be removed by just making sure that + // NGSIv2 batch operations call this function once per entity in the request. + // Or, we'd need to reimplement to make the else-part also support registrations with entity id as a pattern. + // + if (enV.size() == 1) { - const EntityId* en = enV[ix]; + if ((enV[0]->id == "") && (enV[0]->type == "")) + { + LM_E(("Bad Request (too wide query - at least one of entity::id and entity::type must be given)")); + *err = "too wide query"; + return false; + } - if (isTrue(en->isPattern)) + // Entity ID - if not given, no Entity ID in query - then ALL ENTITY IDs match, so not included in query + if (enV[0]->id != "") { - BSONObjBuilder b; + mongo::BSONObjBuilder entityAnd; + mongo::BSONObjBuilder id; - b.appendRegex(contextRegistrationEntitiesId, en->id); - if (en->type != "") - { - b.append(contextRegistrationEntitiesType, en->type); - } - entityOr.append(b.obj()); + // If the Registration is with idPattern, then it matches, as the only idPattern allowed for Registrations is ".*" + entityAnd.append("contextRegistration.entities.isPattern", "true"); + entityAnd.append("contextRegistration.entities.id", ".*"); // TPUT: if isPattern == true, then id MUST be ".*" ! Remove? + entityOr.append(entityAnd.obj()); + + if (enV[0]->isPattern == "true") id.appendRegex("contextRegistration.entities.id", enV[0]->id); + else id.append("contextRegistration.entities.id", enV[0]->id); + + entityOr.append(id.obj()); + queryBuilder.append("$or", entityOr.arr()); + } + + // Entity Type + if (enV[0]->type != "") + { + queryBuilder.append("contextRegistration.entities.type", enV[0]->type); } - else /* isPattern = false */ + } + else // More than one Entity in enV + { + /* Build query based on arguments */ + // FIXME P2: this implementation needs to be refactored for cleanup + std::string contextRegistrationEntities = REG_CONTEXT_REGISTRATION "." REG_ENTITIES; + std::string contextRegistrationEntitiesId = REG_CONTEXT_REGISTRATION "." REG_ENTITIES "." REG_ENTITY_ID; + std::string contextRegistrationEntitiesType = REG_CONTEXT_REGISTRATION "." REG_ENTITIES "." REG_ENTITY_TYPE; + BSONArrayBuilder entitiesWithType; + BSONArrayBuilder entitiesWithoutType; + + for (unsigned int ix = 0; ix < enV.size(); ++ix) { - if (en->type == "") + const EntityId* en = enV[ix]; + + if (isTrue(en->isPattern)) { - entitiesWithoutType.append(en->id); - LM_T(LmtMongo, ("Entity discovery without type: id '%s'", en->id.c_str())); + BSONObjBuilder b; + + b.appendRegex(contextRegistrationEntitiesId, en->id); + if (en->type != "") + { + b.append(contextRegistrationEntitiesType, en->type); + } + entityOr.append(b.obj()); } - else + else /* isPattern = false */ { - /* We have detected that sometimes mongo stores { id: ..., type ...} and sometimes { type: ..., id: ...}, - so we have to take both of them into account - */ - entitiesWithType.append(BSON(REG_ENTITY_ID << en->id << REG_ENTITY_TYPE << en->type)); - entitiesWithType.append(BSON(REG_ENTITY_TYPE << en->type << REG_ENTITY_ID << en->id)); - LM_T(LmtMongo, ("Entity discovery: {id: %s, type: %s}", en->id.c_str(), en->type.c_str())); + if (en->type == "") + { + entitiesWithoutType.append(BSON(REG_ENTITY_ISPATTERN << "true")); + entitiesWithoutType.append(BSON(REG_ENTITY_ID << en->id)); + LM_T(LmtMongo, ("Entity discovery without type: id '%s'", en->id.c_str())); + } + else + { + /* We have detected that sometimes mongo stores { id: ..., type ...} and sometimes { type: ..., id: ...}, + so we have to take both of them into account + */ + entitiesWithType.append(BSON(REG_ENTITY_ID << en->id << REG_ENTITY_TYPE << en->type)); + entitiesWithType.append(BSON(REG_ENTITY_TYPE << en->type << REG_ENTITY_ID << en->id)); + LM_T(LmtMongo, ("Entity discovery: {id: %s, type: %s}", en->id.c_str(), en->type.c_str())); + } } } + + entityOr.append(BSON(contextRegistrationEntities << BSON("$in" << entitiesWithType.arr()))); + entityOr.append(BSON(contextRegistrationEntitiesId << BSON("$in" << entitiesWithoutType.arr()))); + + /* The $or clause could be omitted if it contains only one element, but we can assume that + * it has no impact on MongoDB query optimizer + */ + queryBuilder.append("$or", entityOr.arr()); } + // + // Attributes + // BSONArrayBuilder attrs; + std::string contextRegistrationAttrsNames = REG_CONTEXT_REGISTRATION "." REG_ATTRS "." REG_ATTRS_NAME; for (unsigned int ix = 0; ix < attrL.size(); ++ix) { @@ -2059,17 +2148,6 @@ bool registrationsQuery LM_T(LmtMongo, ("Attribute discovery: '%s'", attrName.c_str())); } - entityOr.append(BSON(contextRegistrationEntities << BSON("$in" << entitiesWithType.arr()))); - entityOr.append(BSON(contextRegistrationEntitiesId << BSON("$in" << entitiesWithoutType.arr()))); - - BSONObjBuilder queryBuilder; - - /* The $or clause could be omitted if it contains only one element, but we can assume that - * it has no impact on MongoDB query optimizer - */ - queryBuilder.append("$or", entityOr.arr()); - queryBuilder.append(REG_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); - if (attrs.arrSize() > 0) { /* If we don't do this check, the {$in: [] } of the attribute name part makes the query fail */ @@ -2077,11 +2155,14 @@ bool registrationsQuery } // - // 'And-in' the service path + // Service Path // - const std::string servicePathString = REG_SERVICE_PATH; + queryBuilder.append(REG_SERVICE_PATH, fillQueryServicePath(servicePathV)); - queryBuilder.append(servicePathString, fillQueryServicePath(servicePathV)); + // + // Expiration + // + queryBuilder.append(REG_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); // @@ -2129,11 +2210,13 @@ bool registrationsQuery std::string format = getStringFieldF(r, REG_FORMAT); ProviderFormat providerFormat = (format == "")? PfJson : (format == "JSON")? PfJson : PfV2; + LM_T(LmtForward, ("--------------- %d registration elements to process", queryContextRegistrationV.size())); for (unsigned int ix = 0 ; ix < queryContextRegistrationV.size(); ++ix) { LM_T(LmtForward, ("Processing ContextRegistrationElement. providerFormat == '%s' (%d)", format.c_str(), providerFormat)); processContextRegistrationElement(queryContextRegistrationV[ix].embeddedObject(), enV, attrL, crrV, mimeType, providerFormat); } + LM_T(LmtForward, ("--------------- %d registration elements processed - %d elementd output in crrV", queryContextRegistrationV.size(), crrV->size())); /* FIXME: note that given the response doesn't distinguish from which registration ID the * response comes, it could have that we have same context registration elements, belong to different @@ -2756,32 +2839,41 @@ void releaseTriggeredSubscriptions(std::map */ void fillContextProviders(ContextElementResponse* cer, const ContextRegistrationResponseVector& crrV) { + LM_T(LmtForward, ("fillContextProviders: %d attributes", cer->entity.attributeVector.size())); for (unsigned int ix = 0; ix < cer->entity.attributeVector.size(); ++ix) { ContextAttribute* ca = cer->entity.attributeVector[ix]; + LM_T(LmtForward, ("fillContextProviders: Attribute %d: %s", ix, ca->name.c_str())); if (ca->found) { + LM_T(LmtForward, ("fillContextProviders: Attribute %s is already found", ca->name.c_str())); continue; } /* Search for some CPr in crrV */ - std::string perEntPa; - std::string perAttrPa; - MimeType perEntPaMimeType = NOMIMETYPE; - MimeType perAttrPaMimeType = NOMIMETYPE; + std::string perEntPa; + std::string perAttrPa; + MimeType perEntPaMimeType = NOMIMETYPE; + MimeType perAttrPaMimeType = NOMIMETYPE; + ProviderFormat providerFormat; + LM_T(LmtForward, ("fillContextProviders: looking up CPr for %s", ca->name.c_str())); cprLookupByAttribute(cer->entity, ca->name, crrV, &perEntPa, &perEntPaMimeType, &perAttrPa, - &perAttrPaMimeType); + &perAttrPaMimeType, + &providerFormat); /* Looking results after crrV processing */ ca->providingApplication.set(perAttrPa == "" ? perEntPa : perAttrPa); ca->providingApplication.setMimeType(perAttrPa == "" ? perEntPaMimeType : perAttrPaMimeType); + ca->providingApplication.setProviderFormat(providerFormat); + LM_T(LmtForward, ("Set providerFormat to %d", providerFormat)); + ca->found = (ca->providingApplication.get() != ""); } } @@ -2828,12 +2920,14 @@ void cprLookupByAttribute std::string* perEntPa, MimeType* perEntPaMimeType, std::string* perAttrPa, - MimeType* perAttrPaMimeType + MimeType* perAttrPaMimeType, + ProviderFormat* providerFormatP ) { *perEntPa = ""; *perAttrPa = ""; + LM_T(LmtForward, ("In cprLookupByAttribute. Looping over %d CRRs", crrV.size())); for (unsigned int crrIx = 0; crrIx < crrV.size(); ++crrIx) { ContextRegistrationResponse* crr = crrV[crrIx]; @@ -2845,8 +2939,12 @@ void cprLookupByAttribute if (regEn->id != en.id || (regEn->type != en.type && regEn->type != "")) { - /* No match (keep searching the CRR) */ - continue; + if (regEn->isPattern != "true") + { + /* No match (keep searching the CRR) */ + LM_T(LmtForward, ("No match. Entity ID: '%s', isPattern: '%s', Type: '%s'", regEn->id.c_str(), regEn->isPattern.c_str(), regEn->type.c_str())); + continue; + } } /* CRR without attributes (keep searching in other CRR) */ @@ -2854,7 +2952,8 @@ void cprLookupByAttribute { *perEntPa = crr->contextRegistration.providingApplication.get(); *perEntPaMimeType = crr->contextRegistration.providingApplication.getMimeType(); - + *providerFormatP = crr->contextRegistration.providingApplication.getProviderFormat(); + LM_T(LmtForward, ("providerFormat: %d", *providerFormatP)); break; /* enIx */ } @@ -2867,6 +2966,8 @@ void cprLookupByAttribute /* We cannot "improve" this result by keep searching the CRR vector, so we return */ *perAttrPa = crr->contextRegistration.providingApplication.get(); *perAttrPaMimeType = crr->contextRegistration.providingApplication.getMimeType(); + *providerFormatP = crr->contextRegistration.providingApplication.getProviderFormat(); + LM_T(LmtForward, ("providerFormat: %d", *providerFormatP)); return; } diff --git a/src/lib/mongoBackend/MongoGlobal.h b/src/lib/mongoBackend/MongoGlobal.h index 5080d9fba4..6a7bcdb885 100644 --- a/src/lib/mongoBackend/MongoGlobal.h +++ b/src/lib/mongoBackend/MongoGlobal.h @@ -505,7 +505,8 @@ extern void cprLookupByAttribute std::string* perEntPa, MimeType* perEntPaMimeType, std::string* perAttrPa, - MimeType* perAttrPaMimeType + MimeType* perAttrPaMimeType, + ProviderFormat* providerFormatP ); diff --git a/src/lib/ngsi10/UpdateContextRequest.cpp b/src/lib/ngsi10/UpdateContextRequest.cpp index 967692012b..bbffcd2e23 100644 --- a/src/lib/ngsi10/UpdateContextRequest.cpp +++ b/src/lib/ngsi10/UpdateContextRequest.cpp @@ -39,6 +39,7 @@ #include "convenience/UpdateContextAttributeRequest.h" + /* **************************************************************************** * * UpdateContextRequest::UpdateContextRequest - @@ -53,13 +54,15 @@ UpdateContextRequest::UpdateContextRequest() * * UpdateContextRequest::UpdateContextRequest - */ -UpdateContextRequest::UpdateContextRequest(const std::string& _contextProvider, Entity* eP) +UpdateContextRequest::UpdateContextRequest(const std::string& _contextProvider, ProviderFormat _providerFormat, Entity* eP) { contextProvider = _contextProvider; + providerFormat = _providerFormat; entityVector.push_back(new Entity(eP->id, eP->type, eP->isPattern)); } + /* **************************************************************************** * * UpdateContextRequest::toJsonV1 - @@ -68,9 +71,10 @@ std::string UpdateContextRequest::toJsonV1(bool asJsonObject) { std::string out = ""; - // JSON commas: - // Both fields are MANDATORY, so, comma after "contextElementVector" - // + // + // About JSON commas: + // Both fields are MANDATORY, so, always comma after "entityVector" + // out += startTag(); out += entityVector.toJsonV1(asJsonObject, UpdateContext, true); out += valueTag("updateAction", actionTypeString(V1, updateActionType), false); @@ -80,6 +84,7 @@ std::string UpdateContextRequest::toJsonV1(bool asJsonObject) } + /* **************************************************************************** * * UpdateContextRequest::check - diff --git a/src/lib/ngsi10/UpdateContextRequest.h b/src/lib/ngsi10/UpdateContextRequest.h index e68838bf74..e8b39424a9 100644 --- a/src/lib/ngsi10/UpdateContextRequest.h +++ b/src/lib/ngsi10/UpdateContextRequest.h @@ -55,9 +55,10 @@ typedef struct UpdateContextRequest ActionType updateActionType; // Mandatory std::string contextProvider; // Not part of the payload - used internally only + ProviderFormat providerFormat; // Not part of the payload - used internally only UpdateContextRequest(); - UpdateContextRequest(const std::string& _contextProvider, Entity* eP); + UpdateContextRequest(const std::string& _contextProvider, ProviderFormat _providerFormat, Entity* eP); std::string toJsonV1(bool asJsonObject); std::string check(ApiVersion apiVersion, bool asJsonObject, const std::string& predetectedError); diff --git a/src/lib/rest/httpRequestSend.cpp b/src/lib/rest/httpRequestSend.cpp index cf77a06c87..c441abb7a2 100644 --- a/src/lib/rest/httpRequestSend.cpp +++ b/src/lib/rest/httpRequestSend.cpp @@ -612,6 +612,7 @@ int httpRequestSendWithCurl int payloadLen = contentLenParse(httpResponse->memory); LM_I(("Notification Successfully Sent to %s", url.c_str())); + LM_T(LmtForward, ("Forwarded Request: %s %s: %s", verb.c_str(), url.c_str(), payload)); outP->assign(httpResponse->memory, httpResponse->size); metricsMgr.add(tenant, servicePath0, METRIC_TRANS_OUT_RESP_SIZE, payloadLen); diff --git a/src/lib/serviceRoutines/postUpdateContext.cpp b/src/lib/serviceRoutines/postUpdateContext.cpp index 114a56092f..838d5b3279 100644 --- a/src/lib/serviceRoutines/postUpdateContext.cpp +++ b/src/lib/serviceRoutines/postUpdateContext.cpp @@ -101,7 +101,7 @@ static bool forwardsPending(UpdateContextResponse* upcrsP) * 7. Freeing memory * */ -static void updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, UpdateContextResponse* upcrsP) +static bool updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, UpdateContextResponse* upcrsP) { std::string ip; std::string protocol; @@ -124,9 +124,10 @@ static void updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda // SccBadRequest should have been returned before, when it was registered! // upcrsP->errorCode.fill(SccContextElementNotFound, ""); - return; + return false; } + LM_T(LmtForward, ("*** Provider Format: %d", upcrP->providerFormat)); // // 2. Render the string of the request we want to forward @@ -137,13 +138,39 @@ static void updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda ciP->outMimeType = JSON; - // - // FIXME: Forwards are done using NGSIv1 only, for now - // This will hopefully change soon ... - // Once we implement forwards in NGSIv2, this toJsonV1() should be like this: - // TIMED_RENDER(payload = upcrP->toJson()); - // - TIMED_RENDER(payload = upcrP->toJsonV1(asJsonObject)); + std::string verb; + std::string resource; + std::string tenant = ciP->tenant; + std::string servicePath = (ciP->httpHeaders.servicePathReceived == true)? ciP->httpHeaders.servicePath : ""; + std::string mimeType = "application/json"; + std::string out; + int r; + + if (upcrP->providerFormat == PfJson) + { + TIMED_RENDER(payload = upcrP->toJsonV1(asJsonObject)); + + verb = "POST"; + resource = prefix + "/updateContext"; + } + else + { + std::vector nullFilter; + Entity* eP = upcrP->entityVector[0]; + + eP->renderId = false; + + TIMED_RENDER(payload = eP->toJson(NGSI_V2_NORMALIZED, nullFilter, false, nullFilter)); + + resource = prefix + "/entities/" + eP->id + "/attrs"; + verb = "PATCH"; + + if (eP->type != "") + { + // Add ?type= to 'resource' + resource += "?type=" + eP->type; + } + } ciP->outMimeType = outMimeType; cleanPayload = (char*) payload.c_str(); @@ -152,14 +179,6 @@ static void updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda // 3. Send the request to the Context Provider (and await the reply) // FIXME P7: Should Rush be used? // - std::string verb = "POST"; - std::string resource = prefix + "/updateContext"; - std::string tenant = ciP->tenant; - std::string servicePath = (ciP->httpHeaders.servicePathReceived == true)? ciP->httpHeaders.servicePath : ""; - std::string mimeType = "application/json"; - std::string out; - int r; - LM_T(LmtCPrForwardRequestPayload, ("forward updateContext request payload: %s", payload.c_str())); std::map noHeaders; @@ -188,82 +207,117 @@ static void updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda { upcrsP->errorCode.fill(SccContextElementNotFound, "error forwarding update"); LM_E(("Runtime Error (error '%s' forwarding 'Update' to providing application)", out.c_str())); - return; + return false; } LM_T(LmtCPrForwardResponsePayload, ("forward updateContext response payload: %s", out.c_str())); - // - // 4. Parse the response and fill in a binary UpdateContextResponse + // If NGSIv1: + // 4. Parse the response and fill in a binary UpdateContextResponse + // 5. Fill in the response from the redirection into the response of this function + // 6. 'Fix' StatusCode + // 7. Free up memory + // + // If NGSIv2: + // 4. Look for "204 No Content" in the response of the forwarded request + // 5. If found: OK, else, error // - std::string s; - std::string errorMsg; + if (upcrP->providerFormat == PfJson) + { + LM_T(LmtForward, ("upcrP->providerFormat == PfJson")); + // + // 4. Parse the response and fill in a binary UpdateContextResponse + // + std::string s; + std::string errorMsg; - cleanPayload = jsonPayloadClean(out.c_str()); + cleanPayload = jsonPayloadClean(out.c_str()); + + if ((cleanPayload == NULL) || (cleanPayload[0] == 0)) + { + // + // This is really an internal error in the Context Provider + // It is not in the orion broker though, so 404 is returned + // + LM_W(("Other Error (context provider response to UpdateContext is empty)")); + upcrsP->errorCode.fill(SccContextElementNotFound, "invalid context provider response"); + return false; + } - if ((cleanPayload == NULL) || (cleanPayload[0] == 0)) - { // - // This is really an internal error in the Context Provider - // It is not in the orion broker though, so 404 is returned + // NOTE + // When coming from a convenience operation, such as GET /v1/contextEntities/EID/attributes/attrName, + // the verb/method in ciP is GET. However, the parsing function expects a POST, as if it came from a + // POST /v1/updateContext. + // So, here we change the verb/method for POST. // - LM_W(("Other Error (context provider response to UpdateContext is empty)")); - upcrsP->errorCode.fill(SccContextElementNotFound, "invalid context provider response"); - return; - } + ParseData parseData; - // - // NOTE - // When coming from a convenience operation, such as GET /v1/contextEntities/EID/attributes/attrName, - // the verb/method in ciP is GET. However, the parsing function expects a POST, as if it came from a - // POST /v1/updateContext. - // So, here we change the verb/method for POST. - // - ParseData parseData; + ciP->verb = POST; + ciP->method = "POST"; - ciP->verb = POST; - ciP->method = "POST"; + parseData.upcrs.res.errorCode.fill(SccOk); - parseData.upcrs.res.errorCode.fill(SccOk); + LM_T(LmtForward, ("Parsing Response of Forwarded Request: '%s'", cleanPayload)); + s = jsonTreat(cleanPayload, ciP, &parseData, RtUpdateContextResponse, NULL); + LM_T(LmtForward, ("Parse Result: %s", s.c_str())); - s = jsonTreat(cleanPayload, ciP, &parseData, RtUpdateContextResponse, NULL); + if (s != "OK") + { + LM_W(("Internal Error (error parsing reply from prov app: %s)", errorMsg.c_str())); + upcrsP->errorCode.fill(SccContextElementNotFound, ""); + parseData.upcr.res.release(); + parseData.upcrs.res.release(); + return false; + } - if (s != "OK") - { - LM_W(("Internal Error (error parsing reply from prov app: %s)", errorMsg.c_str())); - upcrsP->errorCode.fill(SccContextElementNotFound, ""); - parseData.upcr.res.release(); - parseData.upcrs.res.release(); - return; - } + // + // 5. Fill in the response from the redirection into the response of this function + // + upcrsP->fill(&parseData.upcrs.res); - // - // 5. Fill in the response from the redirection into the response of this function - // - upcrsP->fill(&parseData.upcrs.res); + // + // 6. 'Fix' StatusCode + // + if (upcrsP->errorCode.code == SccNone) + { + upcrsP->errorCode.fill(SccOk); + } - // - // 6. 'Fix' StatusCode - // - if (upcrsP->errorCode.code == SccNone) - { - upcrsP->errorCode.fill(SccOk); - } + if ((upcrsP->contextElementResponseVector.size() == 1) && (upcrsP->contextElementResponseVector[0]->statusCode.code == SccContextElementNotFound)) + { + upcrsP->errorCode.fill(SccContextElementNotFound); + } - if ((upcrsP->contextElementResponseVector.size() == 1) && (upcrsP->contextElementResponseVector[0]->statusCode.code == SccContextElementNotFound)) - { - upcrsP->errorCode.fill(SccContextElementNotFound); + // + // 7. Free up memory + // + parseData.upcr.res.release(); + parseData.upcrs.res.release(); + + LM_T(LmtForward, ("V1 Forward OK")); + return true; } + else // V2 + { + LM_T(LmtForward, ("upcrP->providerFormat == V2. out: '%s'", out.c_str())); + // NGSIv2 forward - no payload to be received, just a 204 No Content HTTP Header + if (strstr(out.c_str(), "204 No Content") != NULL) + { + LM_T(LmtForward, ("Found '204 No Content'")); + upcrsP->errorCode.fill(SccNone); + return true; + } + LM_W(("Other Error (not the expected response from content provider: %s)", out.c_str())); + upcrsP->errorCode.fill(SccReceiverInternalError); + return false; + } - // - // 7. Freeing memory - // - parseData.upcr.res.release(); - parseData.upcrs.res.release(); + // Can't reach this point - no return-statement needed } @@ -525,6 +579,7 @@ std::string postUpdateContext // If there is nothing to forward, just return the result // bool forwarding = forwardsPending(upcrsP); + LM_T(LmtForward, ("forwardsPending returned %s", FT(forwarding))); if (forwarding == false) { TIMED_RENDER(answer = upcrsP->toJsonV1(asJsonObject)); @@ -623,7 +678,7 @@ std::string postUpdateContext UpdateContextRequest* reqP = requestV.lookup(aP->providingApplication.get()); if (reqP == NULL) { - reqP = new UpdateContextRequest(aP->providingApplication.get(), &cerP->entity); + reqP = new UpdateContextRequest(aP->providingApplication.get(), aP->providingApplication.providerFormat, &cerP->entity); reqP->updateActionType = ActionTypeUpdate; requestV.push_back(reqP); } @@ -659,6 +714,7 @@ std::string postUpdateContext // Calling each of the Context Providers, merging their results into the // total response 'response' // + bool forwardOk = true; for (unsigned int ix = 0; ix < requestV.size() && ix < cprForwardLimit; ++ix) { @@ -669,8 +725,14 @@ std::string postUpdateContext } UpdateContextResponse upcrs; + bool b; - updateForward(ciP, requestV[ix], &upcrs); + b = updateForward(ciP, requestV[ix], &upcrs); + + if (b == false) + { + forwardOk = false; + } // // Add the result from the forwarded update to the total response in 'response' @@ -678,26 +740,30 @@ std::string postUpdateContext response.merge(&upcrs); } + // // Note this is a slight break in the separation of concerns among the different layers (i.e. // serviceRoutine/ logic should work in a "NGSIv1 isolated context"). However, it seems to be // a smart way of dealing with partial update situations + // if (ciP->apiVersion == V2) { + LM_T(LmtForward, ("ciP->apiVersion == V2")); + // // Adjust OrionError response in the case of partial updates. This may happen in CPr forwarding // scenarios. Note that mongoBackend logic "splits" successfull updates and failing updates in // two different CER (maybe using the same entity) - + // std::string failing = ""; - unsigned int fails = 0; + unsigned int failures = 0; + LM_T(LmtForward, ("Going over a contextElementResponseVector of %d items", response.contextElementResponseVector.size())); for (unsigned int ix = 0; ix < response.contextElementResponseVector.size(); ++ix) { - ContextElementResponse* cerP = response.contextElementResponseVector[ix]; if (cerP->statusCode.code != SccOk) { - fails++; + failures++; std::string failingPerCer = ""; for (unsigned int jx = 0; jx < cerP->entity.attributeVector.size(); ++jx) @@ -713,29 +779,35 @@ std::string postUpdateContext } } + // // Note that we modify parseDataP->upcrs.res.oe and not response.oe, as the former is the // one used by the calling postBatchUpdate() function at serviceRoutineV2 library - if (fails == response.contextElementResponseVector.size()) + // + if ((forwardOk == true) && (failures == 0)) + { + parseDataP->upcrs.res.oe.fill(SccNone, ""); + } + else if (failures == response.contextElementResponseVector.size()) { parseDataP->upcrs.res.oe.fill(SccContextElementNotFound, ERROR_DESC_NOT_FOUND_ENTITY, ERROR_NOT_FOUND); } - else if (fails > 0) + else if (failures > 0) { // Removing trailing ", " failing = failing.substr(0, failing.size() - 2); - // If some CER (but not all) fails, then it is a partial update + // If some CER (but not all) fail, then it is a partial update parseDataP->upcrs.res.oe.fill(SccContextElementNotFound, "Attributes that were not updated: { " + failing + " }", "PartialUpdate"); } - else // fails == 0 + else // failures == 0 { // No failure, so invalidate any possible OrionError filled by mongoBackend on the mongoUpdateContext step parseDataP->upcrs.res.oe.fill(SccNone, ""); } - } else // v1 { + LM_T(LmtForward, ("ciP->apiVersion != V2")); // Note that v2 case doesn't use an actual response (so no need to waste time rendering it). // We render in the v1 case only TIMED_RENDER(answer = response.toJsonV1(asJsonObject)); diff --git a/src/lib/serviceRoutinesV2/patchEntity.cpp b/src/lib/serviceRoutinesV2/patchEntity.cpp index 8c505d4bc0..c7ef883a89 100644 --- a/src/lib/serviceRoutinesV2/patchEntity.cpp +++ b/src/lib/serviceRoutinesV2/patchEntity.cpp @@ -25,6 +25,9 @@ #include #include +#include "logMsg/logMsg.h" +#include "logMsg/traceLevels.h" + #include "common/statistics.h" #include "common/clockFunctions.h" #include "common/errorMessages.h" @@ -49,7 +52,7 @@ * Payload Out: None * * URI parameters: -* - +* type= * * 01. Fill in UpdateContextRequest * 02. Call standard op postUpdateContext @@ -86,7 +89,7 @@ std::string patchEntity postUpdateContext(ciP, components, compV, parseDataP); // 03. Check output from mongoBackend - any errors? - if (parseDataP->upcrs.res.oe.code != SccNone ) + if (parseDataP->upcrs.res.oe.code != SccNone) { TIMED_RENDER(answer = parseDataP->upcrs.res.oe.toJson()); ciP->httpStatusCode = parseDataP->upcrs.res.oe.code; @@ -94,6 +97,7 @@ std::string patchEntity else { ciP->httpStatusCode = SccNoContent; + answer = ""; } // 05. Cleanup and return result diff --git a/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_accumulator.test b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_accumulator.test new file mode 100644 index 0000000000..f3cd0d8a0c --- /dev/null +++ b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_accumulator.test @@ -0,0 +1,132 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Simple forwarded update in NGSIv2 + +--SHELL-INIT-- +dbInit CB +brokerStart CB 38,186-187,235 IPV4 +accumulatorStart --pretty-print --url /v2/entities/E1/attrs + +--SHELL-- + +# +# 01. Register entities of type T1, with ID .*, and attribute A1, for the accumulator +# 02. Update E1/T1 in CB, to provoke an Update Forward - see 404 Not Found as the accumulator doesn't respond +# 03. Dump the accumulator, see the forwarded request +# + +echo "01. Register entities of type T1, with ID .*, and attribute A1, for the accumulator" +echo "===================================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "T1", + "idPattern": ".*" + } + ], + "attrs": [ "A1" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${LISTENER_PORT}'/v2" + }, + "legacyForwarding": false + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "02. Update E1/T1 in CB, to provoke an Update Forward - see 404 Not Found as the accumulator doesn't respond" +echo "===========================================================================================================" +payload='{ + "A1": { + "value": 12 + } +}' +orionCurl --url /v2/entities/E1/attrs?type=T1 -X PATCH --payload "$payload" +echo +echo + + +echo "03. Dump the accumulator, see the forwarded request" +echo "===================================================" +accumulatorDump +accumulatorReset +echo +echo + + +--REGEXPECT-- +01. Register entities of type T1, with ID .*, and attribute A1, for the accumulator +=================================================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +02. Update E1/T1 in CB, to provoke an Update Forward - see 404 Not Found as the accumulator doesn't respond +=========================================================================================================== +HTTP/1.1 404 Not Found +Content-Length: 95 +Content-Type: application/json +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + +{ + "description": "The requested entity has not been found. Check type and id", + "error": "NotFound" +} + + +03. Dump the accumulator, see the forwarded request +=================================================== +PATCH http://localhost:REGEX(\d+)/v2/entities/E1/attrs?type=T1 +Fiware-Servicepath: / +Content-Length: 49 +User-Agent: REGEX(.*) +Host: localhost:REGEX(\d+) +Accept: application/json +Content-Type: application/json; charset=utf-8 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) + +{ + "A1": { + "metadata": {}, + "type": "Number", + "value": 12 + } +} +======================================= + + +--TEARDOWN-- +brokerStop CB +accumulatorStop +dbDrop CB diff --git a/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_one_provider.test b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_one_provider.test new file mode 100644 index 0000000000..16543ed944 --- /dev/null +++ b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_one_provider.test @@ -0,0 +1,152 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Forwarded update in pure NGSIv2 with one single provider + +--SHELL-INIT-- +dbInit CB +dbInit CP1 +brokerStart CB 38,186-187,235 IPV4 +brokerStart CP1 + +--SHELL-- + +# +# 01. Register entities of type T1, with ID .*, and attribute A1, for CP1 +# 02. Create E2/T1 with attribute A1='untouched' in CP1 +# 03. Update entities E2 of type T1, in CB, setting A1 to 'updated-3' +# 04. Query CP1, see E2/A1 == 'updated-3' +# + +echo "01. Register entities of type T1, with ID .*, and attribute A1, for CP1" +echo "=======================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "T1", + "idPattern": ".*" + } + ], + "attrs": [ "A1" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP1_PORT}'/v2" + }, + "legacyForwarding": false + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "02. Create E2/T1 with attribute A1='untouched' in CP1" +echo "=====================================================" +payload='{ + "id": "E2", + "type": "T1", + "A1": { + "value": "untouched" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP1_PORT +echo +echo + + +echo "03. Update entities E2 of type T1, in CB, setting A1 to 'updated-3'" +echo "===================================================================" +payload='{ + "A1": { + "value": "updated-3" + } +}' +orionCurl --url /v2/entities/E2/attrs?type=T1 -X PATCH --payload "$payload" +echo +echo + + +echo "04. Query CP1, see E2/A1 == 'updated-3'" +echo "=======================================" +orionCurl --url /v2/entities --port $CP1_PORT +echo +echo + + +--REGEXPECT-- +01. Register entities of type T1, with ID .*, and attribute A1, for CP1 +======================================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +02. Create E2/T1 with attribute A1='untouched' in CP1 +===================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/entities/E2?type=T1 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +03. Update entities E2 of type T1, in CB, setting A1 to 'updated-3' +=================================================================== +HTTP/1.1 204 No Content +Content-Length: 0 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +04. Query CP1, see E2/A1 == 'updated-3' +======================================= +HTTP/1.1 200 OK +Content-Length: 80 +Content-Type: application/json +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + +[ + { + "A1": { + "metadata": {}, + "type": "Text", + "value": "updated-3" + }, + "id": "E2", + "type": "T1" + } +] + + +--TEARDOWN-- +brokerStop CB +accumulatorStop +dbDrop CB diff --git a/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_providers.test b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_providers.test new file mode 100644 index 0000000000..d5f9b9bf32 --- /dev/null +++ b/test/functionalTest/cases/3068_ngsi_v2_update_forwarding/update_forwards_with_providers.test @@ -0,0 +1,415 @@ +# Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Forwarded updates in pure NGSIv2 with providers + +--SHELL-INIT-- +dbInit CB +dbInit CP1 +dbInit CP2 +dbInit CP3 +brokerStart CB 38,186-187,235 IPV4 +brokerStart CP1 +brokerStart CP2 +brokerStart CP3 + +--SHELL-- + +# +# 01. Register entities of type T1, with ID .*, and attribute A1, for CP1 +# 02. Register entities of type T1, with ID .*, and attribute A2, for CP2 +# 03. Register entities of type T1, with ID .*, and attribute A3+A4, for CP3 +# 04. Create E4/T1 with attribute A1='untouched' in CP1 +# 05. Create E5/T1 with attribute A2='untouched' in CP2 +# 06. Create E6/T1 with attribute A3='untouched' in CP3 +# 07. Create E7/T1 with attribute A4='untouched' in CP3 +# 08. Update entities E4 of type T1, setting A1 to 'updated-4' +# 09. Update entities E5 of type T1, setting A2 to 'updated-5' +# 10. Update entities E6 of type T1, setting A3 to 'updated-6' +# 11. Query CP1, see E4/A1 == 'updated-4' +# 12. Query CP2, see E5/A2 == 'updated-5' +# 13. Query CP3, see E6/A3 == 'updated-6' and E7/A4 == 'untouched' +# +# NOTE +# The updates of step 8-10 are done in the CB, that then forwards the update requests +# + +echo "01. Register entities of type T1, with ID .*, and attribute A1, for CP1" +echo "=======================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "T1", + "idPattern": ".*" + } + ], + "attrs": [ "A1" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP1_PORT}'/v2" + }, + "legacyForwarding": false + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "02. Register entities of type T1, with ID .*, and attribute A2, for CP2" +echo "=======================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "T1", + "idPattern": ".*" + } + ], + "attrs": [ "A2" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP2_PORT}'/v2" + }, + "legacyForwarding": false + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "03. Register entities of type T1, with ID .*, and attribute A3, for CP3" +echo "=======================================================================" +payload='{ + "dataProvided": { + "entities": [ + { + "type": "T1", + "idPattern": ".*" + } + ], + "attrs": [ "A3", "A4" ] + }, + "provider": { + "http": { + "url": "http://localhost:'${CP3_PORT}'/v2" + }, + "legacyForwarding": false + } +}' +orionCurl --url /v2/registrations --payload "$payload" +echo +echo + + +echo "04. Create E4/T1 with attribute A1='untouched' in CP1" +echo "=====================================================" +payload='{ + "id": "E4", + "type": "T1", + "A1": { + "value": "untouched" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP1_PORT +echo +echo + + +echo "05. Create E5/T1 with attribute A2='untouched' in CP2" +echo "=====================================================" +payload='{ + "id": "E5", + "type": "T1", + "A2": { + "value": "untouched" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP2_PORT +echo +echo + + +echo "06. Create E6/T1 with attribute A3='untouched' in CP3" +echo "=====================================================" +payload='{ + "id": "E6", + "type": "T1", + "A3": { + "value": "untouched" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP3_PORT +echo +echo + + +echo "07. Create E7/T1 with attribute A4='untouched' in CP3" +echo "=====================================================" +payload='{ + "id": "E7", + "type": "T1", + "A4": { + "value": "untouched" + } +}' +orionCurl --url /v2/entities --payload "$payload" --port $CP3_PORT +echo +echo + + +echo "08. Update entities E4 of type T1, setting A1 to 'updated-4'" +echo "============================================================" +payload='{ + "A1": { + "value": "updated-4" + } +}' +orionCurl --url /v2/entities/E4/attrs?type=T1 -X PATCH --payload "$payload" +echo +echo + + +echo "09. Update entities E5 of type T1, setting A2 to 'updated-5'" +echo "============================================================" +payload='{ + "A2": { + "value": "updated-5" + } +}' +orionCurl --url /v2/entities/E5/attrs?type=T1 -X PATCH --payload "$payload" +echo +echo + + +echo "10. Update entities E6 of type T1, setting A3 to 'updated-6'" +echo "============================================================" +payload='{ + "A3": { + "value": "updated-6" + } +}' +orionCurl --url /v2/entities/E6/attrs?type=T1 -X PATCH --payload "$payload" +echo +echo + + +echo "11. Query CP1, see E4/A1 == 'updated-4'" +echo "=======================================" +orionCurl --url /v2/entities --port $CP1_PORT +echo +echo + + +echo "12. Query CP2, see E5/A2 == 'updated-5'" +echo "=======================================" +orionCurl --url /v2/entities --port $CP2_PORT +echo +echo + + +echo "13. Query CP3, see E6/A3 == 'updated-6' and E7/A4 == 'untouched'" +echo "================================================================" +orionCurl --url /v2/entities --port $CP3_PORT +echo +echo + + +--REGEXPECT-- +01. Register entities of type T1, with ID .*, and attribute A1, for CP1 +======================================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +02. Register entities of type T1, with ID .*, and attribute A2, for CP2 +======================================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +03. Register entities of type T1, with ID .*, and attribute A3, for CP3 +======================================================================= +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/registrations/REGEX([0-9a-f\-]{24}) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +04. Create E4/T1 with attribute A1='untouched' in CP1 +===================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/entities/E4?type=T1 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +05. Create E5/T1 with attribute A2='untouched' in CP2 +===================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/entities/E5?type=T1 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +06. Create E6/T1 with attribute A3='untouched' in CP3 +===================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/entities/E6?type=T1 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +07. Create E7/T1 with attribute A4='untouched' in CP3 +===================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Location: /v2/entities/E7?type=T1 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +08. Update entities E4 of type T1, setting A1 to 'updated-4' +============================================================ +HTTP/1.1 204 No Content +Content-Length: 0 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +09. Update entities E5 of type T1, setting A2 to 'updated-5' +============================================================ +HTTP/1.1 204 No Content +Content-Length: 0 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +10. Update entities E6 of type T1, setting A3 to 'updated-6' +============================================================ +HTTP/1.1 204 No Content +Content-Length: 0 +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + + + +11. Query CP1, see E4/A1 == 'updated-4' +======================================= +HTTP/1.1 200 OK +Content-Length: 80 +Content-Type: application/json +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + +[ + { + "A1": { + "metadata": {}, + "type": "Text", + "value": "updated-4" + }, + "id": "E4", + "type": "T1" + } +] + + +12. Query CP2, see E5/A2 == 'updated-5' +======================================= +HTTP/1.1 200 OK +Content-Length: 80 +Content-Type: application/json +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + +[ + { + "A2": { + "metadata": {}, + "type": "Text", + "value": "updated-5" + }, + "id": "E5", + "type": "T1" + } +] + + +13. Query CP3, see E6/A3 == 'updated-6' and E7/A4 == 'untouched' +================================================================ +HTTP/1.1 200 OK +Content-Length: 159 +Content-Type: application/json +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Date: REGEX(.*) + +[ + { + "A3": { + "metadata": {}, + "type": "Text", + "value": "updated-6" + }, + "id": "E6", + "type": "T1" + }, + { + "A4": { + "metadata": {}, + "type": "Text", + "value": "untouched" + }, + "id": "E7", + "type": "T1" + } +] + + +--TEARDOWN-- +brokerStop CB +accumulatorStop +dbDrop CB diff --git a/test/functionalTest/harnessFunctions.sh b/test/functionalTest/harnessFunctions.sh index 273851d57d..8541346964 100644 --- a/test/functionalTest/harnessFunctions.sh +++ b/test/functionalTest/harnessFunctions.sh @@ -715,6 +715,14 @@ function accumulatorStart() shift fi + url="/notify" + if [ "$1" = "--url" ] + then + url="$2" + shift + shift + fi + bindIp=$1 port=$2 @@ -731,7 +739,7 @@ function accumulatorStart() accumulatorStop $port - accumulator-server.py --port $port --url /notify --host $bindIp $pretty $https $key $cert > /tmp/accumulator_${port}_stdout 2> /tmp/accumulator_${port}_stderr & + accumulator-server.py --port $port --url $url --host $bindIp $pretty $https $key $cert > /tmp/accumulator_${port}_stdout 2> /tmp/accumulator_${port}_stderr & echo accumulator running as PID $$ # Wait until accumulator has started or we have waited a given maximum time diff --git a/test/unittests/mongoBackend/mongoDiscoverContextAvailability_test.cpp b/test/unittests/mongoBackend/mongoDiscoverContextAvailability_test.cpp index 4e019ad5fb..382ae8ac9b 100644 --- a/test/unittests/mongoBackend/mongoDiscoverContextAvailability_test.cpp +++ b/test/unittests/mongoBackend/mongoDiscoverContextAvailability_test.cpp @@ -2257,12 +2257,13 @@ TEST(mongoDiscoverContextAvailabilityRequest, mongoDbQueryFail) EXPECT_EQ("Internal Server Error", res.errorCode.reasonPhrase); EXPECT_EQ("Database Error (collection: utest.registrations " - "- query(): { query: { $or: [ { contextRegistration.entities: " - "{ $in: [ { id: \"E3\", type: \"T3\" }, { type: \"T3\", id: \"E3\" } ] } }, " - "{ contextRegistration.entities.id: { $in: [] } } ], " - "expiration: { $gt: 1360232700 }" - ", servicePath: { $in: [ /^/.*/, null ] } }" - ", orderby: { _id: 1 } } - exception: boom!!)", res.errorCode.details); + "- query(): { query: " + "{ $or: [ { contextRegistration.entities.isPattern: \"true\", contextRegistration.entities.id: \".*\" }, " + "{ contextRegistration.entities.id: \"E3\" } ], " + "contextRegistration.entities.type: \"T3\", " + "servicePath: { $in: [ /^/.*/, null ] }, " + "expiration: { $gt: 1360232700 } }, " + "orderby: { _id: 1 } } - exception: boom!!)", res.errorCode.details); EXPECT_EQ(0, res.responseVector.size()); /* Restore real DB connection */ From 65024547a91f8f689b7b168bc03eaa64a1af0f67 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Wed, 24 Apr 2019 13:41:35 +0200 Subject: [PATCH 2/3] Pre-PR fixes --- src/lib/mongoBackend/MongoGlobal.cpp | 15 +++++++-------- src/lib/serviceRoutines/postUpdateContext.cpp | 7 +++---- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp index c50c91f214..3a8540db5f 100644 --- a/src/lib/mongoBackend/MongoGlobal.cpp +++ b/src/lib/mongoBackend/MongoGlobal.cpp @@ -2036,15 +2036,14 @@ bool registrationsQuery mongo::BSONArrayBuilder entityOr; // - // If only one entity in the REST request, the query to mongo can be simplified + // If there's only one entity in the REST request, the query to mongo can be simplified. // NGSIv2 requests like PATCH /v2/entities//attrs can have only ONE entity // and will ALWAYS enter the "then part" of the if-then-else. // - // Note than registrations with entity id as a pattern is treated by the "then part" but not - // by the older "else part". - // - // One NGSIv1 is removed from the source code, the "else part" can be removed by just making sure that - // NGSIv2 batch operations call this function once per entity in the request. + // FIXME: Note than registrations with entity id as a pattern is treated by the "then part" but not by the older "else part". + // When forwarding for batch update is implemented this needs to be fixed, somehow + // Once NGSIv1 is removed from the source code, the "else part" can be removed by just making sure that + // NGSIv2 batch operations call this function once per entity in the request (modifications needed). // Or, we'd need to reimplement to make the else-part also support registrations with entity id as a pattern. // if (enV.size() == 1) @@ -2064,11 +2063,11 @@ bool registrationsQuery // If the Registration is with idPattern, then it matches, as the only idPattern allowed for Registrations is ".*" entityAnd.append("contextRegistration.entities.isPattern", "true"); - entityAnd.append("contextRegistration.entities.id", ".*"); // TPUT: if isPattern == true, then id MUST be ".*" ! Remove? + entityAnd.append("contextRegistration.entities.id", ".*"); // FIXME TPUT: if isPattern == true, then id MUST be ".*" ! Remove? entityOr.append(entityAnd.obj()); if (enV[0]->isPattern == "true") id.appendRegex("contextRegistration.entities.id", enV[0]->id); - else id.append("contextRegistration.entities.id", enV[0]->id); + else id.append("contextRegistration.entities.id", enV[0]->id); entityOr.append(id.obj()); queryBuilder.append("$or", entityOr.arr()); diff --git a/src/lib/serviceRoutines/postUpdateContext.cpp b/src/lib/serviceRoutines/postUpdateContext.cpp index 838d5b3279..93fc90baed 100644 --- a/src/lib/serviceRoutines/postUpdateContext.cpp +++ b/src/lib/serviceRoutines/postUpdateContext.cpp @@ -213,7 +213,7 @@ static bool updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda LM_T(LmtCPrForwardResponsePayload, ("forward updateContext response payload: %s", out.c_str())); // - // If NGSIv1: + // If NGSIv1 (providerFormat == PfJson): // 4. Parse the response and fill in a binary UpdateContextResponse // 5. Fill in the response from the redirection into the response of this function // 6. 'Fix' StatusCode @@ -298,10 +298,9 @@ static bool updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda parseData.upcr.res.release(); parseData.upcrs.res.release(); - LM_T(LmtForward, ("V1 Forward OK")); return true; } - else // V2 + else // NGSIv2 { LM_T(LmtForward, ("upcrP->providerFormat == V2. out: '%s'", out.c_str())); // NGSIv2 forward - no payload to be received, just a 204 No Content HTTP Header @@ -312,7 +311,7 @@ static bool updateForward(ConnectionInfo* ciP, UpdateContextRequest* upcrP, Upda return true; } - LM_W(("Other Error (not the expected response from content provider: %s)", out.c_str())); + LM_W(("Other Error (unexpected response from context provider: %s)", out.c_str())); upcrsP->errorCode.fill(SccReceiverInternalError); return false; } From 5dfea5300b0f3e86aa334a839b3c72a874c9679a Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Wed, 24 Apr 2019 16:19:57 +0200 Subject: [PATCH 3/3] documentation --- doc/manuals/devel/cprs.md | 6 +++--- doc/manuals/user/ngsiv2_implementation_notes.md | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manuals/devel/cprs.md b/doc/manuals/devel/cprs.md index 290833f915..a1cde59eeb 100644 --- a/doc/manuals/devel/cprs.md +++ b/doc/manuals/devel/cprs.md @@ -16,7 +16,7 @@ In NGSIv1 (deprecated), the request `POST /v1/updateContext` has a field called * APPEND_STRICT * REPLACE -> Side-node: The first three are "standard NGSIv1" while the second two were added for NGSIv2. +> Side-note: The first three are "standard NGSIv1" while the second two were added for NGSIv2. * Requests with `UPDATE` or `REPLACE` may provoke forwarding of the request. Only if **not found locally but found in a registration**. @@ -89,9 +89,9 @@ The `QueryContextRequest` items are filled in based on the output of the [**mong _FW-04: `queryForward()` function detail_ * Parse the context provider string to extract IP, port, URI path, etc. (step 1). -* The request to forward has to be built (step 2). In the case of NGSIv1, we need to extract information of the binary object into text to be able to send the REST request (plain text) to the Context Provider using POST /v1/queryContext. In the case of NGSIv2, the use GET /v2/entities and no payload is required. +* The request to forward has to be built (step 2). In the case of NGSIv1, we need to extract information of the binary object into text to be able to send the REST request (plain text) to the Context Provider using POST /v1/queryContext. In the case of NGSIv2, the use of GET /v2/entities (without payload) is required. * The request to forward is sent with the help of `httpRequestSend()` (step 3) which uses [libcurl](https://curl.haxx.se/libcurl/) to forward the request (step 4). libcurl sends in sequence the request to the Context Provider (step 5). -* The textual response from the Context Provider is parsed and an `QueryContextResponse` object is created (step 6). Parsing details are provided in diagram [PP-01](jsonParse.md#flow-pp-01). +* The textual response from the Context Provider is parsed and a `QueryContextResponse` object is created (step 6). Parsing details are provided in diagram [PP-01](jsonParse.md#flow-pp-01). ## A Caveat about shadowing of entities The Context Provider mechanism is implemented using standard registration requests and this might lead to unwanted situations. diff --git a/doc/manuals/user/ngsiv2_implementation_notes.md b/doc/manuals/user/ngsiv2_implementation_notes.md index 50ec9adc64..c8099dc13f 100644 --- a/doc/manuals/user/ngsiv2_implementation_notes.md +++ b/doc/manuals/user/ngsiv2_implementation_notes.md @@ -335,8 +335,9 @@ for the following aspects: directly. I.e., updates must be done deleting and re-creating the registration. Please see [this issue](https://github.com/telefonicaid/fiware-orion/issues/3007) about this. * `idPattern` is supported but only for the exact regular expression `.*` - And, right now (2019-03-21), only forwarding of queries is working for registrations with idPatterns. - forwarded updates do not work for registrations with idPattern (the plan is to fix this asap). + And, right now (2019-04-24), forwarding for registrations with idPatterns works for queries using `GET /v2/entities` + as well as updates using `PATCH /v2/entities/{Entity-ID}/attrs`. + More requests will be added to this list. * `typePattern` is not implemented. * The only valid `supportedForwardingMode` is `all`. Trying to use any other value will end in a 501 Not Implemented error response. Please