Unify json serialization by using rapidjson

CHANGES_NEXT_RELEASE

@@ -1,5 +1,6 @@
 - Add: self-notification loop protection, based on Fiware-Correlator and Ngsiv2-AttrsFormat headers and lastCorrelator field at DB (#2937)
 - Add: Fiware-Correlator and NgsiV2-AttrsFormat headers cannot be overwritten by the custom notification logic (#2937)
+- Hardening: Improve JSON serialization by using rapidjson instead of using custom code
 - Hardening: Mongo driver migrated to legacy-1.1.2 (several bugfixes in the legacy-1.0.7 to legacy-1.1.2 delta)
 - Hardening: Several changes in argument passing in mongoBackend library to avoid passing entire objects on the stack, from "X x" to "const X& x"
 - Fix: several invalid memory accesses

scripts/style_check_in_makefile.sh

@@ -106,7 +106,6 @@ style_check test/unittests/serviceRoutines
 # o common/string.h                                                          (haderding/remove_ngsiv1_indent: 1 include, 1 external declaration)
 # o common/globals.h                                                         (feature/mqtt_notifications_poc: 1 external declaration)
 # o common/globals.h                                                         (haderding/remove_ngsiv1_indent: 1 new external declaration)
-# o common/tag.cpp                                                           (haderding/remove_ngsiv1_indent: 'indent' removed from 6 signatures, slight change in 6 functions)
 # o common/tag.h                                                             (haderding/remove_ngsiv1_indent: 'indent' removed from 6 signatures)
 # o common/macroSubstitute.cpp                                               (haderding/remove_ngsiv1_indent: 2 lines: a param added to toJson())
 #
@@ -349,8 +348,6 @@ style_check test/unittests/serviceRoutines
 #
 # o unittests/main_UnitTest.cpp                                              (haderding/remove_ngsiv1_indent: 1 variable)
 #
-# o unittests/common/commonTag_test.cpp                                      (haderding/remove_ngsiv1_indent: 24 lineas)
-#
 # o unittests/convenience/AppendContextElementRequest_test.cpp               (haderding/remove_ngsiv1_indent: 1 render(), 4 check())
 # o unittests/convenience/AppendContextElementResponse_test.cpp              (haderding/remove_ngsiv1_indent: 2 render(), 3 check())
 # o unittests/convenience/ContextAttributeResponseVector_test.cpp            (haderding/remove_ngsiv1_indent: 2 render(), 3 check())

src/lib/apiTypesV2/Attribute.cpp

@@ -25,7 +25,6 @@
 #include <string>
 #include <vector>
 
-#include "common/tag.h"
 #include "common/errorMessages.h"
 #include "common/RenderFormat.h"
 #include "common/string.h"
@@ -40,65 +39,55 @@
 */
 std::string Attribute::render
 (
-  ApiVersion          apiVersion,          // in parameter (pass-through)
-  bool                acceptedTextPlain,   // in parameter (pass-through)
-  bool                acceptedJson,        // in parameter (pass-through)
-  MimeType            outFormatSelection,  // in parameter (pass-through)
-  MimeType*           outMimeTypeP,        // out parameter (pass-through)
-  HttpStatusCode*     scP,                 // out parameter (pass-through)
-  bool                keyValues,           // in parameter
-  const std::string&  metadataList,        // in parameter
-  RequestType         requestType,         // in parameter
-  bool                comma                // in parameter
+  ApiVersion          apiVersion,
+  MimeType            outFormatSelection,
+  HttpStatusCode*     scP,
+  bool                keyValues,
+  const std::string&  metadataList,
+  RequestType         requestType,
+  int                 indent
 )
 {
   RenderFormat  renderFormat = (keyValues == true)? NGSI_V2_KEYVALUES : NGSI_V2_NORMALIZED;
 
   if (pcontextAttribute)
   {
-    std::string out;
-
     if (requestType == EntityAttributeValueRequest)
     {
-      out = pcontextAttribute->toJsonAsValue(apiVersion,
-                                             acceptedTextPlain,
-                                             acceptedJson,
-                                             outFormatSelection,
-                                             outMimeTypeP,
-                                             scP);
+      return pcontextAttribute->renderAsValue(apiVersion,
+                                              outFormatSelection,
+                                              scP,
+                                              indent);
     }
     else
     {
+      if (indent < 0)
+      {
+        indent = DEFAULT_JSON_INDENT;
+      }
+      JsonHelper writer(indent);
+
       std::vector<std::string> metadataFilter;
 
       if (metadataList != "")
       {
         stringSplit(metadataList, ',', metadataFilter);
       }
 
-      out = "{";
-
-      // First parameter (isLastElement) is 'true' as it is the last and only element
-      out += pcontextAttribute->toJson(true, renderFormat, metadataFilter, requestType);
+      writer.StartObject();
 
-      out += "}";
-    }
+      pcontextAttribute->toJson(writer, renderFormat, metadataFilter, requestType);
 
+      writer.EndObject();
 
-    if (comma)
-    {
-      out += ",";
+      return writer.str();
     }
-
-    return out;
   }
 
-  return oe.toJson();
+  return oe.renderV1();
 }
 
 
-
-
 /* ****************************************************************************
 *
 * Attribute::fill -

src/lib/apiTypesV2/Attribute.h

@@ -29,6 +29,9 @@
 #include <vector>
 #include <cstdlib>
 
+#include "rapidjson/writer.h"
+#include "rapidjson/stringbuffer.h"
+
 #include "ngsi/ContextAttributeVector.h"
 #include "ngsi/ContextAttribute.h"
 #include "rest/OrionError.h"
@@ -55,15 +58,12 @@ class Attribute
 
   Attribute(): pcontextAttribute(0) {}
   std::string  render(ApiVersion          apiVersion,
-                      bool                acceptedTextPlain,
-                      bool                acceptedJson,
                       MimeType            outFormatSelection,
-                      MimeType*           outMimeTypeP,
                       HttpStatusCode*     scP,
                       bool                keyValues,
                       const std::string&  metadataList,
                       RequestType         requestType,
-                      bool                comma = false);
+                      int                 indent = -1);
   void         fill(QueryContextResponse* qcrsP, std::string attrName);
 };
 

src/lib/apiTypesV2/Entities.cpp

@@ -63,13 +63,34 @@ Entities::~Entities()
 std::string Entities::render
 (
   std::map<std::string, bool>&         uriParamOptions,
-  std::map<std::string, std::string>&  uriParam
+  std::map<std::string, std::string>&  uriParam,
+  int                                  indent
 )
 {
-  return vec.render(uriParamOptions, uriParam);
+  if (indent < 0)
+  {
+    indent = DEFAULT_JSON_INDENT;
+  }
+  JsonHelper writer(indent);
+  toJson(writer, uriParamOptions, uriParam);
+  return writer.str();
 }
 
 
+/* ****************************************************************************
+*
+* Entities::toJson -
+*
+*/
+void Entities::toJson
+(
+  JsonHelper&                          writer,
+  std::map<std::string, bool>&         uriParamOptions,
+  std::map<std::string, std::string>&  uriParam
+)
+{
+  return vec.toJson(writer, uriParamOptions, uriParam);
+}
 
 /* ****************************************************************************
 *

src/lib/apiTypesV2/Entities.h

@@ -29,6 +29,8 @@
 #include <vector>
 #include <map>
 
+#include "common/JsonHelper.h"
+
 #include "apiTypesV2/EntityVector.h"
 #include "rest/OrionError.h"
 
@@ -55,8 +57,12 @@ class Entities
   Entities();
   ~Entities();
 
-  std::string  render(std::map<std::string, bool>&         uriParamOptions,
+  void         toJson(JsonHelper&                          writer,
+                      std::map<std::string, bool>&         uriParamOptions,
                       std::map<std::string, std::string>&  uriParam);
+  std::string  render(std::map<std::string, bool>&         uriParamOptions,
+                      std::map<std::string, std::string>&  uriParam,
+                      int                                  indent = -1);
 
   std::string  check(ApiVersion apiVersion, RequestType requestType);
   void         present(const std::string& indent);

src/lib/apiTypesV2/Entity.cpp

@@ -28,7 +28,6 @@
 
 #include "logMsg/traceLevels.h"
 #include "logMsg/logMsg.h"
-#include "common/tag.h"
 #include "common/string.h"
 #include "common/globals.h"
 #include "common/errorMessages.h"
@@ -76,12 +75,30 @@ std::string Entity::render
 (
   std::map<std::string, bool>&         uriParamOptions,
   std::map<std::string, std::string>&  uriParam,
-  bool                                 comma
+  int                                  indent
+)
+{
+  JsonHelper writer(indent);
+  toJson(writer, uriParamOptions, uriParam);
+  return writer.str();
+}
+
+
+/* ****************************************************************************
+*
+* Entity::toJson -
+*
+*/
+void Entity::toJson
+(
+  JsonHelper&                          writer,
+  std::map<std::string, bool>&         uriParamOptions,
+  std::map<std::string, std::string>&  uriParam
 )
 {
   if ((oe.details != "") || ((oe.reasonPhrase != "OK") && (oe.reasonPhrase != "")))
   {
-    return oe.toJson();
+    oe.toJson(writer);
   }
 
   RenderFormat  renderFormat = NGSI_V2_NORMALIZED;
@@ -90,7 +107,6 @@ std::string Entity::render
   else if (uriParamOptions[OPT_VALUES]        == true)  { renderFormat = NGSI_V2_VALUES;        }
   else if (uriParamOptions[OPT_UNIQUE_VALUES] == true)  { renderFormat = NGSI_V2_UNIQUE_VALUES; }
 
-  std::string               out;
   std::vector<std::string>  metadataFilter;
   std::vector<std::string>  attrsFilter;
 
@@ -118,57 +134,40 @@ std::string Entity::render
 
   if ((renderFormat == NGSI_V2_VALUES) || (renderFormat == NGSI_V2_UNIQUE_VALUES))
   {
-    out = "[";
+    writer.StartArray();
     if (attributeVector.size() != 0)
     {
-      out += attributeVector.toJson(renderFormat, attrsFilter, metadataFilter, false);
+      attributeVector.toJson(writer, renderFormat, attrsFilter, metadataFilter, false);
     }
-    out += "]";
+    writer.EndArray();
   }
   else
   {
-    out = "{";
+    writer.StartObject();
 
     if (renderId)
     {
-      out += JSON_VALUE("id", id);
-      out += ",";
+      writer.String("id", id);
 
+      writer.Key("type");
       /* This is needed for entities coming from NGSIv1 (which allows empty or missing types) */
-      out += JSON_STR("type") + ":" + ((type != "")? JSON_STR(type) : JSON_STR(DEFAULT_ENTITY_TYPE));
-    }
-
-    std::string attrsOut;
-    if (attributeVector.size() != 0)
-    {
-      attrsOut += attributeVector.toJson(renderFormat, attrsFilter, metadataFilter, false);
-    }
-
-    //
-    // Note that just attributeVector.size() != 0 (used in previous versions) cannot be used
-    // as ciP->uriParam["attrs"] filter could remove all the attributes
-    //
-    if (attrsOut != "")
-    {
-      if (renderId)
+      if (type != "")
       {
-        out +=  "," + attrsOut;
+        writer.String(type);
       }
       else
       {
-        out += attrsOut;
+        writer.String(DEFAULT_ENTITY_TYPE);
       }
     }
 
-    out += "}";
-  }
+    if (attributeVector.size() != 0)
+    {
+      attributeVector.toJson(writer, renderFormat, attrsFilter, metadataFilter, false);
+    }
 
-  if (comma)
-  {
-    out += ",";
+    writer.EndObject();
   }
-
-  return out;
 }
 
 

src/lib/apiTypesV2/Entity.h

@@ -29,6 +29,8 @@
 #include <vector>
 #include <map>
 
+#include "common/JsonHelper.h"
+
 #include "ngsi/ContextAttributeVector.h"
 #include "rest/OrionError.h"
 
@@ -66,9 +68,12 @@ class Entity
   Entity();
   ~Entity();
 
+  void         toJson(JsonHelper&                          writer,
+                      std::map<std::string, bool>&         uriParamOptions,
+                      std::map<std::string, std::string>&  uriParam);
   std::string  render(std::map<std::string, bool>&         uriParamOptions,
                       std::map<std::string, std::string>&  uriParam,
-                      bool                                 comma = false);
+                      int                                  indent = -1);
 
   std::string  check(ApiVersion apiVersion, RequestType requestType);
   void         present(const std::string& indent);