Skip to content

Commit

Permalink
json: Escaping and unescaping special characters in string (fixes #22)
Browse files Browse the repository at this point in the history 
automated json tests
  • Loading branch information
@loentar
loentar committed Feb 20, 2017
1 parent 37de177 commit 7cbbb98
Show file tree
Hide file tree
Showing 3 changed files with 221 additions and 17 deletions.
93 changes: 91 additions & 2 deletions core/json/src/JsonReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ class JsonReaderImpl {
return start;
}

#ifdef NGREST_JSON_NO_QUOTE_STRING
inline char* tokenString() {
++curr; // skip '"'
char* start = curr;
Expand All @@ -99,6 +100,94 @@ class JsonReaderImpl {
++curr;
return start;
}
#else

inline char fromHexChar(short hex)
{
static const char table[256] = {
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x00
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x10
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x20
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x30
0x10, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x40
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x50
0x10, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x60
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x70
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x80
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // x90
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // xA0
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // xB0
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // xC0
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // xD0
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // xE0
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10 // xF0
};
NGREST_ASSERT(table[hex] != 0x10, "Unexpected character while reading unicode hex digit");
return table[hex];
}

inline char* tokenString() {
++curr; // skip '"'
char* start = curr;
char* out = nullptr;
while (*curr != '"') {
if (*curr == '\\') { // parse quoted characters
if (!out) {
out = curr;
}
++curr;
switch (*curr) {
case 'b':
*out = '\b';
break;
case 'f':
*out = '\f';
break;
case 'n':
*out = '\n';
break;
case 'r':
*out = '\r';
break;
case 't':
*out = '\t';
break;
case 'u':
++curr;
*out = fromHexChar(*curr) << 4;
++curr;
*out |= fromHexChar(*curr);
if (*out) { // \u00XX
++out;
}
++curr;
*out = fromHexChar(*curr) << 4;
++curr;
*out |= fromHexChar(*curr);
break;
default:
*out = *curr;
break;
}
++out;
} else {
NGREST_ASSERT(*curr >= 0x20, "Unexpected control character while reading string");
if (out) {
*out = *curr;
++out;
}
}
++curr;
}
if (out) {
*out = '\0';
} else {
*curr = '\0';
}
++curr;
return start;
}
#endif


inline JsonReaderImpl(char* buff, MemPool* memPool):
Expand Down Expand Up @@ -141,7 +230,7 @@ class JsonReaderImpl {
if (!strncmp(token, "NaN", len))
return pool->alloc<Value>(ValueType::NaN);

NGREST_THROW_ASSERT("Unexpected token");
NGREST_THROW_ASSERT(std::string("Unexpected token: [") + token + "]");
}

inline Node* readArrayOrObject()
Expand All @@ -155,7 +244,7 @@ class JsonReaderImpl {
return readObject();
}

NGREST_THROW_ASSERT("Unexpected symbol");
NGREST_THROW_ASSERT(std::string("Unexpected symbol: [") + *curr + "]");
}

inline Array* readArray()
Expand Down
75 changes: 69 additions & 6 deletions core/json/src/JsonWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,73 @@ class JsonWriterImpl {
{
}

inline char toHexChar(short dec)
{
static char hexTable[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
return hexTable[dec];
}

inline void putString(const char* str)
{
pool->putChar('"');
#ifdef NGREST_JSON_NO_QUOTE_STRING
pool->putCString(str);
#else
size_t len = strlen(str);
pool->reserve(len);
const char* start = str;
const char* curr = str;
for (; *curr; ++curr) {
if (*curr < 0x20) {
if (start) {
if (curr > start) {
pool->putData(start, curr - start);
}
}
switch (*curr) {
case '\b':
pool->putData("\\b", 2);
break;
case '\f':
pool->putData("\\f", 2);
break;
case '\n':
pool->putData("\\n", 2);
break;
case '\r':
pool->putData("\\r", 2);
break;
case '\t':
pool->putData("\\t", 2);
break;
default:
pool->putData("\\u00", 4);
pool->putChar(toHexChar((*curr >> 4) & 0x0f));
pool->putChar(toHexChar(*curr & 0x0f));
break;
}
start = curr + 1;
} else {
if (*curr == '"' || *curr == '\\') {
if (start) {
if (curr > start) {
pool->putData(start, curr - start);
}
start = curr + 1;
}

pool->putChar('\\');
pool->putChar(*curr);
}
}
}
if (start && curr > start) {
pool->putData(start, curr - start);
}
#endif
pool->putChar('"');
}

inline void writeNode(const Node* node)
{
if (!node) {
Expand All @@ -56,9 +123,7 @@ class JsonWriterImpl {
for (const NamedNode* child = object->firstChild; child; child = child->nextSibling) {
if (child != object->firstChild)
pool->putChar(',');
pool->putChar('"');
pool->putCString(child->name);
pool->putChar('"');
putString(child->name);
pool->putChar(':');
writeNode(child->node);
}
Expand Down Expand Up @@ -86,9 +151,7 @@ class JsonWriterImpl {
break;

case ValueType::String:
pool->putChar('"');
pool->putCString(value->value);
pool->putChar('"');
putString(value->value);
break;

case ValueType::Number:
Expand Down
70 changes: 61 additions & 9 deletions tests/json/src/jsontest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ int main()
const int testsCount = 8;
char test1[] = "{}";
char test2[] = "[]";
char test3[] = "{\"abc\": 1}";
char test3[] = "{\"ab c\": 1}";
char test4[] = "{ \"q\": [ ] }";
char test5[] = "{\"x\": {\"abc\": 1}}";
char test6[] = "{ \"x\" : {\"abc\": 1 }, \"y\": [1, 2e2, \"3\", null , NaN]}";
Expand All @@ -55,26 +55,78 @@ int main()
test8
};

const char* testsOut[testsCount] = {
"{}",
"[]",
"{\"ab c\":1}",
"{\"q\":[]}",
"{\"x\":{\"abc\":1}}",
"{\"x\":{\"abc\":1},\"y\":[1,2e2,\"3\",null,NaN]}",
"[{},{\"\":\"\"}]",
"[[[],[]],{\"1\":[]}]"
};

for (int t = 0; t < testsCount; ++t) {
std::cout << "Basic test: #" << t << std::endl;
ngrest::MemPool pool;
ngrest::MemPool poolOut;
ngrest::Node* root = ngrest::json::JsonReader::read(tests[t], &pool);

ngrest::json::JsonWriter::write(root, &poolOut, 2);

int chunksCount = poolOut.getChunkCount();
const ngrest::MemPool::Chunk* chunks = poolOut.getChunks();
printf("test #%d \n----------\n", t + 1);
for (int i = 0; i < chunksCount; ++i) {
const ngrest::MemPool::Chunk* chunk = chunks + i;
fwrite(chunk->buffer, chunk->size, 1, stdout);
}
printf("\n----------\n");
const ngrest::MemPool::Chunk* chunk = poolOut.flatten();
NGREST_ASSERT(!strcmp(chunk->buffer, testsOut[t]), std::string("Basic Test failed. Expected [")
+ testsOut[t] + "] found [" + chunk->buffer + "].")
}

} catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
return 1;
}



// quoting test
try {
const char* json = R"(["no special chars","\b\f\n\r\t\"\\\/\u0008\u000a","__\n\r__","\t__","__\t"])";
const char* values[] = {
"no special chars", "\b\f\n\r\t\"\\/\x08\x0a", "__\n\r__", "\t__", "__\t"
};
const char* jsonOut = R"(["no special chars","\b\f\n\r\t\"\\/\b\n","__\n\r__","\t__","__\t"])";
ngrest::MemPool poolIn;

// test reader
char* jsonIn = poolIn.putCString(json, true);
const ngrest::Node* root = ngrest::json::JsonReader::read(jsonIn, &poolIn);
NGREST_ASSERT(root->type == ngrest::NodeType::Array, "Read node is not array");
const ngrest::Array* arr = static_cast<const ngrest::Array*>(root);
int index = 0;
for (const ngrest::LinkedNode* node = arr->firstChild; node; node = node->nextSibling, ++index) {
std::cout << "Reader test: #" << index << std::endl;
NGREST_ASSERT_NULL(node->node);
NGREST_ASSERT(node->node->type == ngrest::NodeType::Value, "Child is not Value");
const ngrest::Value* value = static_cast<const ngrest::Value*>(node->node);
NGREST_ASSERT_NULL(value->value);
NGREST_ASSERT(!strcmp(value->value, values[index]), std::string("Reader Test failed. Expected [")
+ values[index] + "] found [" + value->value + "].")
}


std::cout << "Writer test" << std::endl;
// test writer
ngrest::MemPool poolOut;
ngrest::json::JsonWriter::write(root, &poolOut);
ngrest::MemPool::Chunk* res = poolOut.flatten();
NGREST_ASSERT_NULL(res);
NGREST_ASSERT(!strcmp(res->buffer, jsonOut), std::string("Writer Test failed. Expected:\n===================\n")
+ jsonOut + "\n===================\nfound:\n===================\n"
+ res->buffer + "\n===================\n")
} catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
return 1;
}

std::cout << "All json tests passed" << std::endl;

return 0;
}

0 comments on commit 7cbbb98

Please sign in to comment.