From affc0d1a95a51104fe9aaef56d054a9c88db5968 Mon Sep 17 00:00:00 2001 From: Aman <40791522+AmandeepArora@users.noreply.github.com> Date: Wed, 6 Feb 2019 21:01:24 +0530 Subject: [PATCH 01/86] FOGL-2384: Optimize C unit test execution time (#1383) * FOGL-2384: Compiling common code into shared libraries beforehand to improve overall unit test execution speed * Cleanupi & formatting changes --- tests/unit/C/CMakeLists.txt | 59 ++++++++++++++++++++++ tests/unit/C/common/CMakeLists.txt | 11 +++- tests/unit/C/plugins/common/CMakeLists.txt | 22 +++++--- tests/unit/C/scripts/RunAllTests.sh | 9 +++- tests/unit/C/services/core/CMakeLists.txt | 12 +++-- 5 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 tests/unit/C/CMakeLists.txt diff --git a/tests/unit/C/CMakeLists.txt b/tests/unit/C/CMakeLists.txt new file mode 100644 index 0000000000..15ddc09821 --- /dev/null +++ b/tests/unit/C/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 2.6) + +set(CMAKE_CXX_FLAGS "-std=c++11 -O3") +set(UUIDLIB -luuid) +set(COMMONLIB -ldl) + +set(BOOST_COMPONENTS system thread) +# Late 2017 TODO: remove the following checks and always use std::regex +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_BOOST_REGEX") + endif() +endif() +find_package(Boost 1.53.0 COMPONENTS ${BOOST_COMPONENTS} REQUIRED) +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) + +# Find python3.x dev/lib package +find_package(PythonLibs 3 REQUIRED) + +include_directories(../../../C/common/include) +include_directories(../../../C/plugins/common/include) +include_directories(../../../C/services/common/include) +include_directories(../../../C/thirdparty/rapidjson/include) +include_directories(../../../C/thirdparty/Simple-Web-Server) + +# Add Python 3.x header files +include_directories(${PYTHON_INCLUDE_DIRS}) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/../lib) + +# Find source files +file(GLOB COMMON_LIB_SOURCES ../../../C/common/*.cpp) + +# Create shared library +add_library(common-lib SHARED ${COMMON_LIB_SOURCES}) +target_link_libraries(common-lib ${UUIDLIB}) +target_link_libraries(common-lib ${Boost_LIBRARIES}) +set_target_properties(common-lib PROPERTIES SOVERSION 1) + + +# Find source files +file(GLOB SERVICES_COMMON_LIB_SOURCES ../../../C/services/common/*.cpp) + +# Create shared library +add_library(services-common-lib SHARED ${SERVICES_COMMON_LIB_SOURCES}) +target_link_libraries(services-common-lib ${COMMONLIB}) +target_link_libraries(services-common-lib ${PYTHON_LIBRARIES}) +set_target_properties(services-common-lib PROPERTIES SOVERSION 1) + + +# Find source files +file(GLOB PLUGINS_COMMON_LIB_SOURCES ../../../C/plugins/common/*.cpp) + +# Create shared library +add_library(plugins-common-lib SHARED ${PLUGINS_COMMON_LIB_SOURCES}) +target_link_libraries(plugins-common-lib ${Boost_LIBRARIES} common-lib services-common-lib z ssl crypto) + +set_target_properties(plugins-common-lib PROPERTIES SOVERSION 1) diff --git a/tests/unit/C/common/CMakeLists.txt b/tests/unit/C/common/CMakeLists.txt index e91bbc8756..26866b23ed 100644 --- a/tests/unit/C/common/CMakeLists.txt +++ b/tests/unit/C/common/CMakeLists.txt @@ -25,7 +25,10 @@ include_directories(../../../../C/services/common/include) include_directories(../../../../C/thirdparty/rapidjson/include) include_directories(../../../../C/thirdparty/Simple-Web-Server) -file(GLOB test_sources "../../../../C/common/*.cpp" "../../../../C/plugins/common/*.cpp" "../../../../C/services/common/*.cpp") +set(COMMON_LIB common-lib) +set(SERVICE_COMMON_LIB services-common-lib) +set(PLUGINS_COMMON_LIB plugins-common-lib) + file(GLOB unittests "*.cpp") # Find python3.x dev/lib package @@ -34,14 +37,18 @@ find_package(PythonLibs 3 REQUIRED) # Add Python 3.x header files include_directories(${PYTHON_INCLUDE_DIRS}) +link_directories(${PROJECT_BINARY_DIR}/../../lib) # Link runTests with what we want to test and the GTest and pthread library -add_executable(RunTests ${test_sources} ${unittests}) +add_executable(RunTests ${unittests}) target_link_libraries(RunTests ${GTEST_LIBRARIES} pthread) target_link_libraries(RunTests ${Boost_LIBRARIES}) target_link_libraries(RunTests ${UUIDLIB}) target_link_libraries(RunTests ${COMMONLIB}) target_link_libraries(RunTests -lssl -lcrypto -lz) +target_link_libraries(RunTests ${COMMON_LIB}) +target_link_libraries(RunTests ${SERVICE_COMMON_LIB}) +target_link_libraries(RunTests ${PLUGINS_COMMON_LIB}) # Add Python 3.x library target_link_libraries(RunTests ${PYTHON_LIBRARIES}) diff --git a/tests/unit/C/plugins/common/CMakeLists.txt b/tests/unit/C/plugins/common/CMakeLists.txt index 5d13cb8d08..b3353b263a 100644 --- a/tests/unit/C/plugins/common/CMakeLists.txt +++ b/tests/unit/C/plugins/common/CMakeLists.txt @@ -25,9 +25,10 @@ include_directories(../../../../../C/services/common/include) include_directories(../../../../../C/thirdparty/rapidjson/include) include_directories(../../../../../C/thirdparty/Simple-Web-Server) -file(GLOB common_sources "../../../../../C/common/*.cpp") -file(GLOB plugin_common_sources "../../../../../C/plugins/common/*.cpp") -file(GLOB services_common_sources "../../../../../C/services/common/*.cpp") +set(COMMON_LIB common-lib) +set(SERVICE_COMMON_LIB services-common-lib) +set(PLUGINS_COMMON_LIB plugins-common-lib) + file(GLOB unittests "*.cpp") # Find python3.x dev/lib package @@ -36,13 +37,18 @@ find_package(PythonLibs 3 REQUIRED) # Add Python 3.x header files include_directories(${PYTHON_INCLUDE_DIRS}) +link_directories(${PROJECT_BINARY_DIR}/../../../lib) + # Link runTests with what we want to test and the GTest and pthread library -add_executable(RunTests ${common_sources} ${plugin_common_sources} ${services_common_sources} ${unittests}) +add_executable(RunTests ${unittests}) target_link_libraries(RunTests ${GTEST_LIBRARIES} pthread) -target_link_libraries(RunTests ${Boost_LIBRARIES}) -target_link_libraries(RunTests ${UUIDLIB}) -target_link_libraries(RunTests ${COMMONLIB}) -target_link_libraries(RunTests -lssl -lcrypto -lz) +target_link_libraries(RunTests ${Boost_LIBRARIES}) +target_link_libraries(RunTests ${UUIDLIB}) +target_link_libraries(RunTests ${COMMONLIB}) +target_link_libraries(RunTests -lssl -lcrypto -lz) +target_link_libraries(RunTests ${COMMON_LIB}) +target_link_libraries(RunTests ${SERVICE_COMMON_LIB}) +target_link_libraries(RunTests ${PLUGINS_COMMON_LIB}) # Add Python 3.x library target_link_libraries(RunTests ${PYTHON_LIBRARIES}) diff --git a/tests/unit/C/scripts/RunAllTests.sh b/tests/unit/C/scripts/RunAllTests.sh index e0e4283115..3b68b4d833 100755 --- a/tests/unit/C/scripts/RunAllTests.sh +++ b/tests/unit/C/scripts/RunAllTests.sh @@ -18,7 +18,14 @@ cd $FOGLAMP_ROOT/tests/unit/C if [ ! -d results ] ; then mkdir results fi -cmakefile=`find . -name CMakeLists.txt` + +if [ -f "./CMakeLists.txt" ] ; then + echo -n "Compiling libraries..." + (rm -rf build && mkdir build && cd build && cmake .. && make ${jobs} && cd ..) >/dev/null + echo "done" +fi + +cmakefile=`find . -name CMakeLists.txt | grep -v "\.\/CMakeLists.txt"` for f in $cmakefile; do dir=`dirname $f` echo Testing $dir diff --git a/tests/unit/C/services/core/CMakeLists.txt b/tests/unit/C/services/core/CMakeLists.txt index b235121ab2..0c2ac21b5f 100644 --- a/tests/unit/C/services/core/CMakeLists.txt +++ b/tests/unit/C/services/core/CMakeLists.txt @@ -25,9 +25,10 @@ include_directories(../../../../../C/services/core/include) include_directories(../../../../../C/thirdparty/rapidjson/include) include_directories(../../../../../C/thirdparty/Simple-Web-Server) +set(COMMON_LIB common-lib) +set(SERVICE_COMMON_LIB services-common-lib) + file(GLOB test_sources "../../../../../C/services/core/*.cpp") -file(GLOB common_services "../../../../../C/services/common/*.cpp") -file(GLOB common_sources "../../../../../C/common/*.cpp") file(GLOB unittests "*.cpp") # Find python3.x dev/lib package @@ -36,12 +37,17 @@ find_package(PythonLibs 3 REQUIRED) # Add Python 3.x header files include_directories(${PYTHON_INCLUDE_DIRS}) +link_directories(${PROJECT_BINARY_DIR}/../../../lib) + # Link runTests with what we want to test and the GTest and pthread library -add_executable(RunTests ${test_sources} ${common_services} ${common_sources} ${unittests}) +add_executable(RunTests ${test_sources} ${unittests}) target_link_libraries(RunTests ${GTEST_LIBRARIES} pthread) target_link_libraries(RunTests ${Boost_LIBRARIES}) target_link_libraries(RunTests ${UUIDLIB}) target_link_libraries(RunTests ${COMMONLIB}) +target_link_libraries(RunTests -lssl -lcrypto -lz) +target_link_libraries(RunTests ${COMMON_LIB}) +target_link_libraries(RunTests ${SERVICE_COMMON_LIB}) # Add Python 3.x library target_link_libraries(RunTests ${PYTHON_LIBRARIES}) From 157ad69929ec226ecca975b7f6274533ed70e8af Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Thu, 7 Feb 2019 15:25:17 +0000 Subject: [PATCH 02/86] FOGL-2393 Include displayName in default config category --- C/common/config_category.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/C/common/config_category.cpp b/C/common/config_category.cpp index 67005b0685..17e73a5c20 100644 --- a/C/common/config_category.cpp +++ b/C/common/config_category.cpp @@ -1255,6 +1255,11 @@ ostringstream convert; convert << ", \"order\" : \"" << m_order << "\""; } + if (!m_displayName.empty()) + { + convert << ", \"displayName\" : \"" << m_displayName << "\""; + } + if (!m_minimum.empty()) { convert << ", \"minimum\" : \"" << m_minimum << "\""; From e97c12b64ac9b81524eef072ba83cf2e16d540c4 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 8 Feb 2019 15:26:30 +0530 Subject: [PATCH 03/86] FOGL-2401 - Add support for in and not in operators in PayloadBuilder --- .../common/storage_client/payload_builder.py | 7 +++++-- python/foglamp/services/core/api/scheduler.py | 16 +++++++++++----- .../data/payload_condition_in.json | 7 +++++++ .../data/payload_condition_not_in.json | 7 +++++++ .../storage_client/test_payload_builder.py | 5 +++-- .../services/core/api/test_scheduler_api.py | 4 ++-- 6 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 tests/unit/python/foglamp/common/storage_client/data/payload_condition_in.json create mode 100644 tests/unit/python/foglamp/common/storage_client/data/payload_condition_not_in.json diff --git a/python/foglamp/common/storage_client/payload_builder.py b/python/foglamp/common/storage_client/payload_builder.py index 4ac7ef349c..0178ae5fbb 100644 --- a/python/foglamp/common/storage_client/payload_builder.py +++ b/python/foglamp/common/storage_client/payload_builder.py @@ -55,9 +55,12 @@ def verify_condition(arg): retval = False if isinstance(arg, list): if len(arg) == 3: - # TODO: Implement LIKE and IN later when support becomes available in storage service - if arg[1] in ['<', '>', '=', '>=', '<=', '!=', 'newer', 'older']: + # TODO: Implement LIKE later when support becomes available in storage service + if arg[1] in ['<', '>', '=', '>=', '<=', '!=', 'newer', 'older', 'in', 'not in']: retval = True + if arg[1] in ['in', 'not in']: + if isinstance(arg[2], list): + retval = True return retval @staticmethod diff --git a/python/foglamp/services/core/api/scheduler.py b/python/foglamp/services/core/api/scheduler.py index a40fe26524..da107710c9 100644 --- a/python/foglamp/services/core/api/scheduler.py +++ b/python/foglamp/services/core/api/scheduler.py @@ -73,19 +73,25 @@ async def get_scheduled_process(request): a list of all the defined scheduled_processes from scheduled_processes table :Example: - curl -X GET http://localhost:8081/foglamp/schedule/process/purge + curl -X GET http://localhost:8081/foglamp/schedule/process/purge + curl -X GET http://localhost:8081/foglamp/schedule/process/purge%2Cbackup%2Crestore + curl -X GET http://localhost:8081/foglamp/schedule/process/purge%2Cbackup%2Cstats%20collector """ - scheduled_process_name = request.match_info.get('scheduled_process_name', None) - - payload = PayloadBuilder().SELECT("name").WHERE(["name", "=", scheduled_process_name]).payload() + scheduled_process_names = request.match_info.get('scheduled_process_name', None) + scheduled_process_name = scheduled_process_names.split(',') + payload = PayloadBuilder().SELECT("name").WHERE(["name", "in", scheduled_process_name]).payload() _storage = connect.get_storage_async() scheduled_process = await _storage.query_tbl_with_payload('scheduled_processes', payload) if len(scheduled_process['rows']) == 0: raise web.HTTPNotFound(reason='No such Scheduled Process: {}.'.format(scheduled_process_name)) - return web.json_response(scheduled_process['rows'][0].get("name")) + if len(scheduled_process['rows']) == 1: + retval = scheduled_process['rows'][0].get("name") + else: + retval = scheduled_process['rows'] + return web.json_response(retval) ################################# diff --git a/tests/unit/python/foglamp/common/storage_client/data/payload_condition_in.json b/tests/unit/python/foglamp/common/storage_client/data/payload_condition_in.json new file mode 100644 index 0000000000..8c7fe7115e --- /dev/null +++ b/tests/unit/python/foglamp/common/storage_client/data/payload_condition_in.json @@ -0,0 +1,7 @@ + { + "where": { + "column": "plugin_type", + "condition": "in", + "value": ["north", "south", "rule", "delivery", "filter"] + } + } \ No newline at end of file diff --git a/tests/unit/python/foglamp/common/storage_client/data/payload_condition_not_in.json b/tests/unit/python/foglamp/common/storage_client/data/payload_condition_not_in.json new file mode 100644 index 0000000000..5456a74b9f --- /dev/null +++ b/tests/unit/python/foglamp/common/storage_client/data/payload_condition_not_in.json @@ -0,0 +1,7 @@ + { + "where": { + "column": "plugin_type", + "condition": "not in", + "value": ["north", "south"] + } + } \ No newline at end of file diff --git a/tests/unit/python/foglamp/common/storage_client/test_payload_builder.py b/tests/unit/python/foglamp/common/storage_client/test_payload_builder.py index f24fc0496b..bd881a5796 100644 --- a/tests/unit/python/foglamp/common/storage_client/test_payload_builder.py +++ b/tests/unit/python/foglamp/common/storage_client/test_payload_builder.py @@ -86,8 +86,9 @@ def test_from_payload(self, test_input, expected): (["id", "<=", 99], _payload("data/payload_conditions5.json")), (["id", "!=", "False"], _payload("data/payload_conditions6.json")), (["ts", "newer", 3600], _payload("data/payload_newer_condition.json")), - (["ts", "older", 600], _payload("data/payload_older_condition.json")) - + (["ts", "older", 600], _payload("data/payload_older_condition.json")), + (["plugin_type", "in", ['north', 'south', 'rule', 'delivery', 'filter']], _payload("data/payload_condition_in.json")), + (["plugin_type", "not in", ['north', 'south']], _payload("data/payload_condition_not_in.json")) ]) def test_conditions_payload(self, test_input, expected): res = PayloadBuilder().WHERE(test_input).payload() diff --git a/tests/unit/python/foglamp/services/core/api/test_scheduler_api.py b/tests/unit/python/foglamp/services/core/api/test_scheduler_api.py index fbb4cac098..471c31262e 100644 --- a/tests/unit/python/foglamp/services/core/api/test_scheduler_api.py +++ b/tests/unit/python/foglamp/services/core/api/test_scheduler_api.py @@ -69,7 +69,7 @@ async def mock_coro(): async def test_get_scheduled_process(self, client): storage_client_mock = MagicMock(StorageClientAsync) - payload = '{"return": ["name"], "where": {"column": "name", "condition": "=", "value": "purge"}}' + payload = '{"return": ["name"], "where": {"column": "name", "condition": "in", "value": ["purge"]}}' response = {'rows': [{'name': 'purge'}], 'count': 1} with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(storage_client_mock, 'query_tbl_with_payload', @@ -89,7 +89,7 @@ async def test_get_scheduled_process_bad_data(self, client): return_value=mock_coro_response(response)): resp = await client.get('/foglamp/schedule/process/bla') assert 404 == resp.status - assert 'No such Scheduled Process: bla.' == resp.reason + assert "No such Scheduled Process: ['bla']." == resp.reason class TestSchedules: From 303c42c05808319687d21a57c7c5fde92aced7a2 Mon Sep 17 00:00:00 2001 From: Praveen Garg Date: Fri, 8 Feb 2019 17:29:12 +0530 Subject: [PATCH 04/86] summary should be calculatd on latest reading's attribute of an asset not the oldest one; (#1387) --- python/foglamp/services/core/api/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/foglamp/services/core/api/browser.py b/python/foglamp/services/core/api/browser.py index bad6e07049..b84a309106 100644 --- a/python/foglamp/services/core/api/browser.py +++ b/python/foglamp/services/core/api/browser.py @@ -240,7 +240,7 @@ async def asset_all_readings_summary(request): # TODO: FOGL-1768 when support available from storage layer then avoid multiple calls # Find keys in readings - reading_keys = list(results['rows'][0]['reading'].keys()) + reading_keys = list(results['rows'][-1]['reading'].keys()) response = [] _where = PayloadBuilder().WHERE(["asset_code", "=", asset_code]).chain_payload() if 'seconds' in request.query or 'minutes' in request.query or 'hours' in request.query: From d3e2069d8b633b961c0f30817b322a08e3a31527 Mon Sep 17 00:00:00 2001 From: Amarendra Date: Fri, 8 Feb 2019 21:31:17 +0530 Subject: [PATCH 05/86] FOGL-2386 - Python South Async plugins having FogLAMP req as >= 1.5 should use south_c as process_name and should disable the schedule upon upgrade (#1372) * FOGL-2386 - Python South Async plugins 2.0 should use south_c as process_name and should disable the schedule upon upgrade * Feedback changes for http, host and port * MInor refactoring * More refactoring * Feedback changes * Minor refactoring * Feedback changes * https support added * Upgrade/Downgrade scripts added * Postgres upgrade scripts fixed --- VERSION | 2 +- .../foglamp/services/south/modify_process.py | 144 ++++++++++++++++++ python/foglamp/services/south/server.py | 10 +- .../plugins/storage/postgres/downgrade/25.sql | 1 + .../plugins/storage/postgres/upgrade/26.sql | 1 + .../plugins/storage/sqlite/downgrade/25.sql | 1 + scripts/plugins/storage/sqlite/upgrade/26.sql | 2 + 7 files changed, 159 insertions(+), 2 deletions(-) create mode 100755 python/foglamp/services/south/modify_process.py create mode 100644 scripts/plugins/storage/postgres/downgrade/25.sql create mode 100644 scripts/plugins/storage/postgres/upgrade/26.sql create mode 100644 scripts/plugins/storage/sqlite/downgrade/25.sql create mode 100644 scripts/plugins/storage/sqlite/upgrade/26.sql diff --git a/VERSION b/VERSION index d2211184cb..e7c567e21d 100755 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ foglamp_version=1.4.1 -foglamp_schema=25 +foglamp_schema=26 diff --git a/python/foglamp/services/south/modify_process.py b/python/foglamp/services/south/modify_process.py new file mode 100755 index 0000000000..427509b954 --- /dev/null +++ b/python/foglamp/services/south/modify_process.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +"""Modify process_name for Python Async plugins from south to south_c""" + +import aiohttp +import logging + +from foglamp.common import logger +from foglamp.common.storage_client.exceptions import StorageServerError +from foglamp.common.storage_client.payload_builder import PayloadBuilder +from foglamp.common.storage_client.storage_client import StorageClientAsync +from foglamp.common.configuration_manager import ConfigurationManager + +__author__ = "Amarendra K Sinha" +__copyright__ = "Copyright (c) 2019 Dianomic Systems" +__license__ = "Apache 2.0" +__version__ = "${VERSION}" + +_LOGGER = logger.setup(__name__, level=logging.INFO) +_HTTP = 'http' +_HOST = 'localhost' +_PORT = 8081 + + +async def disable_service(service_name): + try: + put_url = "{}://{}:{}/foglamp/schedule/disable".format(_HTTP, _HOST, _PORT) + data = '{"schedule_name": "%s"}' % service_name + verify_ssl = False if _HTTP == 'http' else True + connector = aiohttp.TCPConnector(verify_ssl=verify_ssl) + async with aiohttp.ClientSession(connector=connector) as session: + async with session.put(put_url, data=data) as resp: + status_code = resp.status + jdoc = await resp.json() + if status_code not in range(200, 209): + _LOGGER.error("Error code: %d, reason: %s, details: %s, url: %s", resp.status, resp.reason, jdoc, + put_url) + raise StorageServerError(code=resp.status, reason=resp.reason, error=jdoc) + except Exception: + raise + else: + _LOGGER.info('Disabled Python South service [%s] to change process_name to south_c', service_name) + + +async def unregister_service(service_name, core_management_port, microservice_id): + try: + delete_url = "{}://{}:{}/foglamp/service/{}".format(_HTTP, _HOST, core_management_port, microservice_id) + verify_ssl = False if _HTTP == 'http' else True + connector = aiohttp.TCPConnector(verify_ssl=verify_ssl) + async with aiohttp.ClientSession(connector=connector) as session: + async with session.delete(delete_url) as resp: + status_code = resp.status + jdoc = await resp.text() + if status_code not in range(200, 209): + # In case of serviceRegistry.DoesNotExist, HTTPInternalServerError(500) is raised. Hence this + # scenario is being handled indirectly by RuntimeError. + raise RuntimeError(jdoc) + except RuntimeError as ex: + if 'DoesNotExist' in str(ex): + pass + else: + _LOGGER.error('Exception [%s] at unregistering service %s', str(ex), service_name) + else: + _LOGGER.info('Unregistered service %s', service_name) + + +async def modify_process_name(service_name, core_management_host, core_management_port): + storage = StorageClientAsync(core_management_host, core_management_port) + + try: + payload = PayloadBuilder().SELECT("id").WHERE(['schedule_name', '=', service_name]).payload() + result = await storage.query_tbl_with_payload('schedules', payload) + except Exception: + raise + + if int(result['count']): + sch_id = result['rows'][0]['id'] + else: + _LOGGER.error('No schedule id found for %s. Exiting...', service_name) + raise RuntimeError('No schedule id found for %s. Exiting...', service_name) + + # Modify process name + try: + put_url = "{}://{}:{}/foglamp/schedule/{}".format(_HTTP, _HOST, _PORT, sch_id) + data = '{"process_name": "south_c"}' + verify_ssl = False if _HTTP == 'http' else True + connector = aiohttp.TCPConnector(verify_ssl=verify_ssl) + async with aiohttp.ClientSession(connector=connector) as session: + async with session.put(put_url, data=data) as resp: + status_code = resp.status + jdoc = await resp.text() + if status_code not in range(200, 209): + _LOGGER.error("Error code: %d, reason: %s, details: %s, url: %s", resp.status, resp.reason, jdoc, + put_url) + raise StorageServerError(code=resp.status, reason=resp.reason, error=jdoc) + except Exception: + raise + else: + _LOGGER.info('Modified process_name from "south" to "south_c" for Python South service [%s]: %s', service_name, jdoc) + + +async def reenable_service(service_name): + try: + put_url = "{}://{}:{}/foglamp/schedule/enable".format(_HTTP, _HOST, _PORT) + data = '{"schedule_name": "%s"}' % service_name + verify_ssl = False if _HTTP == 'http' else True + connector = aiohttp.TCPConnector(verify_ssl=verify_ssl) + async with aiohttp.ClientSession(connector=connector) as session: + async with session.put(put_url, data=data) as resp: + status_code = resp.status + jdoc = await resp.json() + if status_code not in range(200, 209): + _LOGGER.error("Error code: %d, reason: %s, details: %s, url: %s", resp.status, resp.reason, jdoc, + put_url) + raise StorageServerError(code=resp.status, reason=resp.reason, error=jdoc) + except Exception: + raise + else: + _LOGGER.info('Enabled Python South service [%s] after changing process_name to south_c', service_name) + + +async def change_to_south_c(service_name, microservice_management_host, core_management_host, core_management_port, microservice_id): + global _HTTP, _HOST, _PORT + + storage = StorageClientAsync(core_management_host, core_management_port) + configuration_manager = ConfigurationManager(storage) + config = await configuration_manager.get_category_all_items('rest_api') + + is_rest_server_http_enabled = False if config['enableHttp']['value'] == 'false' else True + port_from_config = config['httpPort']['value'] if is_rest_server_http_enabled \ + else config['httpsPort']['value'] + + _HTTP = 'http' if is_rest_server_http_enabled else 'https' + _HOST = microservice_management_host + _PORT = int(port_from_config) + + await disable_service(service_name) + await unregister_service(service_name, core_management_port, microservice_id) + await modify_process_name(service_name,core_management_host, core_management_port) + await reenable_service(service_name) diff --git a/python/foglamp/services/south/server.py b/python/foglamp/services/south/server.py index ef588e40ed..7ddbed6dde 100755 --- a/python/foglamp/services/south/server.py +++ b/python/foglamp/services/south/server.py @@ -8,6 +8,7 @@ import json import asyncio +import sys from foglamp.services.south import exceptions from foglamp.common import logger from foglamp.services.south.ingest import Ingest @@ -110,6 +111,12 @@ async def _start(self, loop) -> None: # Plugin initialization self._plugin_info = self._plugin.plugin_info() + if float(self._plugin_info['version'][:3]) >= 1.5 and self._plugin_info['mode'] == 'async': + _LOGGER.warning('Plugin %s should be loaded by C version of South server', self._name) + import foglamp.services.south.modify_process as mp + await mp.change_to_south_c(self._name, self._microservice_management_host, self._core_management_host, self._core_management_port, self._microservice_id) + sys.exit(1) + default_config = self._plugin_info['config'] default_plugin_descr = self._name if (default_config['plugin']['description']).strip() == "" else \ default_config['plugin']['description'] @@ -241,7 +248,8 @@ async def _stop(self, loop): raise ex try: - self._task_main.cancel() + if self._task_main is not None: + self._task_main.cancel() # Cancel all pending asyncio tasks after a timeout occurs done, pending = await asyncio.wait(asyncio.Task.all_tasks(), timeout=_CLEAR_PENDING_TASKS_TIMEOUT) for task_pending in pending: diff --git a/scripts/plugins/storage/postgres/downgrade/25.sql b/scripts/plugins/storage/postgres/downgrade/25.sql new file mode 100644 index 0000000000..c02ea1518c --- /dev/null +++ b/scripts/plugins/storage/postgres/downgrade/25.sql @@ -0,0 +1 @@ +-- No actions diff --git a/scripts/plugins/storage/postgres/upgrade/26.sql b/scripts/plugins/storage/postgres/upgrade/26.sql new file mode 100644 index 0000000000..2fbe3aea54 --- /dev/null +++ b/scripts/plugins/storage/postgres/upgrade/26.sql @@ -0,0 +1 @@ +INSERT INTO scheduled_processes ( name, script ) VALUES ( 'south_c', '["services/south_c"]' ) ON CONFLICT DO NOTHING; diff --git a/scripts/plugins/storage/sqlite/downgrade/25.sql b/scripts/plugins/storage/sqlite/downgrade/25.sql new file mode 100644 index 0000000000..c02ea1518c --- /dev/null +++ b/scripts/plugins/storage/sqlite/downgrade/25.sql @@ -0,0 +1 @@ +-- No actions diff --git a/scripts/plugins/storage/sqlite/upgrade/26.sql b/scripts/plugins/storage/sqlite/upgrade/26.sql new file mode 100644 index 0000000000..1018f2e190 --- /dev/null +++ b/scripts/plugins/storage/sqlite/upgrade/26.sql @@ -0,0 +1,2 @@ +INSERT OR IGNORE INTO scheduled_processes ( name, script ) VALUES ( 'south_c', '["services/south_c"]' ); + From 0926b209944b2bf99a8e1d4864857d31da205b9b Mon Sep 17 00:00:00 2001 From: pintomax Date: Fri, 8 Feb 2019 17:22:52 +0100 Subject: [PATCH 06/86] FOGL-2436: Variable asset structure in OMF (#1392) FOGL-2436: Variable asset structure in OMF FOGL-2436: unit tests updated --- C/plugins/common/include/omf.h | 9 +- C/plugins/common/omf.cpp | 172 ++++++++++++++++-- .../C/plugins/common/test_omf_translation.cpp | 50 +++++ 3 files changed, 213 insertions(+), 18 deletions(-) diff --git a/C/plugins/common/include/omf.h b/C/plugins/common/include/omf.h index fd75d1e1c5..4e875564c0 100644 --- a/C/plugins/common/include/omf.h +++ b/C/plugins/common/include/omf.h @@ -87,13 +87,20 @@ class OMF // Check DataTypeError bool isDataTypeError(const char* message); + // Map object types found in input data + void setMapObjectTypes(const std::vector& data, + std::map& dataSuperSet) const; + // Removed mapped object types found in input data + void unsetMapObjectTypes(std::map& dataSuperSet) const; + private: /** * Builds the HTTP header to send * messagetype header takes the passed type value: * 'Type', 'Container', 'Data' */ - const std::vector> createMessageHeader(const std::string& type) const; + const std::vector> + createMessageHeader(const std::string& type) const; // Create data for Type message for current row const std::string createTypeData(const Reading& reading) const; diff --git a/C/plugins/common/omf.cpp b/C/plugins/common/omf.cpp index fbfbf08a63..73a072bc2d 100644 --- a/C/plugins/common/omf.cpp +++ b/C/plugins/common/omf.cpp @@ -376,6 +376,13 @@ bool OMF::sendDataTypes(const Reading& row) uint32_t OMF::sendToServer(const vector& readings, bool compression, bool skipSentDataTypes) { + std::map superSetDataPoints; + + // Create a superset of all found datapoints for each assetName + // the superset[assetName] is then passed to routines which handle + // creation of OMF data types + setMapObjectTypes(readings, superSetDataPoints); + /* * Iterate over readings: * - Send/cache Types @@ -405,15 +412,31 @@ uint32_t OMF::sendToServer(const vector& readings, // Always send types true; - // Handle the data types of the current reading - if (sendDataTypes && - // Send data typer - !OMF::handleDataTypes(**elem, skipSentDataTypes) && + Reading* datatypeStructure = NULL; + if (sendDataTypes) + { + // Get the supersetDataPoints for current assetName + auto it = superSetDataPoints.find((**elem).getAssetName()); + if (it != superSetDataPoints.end()) + { + datatypeStructure = (*it).second; + } + } + + // Check first we have supersetDataPoints for the current reading + if ((sendDataTypes && datatypeStructure == NULL) || + // Handle the data types of the current reading + (sendDataTypes && + // Send data type + !OMF::handleDataTypes(*datatypeStructure, skipSentDataTypes) && // Data type not sent: (!m_changeTypeId || // Increment type-id and re-send data types - !OMF::handleTypeErrors(**elem))) + !OMF::handleTypeErrors(*datatypeStructure)))) { + // Remove all assets supersetDataPoints + unsetMapObjectTypes(superSetDataPoints); + // Failure m_lastError = true; return 0; @@ -424,6 +447,9 @@ uint32_t OMF::sendToServer(const vector& readings, (elem < (readings.end() - 1 ) ? ", " : ""); } + // Remove all assets supersetDataPoints + unsetMapObjectTypes(superSetDataPoints); + jsonData << "]"; string json = jsonData.str(); @@ -770,7 +796,7 @@ const std::string OMF::createTypeData(const Reading& reading) const "typename_measurement", tData); - tData.append("\" }]"); + tData.append("\" }]"); // Return JSON string return tData; @@ -936,17 +962,6 @@ bool OMF::handleDataTypes(const Reading& row, OMF::setCreatedTypes(key); } - // Just for dubug right now - //if (skipSending) - //{ - // cerr << "dataTypes for key [" << key << "]" << - // (sendTypes ? " have been sent." : " already sent.") << endl; - //} - //else - //{ - // cerr << "dataTypes for typeId [" << m_typeId << "] have been sent." << endl; - //} - // Success return true; } @@ -1099,3 +1114,126 @@ bool OMF::handleTypeErrors(const Reading& reading) } return ret; } + +/** + * Create a superset data map for each reading and found datapoints + * + * The output map is filled with a Reading object containing + * all the datapoints found for each asset in the inoput reading set. + * The datapoint have a fake value based on the datapoint type + * + * @param readings Current input readings data + * @param dataSuperSet Map to store all datapoints for an assetname + */ +void OMF::setMapObjectTypes(const vector& readings, + std::map& dataSuperSet) const +{ + // Temporary map for [asset][datapoint] = type + std::map> readingAllDataPoints; + + // Fetch ALL Reading pointers in the input vecror + // and create a map of [assetName][datapoint1 .. datapointN] = type + for (vector::const_iterator elem = readings.begin(); + elem != readings.end(); + ++elem) + { + // Get asset name + string assetName = (**elem).getAssetName(); + // Get all datapoints + const vector data = (**elem).getReadingData(); + // Iterate through datapoints + for (vector::const_iterator it = data.begin(); + it != data.end(); + ++it) + { + string omfType = omfTypes[((*it)->getData()).getType()]; + string datapointName = (*it)->getName(); + auto itr = readingAllDataPoints.find(assetName); + // Asset not found in the map + if (itr == readingAllDataPoints.end()) + { + // Set type of current datapoint for ssetName + readingAllDataPoints[assetName][datapointName] = omfType; + } + else + { + // Asset found + auto dpItr = (*itr).second.find(datapointName); + // Datapoint not found + if (dpItr == (*itr).second.end()) + { + // Add datapointName/type to map with key assetName + (*itr).second.emplace(datapointName, omfType); + } + else + { + if ((*dpItr).second.compare(omfType) != 0) + { + // Datapoint already set has changed type + Logger::getLogger()->warn("Datapoint '" + datapointName + \ + "' in asset '" + assetName + \ + "' has changed type from '" + (*dpItr).second + \ + " to " + omfType); + } + + // Update datapointName/type to map with key assetName + // 1- remove element + (*itr).second.erase(dpItr); + // 2- Add new value + readingAllDataPoints[assetName][datapointName] = omfType; + } + } + } + } + + // Loop now only the elements found in the per asset types map + for (auto it = readingAllDataPoints.begin(); + it != readingAllDataPoints.end(); + ++it) + { + string assetName = (*it).first; + vector values; + // Set fake datapoints values + for (auto dp = (*it).second.begin(); + dp != (*it).second.end(); + ++dp) + { + if ((*dp).second.compare(OMF_TYPE_FLOAT) == 0) + { + DatapointValue vDouble(0.1); + values.push_back(new Datapoint((*dp).first, vDouble)); + } + else if ((*dp).second.compare(OMF_TYPE_INTEGER) == 0) + { + DatapointValue vInt((long)1); + values.push_back(new Datapoint((*dp).first, vInt)); + } + else if ((*dp).second.compare(OMF_TYPE_STRING) == 0) + { + DatapointValue vString("v_str"); + values.push_back(new Datapoint((*dp).first, vString)); + } + } + + // Add the superset Reading data with fake values + dataSuperSet.emplace(assetName, new Reading(assetName, values)); + } +} + +/** + * Cleanup the mapped object types for input data + * + * @param dataSuperSet The mapped object to cleanup + */ +void OMF::unsetMapObjectTypes(std::map& dataSuperSet) const +{ + // Remove all assets supersetDataPoints + for (auto m = dataSuperSet.begin(); + m != dataSuperSet.end(); + ++m) + { + (*m).second->removeAllDatapoints(); + delete (*m).second; + } + dataSuperSet.clear(); +} diff --git a/tests/unit/C/plugins/common/test_omf_translation.cpp b/tests/unit/C/plugins/common/test_omf_translation.cpp index 83202363e8..f107112f1d 100644 --- a/tests/unit/C/plugins/common/test_omf_translation.cpp +++ b/tests/unit/C/plugins/common/test_omf_translation.cpp @@ -3,6 +3,7 @@ #include #include #include +#include /* * FogLAMP Readings to OMF translation unit tests @@ -41,6 +42,28 @@ const char *two_readings = R"( } )"; +// 2 readings JSON text +const char *readings_with_different_datapoints = R"( + { + "count" : 2, "rows" : [ + { + "id": 1, "asset_code": "A", + "read_key": "5b3be500-ff95-41ae-b5a4-cc99d08bef4a", + "reading": { "lux": 45204.524 }, + "user_ts": "2018-06-11 14:00:08.532958", + "ts": "2018-06-12 14:47:18.872708" + }, + { + "id": 2, "asset_code": "A", + "read_key": "5b3be50c-ff95-41ae-b5a4-cc99d08bef4a", + "reading": { "temp": 23, "label" : "device_1" }, + "user_ts": "2018-08-21 14:00:09.32958", + "ts": "2018-08-22 14:48:18.72708" + } + ] + } +)"; + // 2 readings translated to OMF JSON text const char *two_translated_readings = R"([{"containerid": ")" TYPE_ID R"(measurement_luxometer", "values": [{"lux": 45204.524, "Time": "2018-06-11T14:00:08.532958Z"}]}, {"containerid": ")" TYPE_ID R"(measurement_luxometer", "values": [{"lux": 76834.361, "Time": "2018-08-21T14:00:09.329580Z"}]}])"; @@ -118,3 +141,30 @@ TEST(OMF_transation, OneReading) ASSERT_EQ((*itr)["values"].GetArray()[0].GetObject().MemberCount(), 3); } } + +// Compare translated readings with a provided JSON value +TEST(OMF_transation, SuperSet) +{ + SimpleHttps sender("0.0.0.0:0", 10, 10, 10, 1); + OMF omf(sender, "/", "1", "ABC"); + // Build a ReadingSet from JSON + ReadingSet readingSet(readings_with_different_datapoints); + vectorreadings = readingSet.getAllReadings(); + + std::map superSetDataPoints; + + // Create a superset of all found datapoints for each assetName + // the superset[assetName] is then passed to routines which handle + // creation of OMF data types + omf.setMapObjectTypes(readings, superSetDataPoints); + + // We have only 1 superset reading as the readings in input + // have same assetName + ASSERT_EQ(1, superSetDataPoints.size()); + auto it = superSetDataPoints.begin(); + // We have 3 datapoints in total in te superset + ASSERT_EQ(3, (*it).second->getDatapointCount()); + omf.unsetMapObjectTypes(superSetDataPoints); + // Superset map is empty + ASSERT_EQ(0, superSetDataPoints.size()); +} From ade4916e851c1d00e392caae4f4b1d724c96fe9e Mon Sep 17 00:00:00 2001 From: Stefano Simonelli Date: Fri, 8 Feb 2019 17:02:16 +0000 Subject: [PATCH 07/86] FOGL-2417: improve the handling of PI-Server new type definition (#1380) * FOGL-2417: working in progress * FOGL-2417: removed debug logs. --- C/plugins/common/omf.cpp | 22 ++++++++++++---------- C/plugins/north/PI_Server_V2/plugin.cpp | 3 ++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/C/plugins/common/omf.cpp b/C/plugins/common/omf.cpp index 73a072bc2d..328bdc011f 100644 --- a/C/plugins/common/omf.cpp +++ b/C/plugins/common/omf.cpp @@ -180,8 +180,8 @@ bool OMF::sendDataTypes(const Reading& row) // Data type error: force type-id change m_changeTypeId = true; } - Logger::getLogger()->error("Sending JSON dataType message 'Type' " - "- %s - error: |%s| - HostPort |%s| - path |%s| - OMF message |%s|", + Logger::getLogger()->warn("Sending JSON dataType message 'Type', " + "not blocking issue: |%s| - message |%s| - HostPort |%s| - path |%s| - OMF message |%s|", (m_changeTypeId ? "Data Type " : "" ), e.what(), m_sender.getHostPort().c_str(), @@ -234,8 +234,8 @@ bool OMF::sendDataTypes(const Reading& row) // Data type error: force type-id change m_changeTypeId = true; } - Logger::getLogger()->error("Sending JSON dataType message 'Container' " - "- %s - error: |%s| - HostPort |%s| - path |%s| - OMF message |%s|", + Logger::getLogger()->warn("Sending JSON dataType message 'Container' " + "not blocking issue: |%s| - message |%s| - HostPort |%s| - path |%s| - OMF message |%s|", (m_changeTypeId ? "Data Type " : "" ), e.what(), m_sender.getHostPort().c_str(), @@ -287,8 +287,8 @@ bool OMF::sendDataTypes(const Reading& row) // Data type error: force type-id change m_changeTypeId = true; } - Logger::getLogger()->error("Sending JSON dataType message 'StaticData'" - "- %s - error: |%s| - HostPort |%s| - path |%s| - OMF message |%s|", + Logger::getLogger()->warn("Sending JSON dataType message 'StaticData'" + "not blocking issue: |%s| - message |%s| - HostPort |%s| - path |%s| - OMF message |%s|", (m_changeTypeId ? "Data Type " : "" ), e.what(), m_sender.getHostPort().c_str(), @@ -345,8 +345,8 @@ bool OMF::sendDataTypes(const Reading& row) // Data type error: force type-id change m_changeTypeId = true; } - Logger::getLogger()->error("Sending JSON dataType message 'Data' (lynk) " - "- %s - error: |%s| - HostPort |%s| - path |%s| - OMF message |%s|", + Logger::getLogger()->warn("Sending JSON dataType message 'Data' (lynk) " + "not blocking issue: |%s| - message |%s| - HostPort |%s| - path |%s| - OMF message |%s|", (m_changeTypeId ? "Data Type " : "" ), e.what(), m_sender.getHostPort().c_str(), @@ -520,7 +520,9 @@ uint32_t OMF::sendToServer(const vector& readings, OMF::clearCreatedTypes(); // Reset error indicator m_lastError = false; - // Just return the size of readings buffer to the caller + + // It returns size instead of 0 as the rows in the block should be skipped in case of an error + // as it is considered a not blocking ones. return readings.size(); } else @@ -1035,7 +1037,7 @@ void OMF::setFormatType(const string &key, string &value) void OMF::setNotBlockingErrors(std::vector& notBlockingErrors) { - m_notBlockingErrors = std::move(notBlockingErrors); + m_notBlockingErrors = notBlockingErrors; } diff --git a/C/plugins/north/PI_Server_V2/plugin.cpp b/C/plugins/north/PI_Server_V2/plugin.cpp index 476647198a..c1ec7aa850 100644 --- a/C/plugins/north/PI_Server_V2/plugin.cpp +++ b/C/plugins/north/PI_Server_V2/plugin.cpp @@ -91,7 +91,8 @@ using namespace rapidjson; "\"default\": \"{\\\"errors400\\\": "\ "["\ "\\\"Redefinition of the type with the same ID is not allowed\\\", "\ - "\\\"Invalid value type for the property\\\" "\ + "\\\"Invalid value type for the property\\\", "\ + "\\\"Property does not exist in the type definition\\\" "\ "]"\ "}\", " \ "\"order\": \"17\" ," \ From 73669c459af3d211c7337b0b41e9dac8d5a871f6 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Mon, 11 Feb 2019 19:25:39 +0530 Subject: [PATCH 08/86] bulk update for previous value in stats history task run --- .../tasks/statistics/statistics_history.py | 15 +++++++++------ .../tasks/statistics/test_statistics_history.py | 13 +++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/python/foglamp/tasks/statistics/statistics_history.py b/python/foglamp/tasks/statistics/statistics_history.py index f5a08da3e9..76a166376d 100644 --- a/python/foglamp/tasks/statistics/statistics_history.py +++ b/python/foglamp/tasks/statistics/statistics_history.py @@ -10,6 +10,7 @@ Fetch information from the statistics table, compute delta and stores the delta value (statistics.value - statistics.previous_value) in the statistics_history table """ +import json from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common import logger @@ -45,17 +46,15 @@ async def _insert_into_stats_history(self, key='', value=0, history_ts=None): payload = PayloadBuilder().INSERT(key=key, value=value, history_ts=history_ts).payload() await self._storage_async.insert_into_tbl("statistics_history", payload) - async def _update_previous_value(self, key='', value=0): + async def _bulk_update_previous_value(self, payload): """ UPDATE previous_value of column to have the same value as snapshot Query: UPDATE statistics_history SET previous_value = value WHERE key = key Args: - key: Key which previous_value gets update - value: value at snapshot + payload: dict containing statistics keys and previous values """ - payload = PayloadBuilder().SET(previous_value=value).WHERE(["key", "=", key]).payload() - await self._storage_async.update_tbl("statistics", payload) + await self._storage_async.update_tbl("statistics", json.dumps(payload, sort_keys=False)) async def run(self): """ SELECT against the statistics table, to get a snapshot of the data at that moment. @@ -66,10 +65,14 @@ async def run(self): """ current_time = utils.local_timestamp() results = await self._storage_async.query_tbl("statistics") + payload = {"updates": []} for r in results['rows']: key = r['key'] value = int(r["value"]) previous_value = int(r["previous_value"]) delta = value - previous_value + # TODO: Once FOGL-1973 is done, use bulk insert for statistics_history await self._insert_into_stats_history(key=key, value=delta, history_ts=current_time) - await self._update_previous_value(key=key, value=value) + payload_item = PayloadBuilder().SET(previous_value=value).WHERE(["key", "=", key]).payload() + payload['updates'].append(json.loads(payload_item)) + await self._bulk_update_previous_value(payload) diff --git a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py index 93fb019552..e8729bc4d5 100644 --- a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py +++ b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py @@ -9,7 +9,7 @@ import asyncio from unittest.mock import patch, MagicMock import pytest -from datetime import datetime + import ast from foglamp.common import logger from foglamp.common.storage_client.storage_client import StorageClientAsync @@ -68,13 +68,14 @@ async def test_update_previous_value(self): with patch.object(logger, "setup"): sh = StatisticsHistory() sh._storage_async = MagicMock(spec=StorageClientAsync) + payload = {'updates': [{'where': {'value': 'Bla', 'condition': '=', 'column': 'key'}, 'values': {'previous_value': 1}}]} with patch.object(sh._storage_async, "update_tbl", return_value=mock_coro(None)) as patch_storage: - await sh._update_previous_value(key='Bla', value=1) + await sh._bulk_update_previous_value(payload) args, kwargs = patch_storage.call_args assert "statistics" == args[0] payload = ast.literal_eval(args[1]) - assert "Bla" == payload["where"]["value"] - assert 1 == payload["values"]["previous_value"] + assert "Bla" == payload["updates"][0]["where"]["value"] + assert 1 == payload["updates"][0]["values"]["previous_value"] async def test_run(self): with patch.object(FoglampProcess, '__init__'): @@ -92,9 +93,9 @@ async def test_run(self): } with patch.object(sh._storage_async, "query_tbl", return_value=mock_coro(retval)) as mock_keys: with patch.object(sh, "_insert_into_stats_history", return_value=mock_coro(None)) as mock_insert_history: - with patch.object(sh, "_update_previous_value", return_value=mock_coro(None)) as mock_update: + with patch.object(sh, "_bulk_update_previous_value", return_value=mock_coro(None)) as mock_update: await sh.run() - assert 2 == mock_update.call_count + assert 1 == mock_update.call_count args, kwargs = mock_insert_history.call_args assert "READINGS" == kwargs["key"] mock_keys.assert_called_once_with('statistics') From 1d1a1c67f6193043a07c22304f0825728f2b9705 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Mon, 11 Feb 2019 11:31:09 +0000 Subject: [PATCH 09/86] FOGL-2451 Make no data to purge a log message and not an error return from the service --- C/plugins/storage/sqlite/connection.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index 270b2bd930..eed12bf546 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -2202,8 +2202,8 @@ int blocks = 0; result = "{ \"removed\" : 0, "; result += " \"unsentPurged\" : 0, "; - result += " \"unsentRetained\", 0, "; - result += " \"readings\" 0 }"; + result += " \"unsentRetained\" : 0, "; + result += " \"readings\" : 0 }"; logger->info("Purge starting..."); @@ -2292,7 +2292,7 @@ int blocks = 0; if (rowidLimit == 0) { - raiseError("purge", "No data to purge"); + logger->info("No data to purge"); return 0; } From 9721880b7ca3b10ef4677b0d8d432dfab04ca59a Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Tue, 12 Feb 2019 20:45:55 +0530 Subject: [PATCH 10/86] FOGL-1121 - FoglampProcess should properly handle improper argument values and throw the correct exceptions --- python/foglamp/common/process.py | 64 +++++++------------ .../python/foglamp/common/test_process.py | 17 ++--- 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index 4ad156faf6..d829184f35 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -18,26 +18,25 @@ __license__ = "Apache 2.0" __version__ = "${VERSION}" - _logger = logger.setup(__name__) class ArgumentParserError(Exception): """ Override default exception to not terminate application """ - pass + def __init__(self, message): + self.message = message -class SilentArgParse(argparse.ArgumentParser): + def __str__(self): + format = '%(message)s' + return format % dict(message=self.message) + +class SilentArgParse(argparse.ArgumentParser): def error(self, message): """ Override default error functionality to not terminate application """ raise ArgumentParserError(message) - def silent_arg_parse(self, argument_name): - self.add_argument(argument_name) - parser_result = self.parse_known_args() - return list(vars(parser_result[0]).values())[0] - class FoglampProcess(ABC): """ FoglampProcess for all non-core python processes. @@ -73,25 +72,27 @@ def __init__(self): --port [core microservice management port] --name [process name] """ - + self._start_time = time.time() - try: - self._core_management_host = self.get_arg_value("--address") - self._core_management_port = self.get_arg_value("--port") - self._name = self.get_arg_value("--name") - except ArgumentParserError: + try: + parser = SilentArgParse() + parser.add_argument("--name", required=True) + parser.add_argument("--address", required=True) + parser.add_argument("--port", required=True, type=int) + namespace, args = parser.parse_known_args() + self._name = getattr(namespace, 'name') + self._core_management_host = getattr(namespace, 'address') + self._core_management_port = getattr(namespace, 'port') + except ArgumentParserError as ex: + _logger.error("Arg parser error: %s", str(ex)) raise - if self._core_management_host is None: - raise ValueError("--address is not specified") - elif self._core_management_port is None: - raise ValueError("--port is not specified") - elif self._name is None: - raise ValueError("--name is not specified") - self._core_microservice_management_client = MicroserviceManagementClient(self._core_management_host,self._core_management_port) + self._core_microservice_management_client = MicroserviceManagementClient(self._core_management_host, + self._core_management_port) - self._readings_storage_async = ReadingsStorageClientAsync(self._core_management_host, self._core_management_port) + self._readings_storage_async = ReadingsStorageClientAsync(self._core_management_host, + self._core_management_port) self._storage_async = StorageClientAsync(self._core_management_host, self._core_management_port) # pure virtual method run() to be implemented by child class @@ -99,25 +100,6 @@ def __init__(self): def run(self): pass - def get_arg_value(self, argument_name): - """ Parses command line arguments for a single argument of name argument_name. Returns the value of the argument specified or None if argument was not specified. - - Keyword Arguments: - argument_name -- name of command line argument to retrieve value for - - Return Values: - Argument value (as a string) - None (if argument was not passed) - - Side Effects: - None - - Known Exceptions: - ArgumentParserError - """ - parser = SilentArgParse() - return parser.silent_arg_parse(argument_name) - def get_services_from_core(self, name=None, _type=None): return self._core_microservice_management_client.get_services(name, _type) diff --git a/tests/unit/python/foglamp/common/test_process.py b/tests/unit/python/foglamp/common/test_process.py index f138d35115..0c96af45c4 100644 --- a/tests/unit/python/foglamp/common/test_process.py +++ b/tests/unit/python/foglamp/common/test_process.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import pytest +import sys from unittest.mock import patch @@ -28,15 +29,15 @@ class FoglampProcessImp(FoglampProcess): fp = FoglampProcessImp() @pytest.mark.parametrize('argslist', - [([ArgumentParserError()]), - (['corehost', ArgumentParserError()]), - (['corehost', 0, ArgumentParserError()]) + [(['pytest']), + (['pytest, ''--address', 'corehost']), + (['pytest', '--address', 'corehost', '--port', 0]) ]) def test_constructor_missing_args(self, argslist): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=argslist): + with patch.object(sys, 'argv', argslist): with pytest.raises(ArgumentParserError) as excinfo: fp = FoglampProcessImp() assert '' in str( @@ -46,7 +47,7 @@ def test_constructor_good(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -66,7 +67,7 @@ def test_get_services_from_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'get_services', return_value=None) as get_patch: with patch.object(ReadingsStorageClientAsync, '__init__', @@ -80,7 +81,7 @@ def test_register_service_with_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'register_service', return_value=None) as register_patch: with patch.object(ReadingsStorageClientAsync, '__init__', @@ -94,7 +95,7 @@ def test_unregister_service_with_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'unregister_service', return_value=None) as unregister_patch: with patch.object(ReadingsStorageClientAsync, '__init__', From e1556bc3625a943848992a57b1eae0cf6eeaf09f Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Tue, 12 Feb 2019 16:33:54 +0000 Subject: [PATCH 11/86] FOGL-2473 Display Name strings for advanced options --- C/common/config_category.cpp | 20 ++++++++++++++++++++ C/common/include/config_category.h | 1 + C/services/south/include/defaults.h | 12 ++++++++---- C/services/south/south.cpp | 4 +++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/C/common/config_category.cpp b/C/common/config_category.cpp index 17e73a5c20..9bb7c9b486 100644 --- a/C/common/config_category.cpp +++ b/C/common/config_category.cpp @@ -275,6 +275,26 @@ void ConfigCategory::addItem(const std::string& name, const std::string descript m_items.push_back(new CategoryItem(name, description, def, value, options)); } +/** + * Set the display name of an item + * + * @param name The item name in the category + * @param displayName The display name to set + * @return true if the item was found + */ +bool ConfigCategory::setItemDisplayName(const std::string& name, const std::string& displayName) +{ + for (unsigned int i = 0; i < m_items.size(); i++) + { + if (name.compare(m_items[i]->m_name) == 0) + { + m_items[i]->m_displayName = displayName; + return true; + } + } + return false; +} + /** * Delete all the items from the configuration category having a specific type * diff --git a/C/common/include/config_category.h b/C/common/include/config_category.h index e95e4da9cf..fee7bf94e8 100644 --- a/C/common/include/config_category.h +++ b/C/common/include/config_category.h @@ -74,6 +74,7 @@ class ConfigCategory { std::string getDescription() const { return m_description; }; unsigned int getCount() const { return m_items.size(); }; bool itemExists(const std::string& name) const; + bool setItemDisplayName(const std::string& name, const std::string& displayName); std::string getValue(const std::string& name) const; std::string getType(const std::string& name) const; std::string getDescription(const std::string& name) const; diff --git a/C/services/south/include/defaults.h b/C/services/south/include/defaults.h index e990db64e7..b2b2129ff3 100644 --- a/C/services/south/include/defaults.h +++ b/C/services/south/include/defaults.h @@ -12,13 +12,17 @@ static struct { const char *name; + const char *displayName; const char *description; const char *type; const char *value; } defaults[] = { - { "readingsPerSec", "Number of readings to generate per sec", "integer", "1" }, - { "maxSendLatency", "Maximum time to spend filling buffer before sending", "integer", "5000" }, - { "bufferThreshold", "Number of readings to buffer before sending", "integer", "100" }, - { NULL, NULL, NULL, NULL } + { "readingsPerSec", "Readings Per Second", + "Number of readings to generate per sec", "integer", "1" }, + { "maxSendLatency", "Maximum Reading Latency (mS)", + "Maximum time to spend filling buffer before sending", "integer", "5000" }, + { "bufferThreshold", "Maximum buffered Readings", + "Number of readings to buffer before sending", "integer", "100" }, + { NULL, NULL, NULL, NULL, NULL } }; #endif diff --git a/C/services/south/south.cpp b/C/services/south/south.cpp index d8e252c88f..d6b0037e2d 100644 --- a/C/services/south/south.cpp +++ b/C/services/south/south.cpp @@ -572,13 +572,15 @@ void SouthService::addConfigDefaults(DefaultConfigCategory& defaultConfig) for (int i = 0; defaults[i].name; i++) { defaultConfig.addItem(defaults[i].name, defaults[i].description, - defaults[i].type, defaults[i].value, defaults[i].value); + defaults[i].type, defaults[i].value, defaults[i].value); + defaultConfig.setItemDisplayName(defaults[i].name, defaults[i].displayName); } /* Add the set of logging levels to the service */ vector logLevels = { "error", "warning", "info", "debug" }; defaultConfig.addItem("logLevel", "Minimum logging level reported", "warning", "warning", logLevels); + defaultConfig.setItemDisplayName("logLevel", "Minimum Log Level"); } /** From 7440837a1852234b0a27778a8a01cb1b748b9b6b Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Wed, 13 Feb 2019 11:33:01 +0530 Subject: [PATCH 12/86] Fixed tests --- .../services/common/test_microservice.py | 7 +-- .../tasks/north/test_sending_process.py | 44 +++++++++---------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/unit/python/foglamp/services/common/test_microservice.py b/tests/unit/python/foglamp/services/common/test_microservice.py index e58c87b35b..318af9aa0f 100644 --- a/tests/unit/python/foglamp/services/common/test_microservice.py +++ b/tests/unit/python/foglamp/services/common/test_microservice.py @@ -5,6 +5,7 @@ from unittest.mock import patch, MagicMock from aiohttp import web import asyncio +import sys from foglamp.common.storage_client.storage_client import ReadingsStorageClientAsync, StorageClientAsync from foglamp.common.process import FoglampProcess, SilentArgParse, ArgumentParserError from foglamp.services.common.microservice import FoglampMicroservice, _logger @@ -91,7 +92,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', @@ -145,7 +146,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', @@ -183,7 +184,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', diff --git a/tests/unit/python/foglamp/tasks/north/test_sending_process.py b/tests/unit/python/foglamp/tasks/north/test_sending_process.py index 1444b988dd..18f9ee2896 100644 --- a/tests/unit/python/foglamp/tasks/north/test_sending_process.py +++ b/tests/unit/python/foglamp/tasks/north/test_sending_process.py @@ -55,7 +55,7 @@ async def mock_audit_failure(): def fixture_sp(event_loop): """" Configures the sending process instance for the tests """ - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -367,7 +367,7 @@ async def test_is_stream_id_valid(self, async def test_is_north_valid(self, plugin_file, plugin_type, plugin_name, expected_result, event_loop): """Tests the possible cases of the function is_north_valid """ - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -392,7 +392,7 @@ async def mock_coroutine(): return True # Checks the Readings handling - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -472,7 +472,7 @@ async def mock_coroutine(): return p_rows # Checks the Readings handling - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -558,7 +558,7 @@ async def test_transform_in_memory_data_readings(self, """ Unit test for - _transform_in_memory_data_readings""" # Checks the Readings handling - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -729,7 +729,7 @@ async def test_load_data_into_memory_statistics(self, """Test _load_data_into_memory handling and transformations for the statistics """ # Checks the Statistics handling - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -822,7 +822,7 @@ async def test_transform_in_memory_data_statistics(self, """ Unit test for - _transform_in_memory_data_statistics""" # Checks the Statistics handling - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -860,7 +860,7 @@ async def mock_query_tbl_row_2(): rows = {"rows": [{"last_object": 10}, {"last_object": 11}]} return rows - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -927,7 +927,7 @@ async def mock_task(): return True - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1101,7 +1101,7 @@ async def retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1289,7 +1289,7 @@ async def retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1433,7 +1433,7 @@ async def mock_retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1602,7 +1602,7 @@ async def mock_retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1786,7 +1786,7 @@ async def mock_send_rows(x): return p_send_result[x]["data_sent"], p_send_result[x]["new_last_object_id"], p_send_result[x]["num_sent"] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1968,7 +1968,7 @@ async def mock_send_rows(x): return p_send_result[x]["data_sent"], p_send_result[x]["new_last_object_id"], p_send_result[x]["num_sent"] # GIVEN - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2204,7 +2204,7 @@ async def mock_task(): """ Dummy async task """ return True - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2230,7 +2230,7 @@ async def mock_task(): async def test_standard_plugins(self, plugin_file, plugin_type, plugin_name, event_loop): """Tests if the standard plugins are available and loadable and if they have the required methods """ - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2290,7 +2290,7 @@ async def test_retrieve_configuration_good(self, expected_config): """ Unit tests - _retrieve_configuration - tests the transformations """ - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2313,7 +2313,7 @@ async def test_retrieve_configuration_good(self, async def test_start_stream_not_valid(self, event_loop): """ Unit tests - _start - stream_id is not valid """ - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2338,7 +2338,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2374,7 +2374,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2414,7 +2414,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(SilentArgParse, 'silent_arg_parse', side_effect=['corehost', 0, 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: From 90d4e72cd599fbc35e53ed1116ca00a113b26b2f Mon Sep 17 00:00:00 2001 From: pintomax Date: Wed, 13 Feb 2019 11:31:38 +0100 Subject: [PATCH 13/86] FOGL-2466: Create/Update category defaults for ServiceName_FilterName (#1398) FOGL:2466: filters configuration merge not working correctly FOGL-2466: fix code formatting --- C/common/filter_pipeline.cpp | 215 ++++++++++++++++++----------------- 1 file changed, 108 insertions(+), 107 deletions(-) diff --git a/C/common/filter_pipeline.cpp b/C/common/filter_pipeline.cpp index c780c3dd4d..c6f3637a0b 100644 --- a/C/common/filter_pipeline.cpp +++ b/C/common/filter_pipeline.cpp @@ -86,130 +86,131 @@ bool FilterPipeline::loadFilters(const string& categoryName) Logger::getLogger()->info("FilterPipeline::loadFilters(): categoryName=%s, filters=%s", categoryName.c_str(), filter.c_str()); if (!filter.empty()) { - std::vector> filterInfo; + std::vector> filterInfo; - // Remove \" and leading/trailing " - // TODO: improve/change this - filter.erase(remove(filter.begin(), filter.end(), '\\' ), filter.end()); - size_t i; - while (! (i = filter.find('"')) || (i = filter.rfind('"')) == static_cast(filter.size() - 1)) - { - filter.erase(i, 1); - } + // Remove \" and leading/trailing " + // TODO: improve/change this + filter.erase(remove(filter.begin(), filter.end(), '\\' ), filter.end()); + size_t i; + while (! (i = filter.find('"')) || (i = filter.rfind('"')) == static_cast(filter.size() - 1)) + { + filter.erase(i, 1); + } - //Parse JSON object for filters - Document theFilters; - theFilters.Parse(filter.c_str()); - // The "pipeline" property must be an array - if (theFilters.HasParseError() || - !theFilters.HasMember(JSON_CONFIG_PIPELINE_ELEM) || - !theFilters[JSON_CONFIG_PIPELINE_ELEM].IsArray()) - { - string errMsg("loadFilters: can not parse JSON '"); - errMsg += string(JSON_CONFIG_FILTER_ELEM) + "' property"; - Logger::getLogger()->fatal(errMsg.c_str()); - throw runtime_error(errMsg); - } - else - { - const Value& filterList = theFilters[JSON_CONFIG_PIPELINE_ELEM]; - if (!filterList.Size()) + //Parse JSON object for filters + Document theFilters; + theFilters.Parse(filter.c_str()); + // The "pipeline" property must be an array + if (theFilters.HasParseError() || + !theFilters.HasMember(JSON_CONFIG_PIPELINE_ELEM) || + !theFilters[JSON_CONFIG_PIPELINE_ELEM].IsArray()) { - // Empty array, just return true - return true; + string errMsg("loadFilters: can not parse JSON '"); + errMsg += string(JSON_CONFIG_FILTER_ELEM) + "' property"; + Logger::getLogger()->fatal(errMsg.c_str()); + throw runtime_error(errMsg); } + else + { + const Value& filterList = theFilters[JSON_CONFIG_PIPELINE_ELEM]; + if (!filterList.Size()) + { + // Empty array, just return true + return true; + } - // Prepare printable list of filters - StringBuffer buffer; - Writer writer(buffer); - filterList.Accept(writer); - string printableList(buffer.GetString()); + // Prepare printable list of filters + StringBuffer buffer; + Writer writer(buffer); + filterList.Accept(writer); + string printableList(buffer.GetString()); - string logMsg("loadFilters: found filter(s) "); - logMsg += printableList + " for plugin '"; - logMsg += categoryName + "'"; + string logMsg("loadFilters: found filter(s) "); + logMsg += printableList + " for plugin '"; + logMsg += categoryName + "'"; - Logger::getLogger()->info(logMsg.c_str()); + Logger::getLogger()->info(logMsg.c_str()); - // Try loading all filter plugins: abort on any error - for (Value::ConstValueIterator itr = filterList.Begin(); itr != filterList.End(); ++itr) - { - // Get "plugin" item fromn filterCategoryName - string filterCategoryName = itr->GetString(); - ConfigCategory filterDetails = mgtClient->getCategory(filterCategoryName); - if (!filterDetails.itemExists("plugin")) - { - string errMsg("loadFilters: 'plugin' item not found "); - errMsg += "in " + filterCategoryName + " category"; - Logger::getLogger()->fatal(errMsg.c_str()); - throw runtime_error(errMsg); - } - string filterName = filterDetails.getValue("plugin"); - PLUGIN_HANDLE filterHandle; - // Load filter plugin only: we don't call any plugin method right now - filterHandle = loadFilterPlugin(filterName); - if (!filterHandle) + // Try loading all filter plugins: abort on any error + for (Value::ConstValueIterator itr = filterList.Begin(); itr != filterList.End(); ++itr) { - string errMsg("Cannot load filter plugin '" + filterName + "'"); - Logger::getLogger()->fatal(errMsg.c_str()); - throw runtime_error(errMsg); + // Get "plugin" item fromn filterCategoryName + string filterCategoryName = itr->GetString(); + ConfigCategory filterDetails = mgtClient->getCategory(filterCategoryName); + if (!filterDetails.itemExists("plugin")) + { + string errMsg("loadFilters: 'plugin' item not found "); + errMsg += "in " + filterCategoryName + " category"; + Logger::getLogger()->fatal(errMsg.c_str()); + throw runtime_error(errMsg); + } + string filterName = filterDetails.getValue("plugin"); + PLUGIN_HANDLE filterHandle; + // Load filter plugin only: we don't call any plugin method right now + filterHandle = loadFilterPlugin(filterName); + if (!filterHandle) + { + string errMsg("Cannot load filter plugin '" + filterName + "'"); + Logger::getLogger()->fatal(errMsg.c_str()); + throw runtime_error(errMsg); + } + else + { + // Save filter handler: key is filterCategoryName + filterInfo.push_back(pair + (filterCategoryName, filterHandle)); + } } - else + + // We have kept filter default config in the filterInfo map + // Handle configuration for each filter + PluginManager *pluginManager = PluginManager::getInstance(); + for (vector>::iterator itr = filterInfo.begin(); + itr != filterInfo.end(); + ++itr) { - // Save filter handler: key is filterCategoryName - filterInfo.push_back(pair - (filterCategoryName, filterHandle)); - } - } + // Get plugin default configuration + string filterConfig = pluginManager->getInfo(itr->second)->config; - // We have kept filter default config in the filterInfo map - // Handle configuration for each filter - PluginManager *pluginManager = PluginManager::getInstance(); - for (vector>::iterator itr = filterInfo.begin(); - itr != filterInfo.end(); - ++itr) - { - // Get plugin default configuration - string filterConfig = pluginManager->getInfo(itr->second)->config; + // Create/Update default filter category items + DefaultConfigCategory filterDefConfig(categoryName + "_" + itr->first, filterConfig); + string filterDescription = "Configuration of '" + itr->first; + filterDescription += "' filter for plugin '" + categoryName + "'"; + filterDefConfig.setDescription(filterDescription); - // Update filter category items - DefaultConfigCategory filterDefConfig(itr->first, filterConfig); - string filterDescription = "Configuration of '" + itr->first; - filterDescription += "' filter for plugin '" + categoryName + "'"; - filterDefConfig.setDescription(filterDescription); + if (!mgtClient->addCategory(filterDefConfig, true)) + { + string errMsg("Cannot create/update '" + \ + categoryName + "' filter category"); + Logger::getLogger()->fatal(errMsg.c_str()); + throw runtime_error(errMsg); + } + children.push_back(categoryName + "_" + itr->first); - if (!mgtClient->addCategory(filterDefConfig, true)) - { - string errMsg("Cannot create/update '" + \ - categoryName + "' filter category"); - Logger::getLogger()->fatal(errMsg.c_str()); - throw runtime_error(errMsg); - } - children.push_back(categoryName + "_" + itr->first); + // Instantiate the FilterPlugin class + // in order to call plugin entry points + FilterPlugin* currentFilter = new FilterPlugin(itr->first, + itr->second); - // Instantiate the FilterPlugin class - // in order to call plugin entry points - FilterPlugin* currentFilter = new FilterPlugin(itr->first, - itr->second); - - // Add filter to filters vector - m_filters.push_back(currentFilter); + // Add filter to filters vector + m_filters.push_back(currentFilter); + } } } - } - /* - * Put all the new catregories in the Filter category parent - * Create an empty South category if one doesn't exist - */ - string parentName = categoryName + " Filters"; - DefaultConfigCategory filterConfig(parentName, string("{}")); - filterConfig.setDescription("Filters for " + categoryName); - mgtClient->addCategory(filterConfig, true); - mgtClient->addChildCategories(parentName, children); - vector children1; - children1.push_back(parentName); - mgtClient->addChildCategories(categoryName, children1); - return true; + + /* + * Put all the new catregories in the Filter category parent + * Create an empty South category if one doesn't exist + */ + string parentName = categoryName + " Filters"; + DefaultConfigCategory filterConfig(parentName, string("{}")); + filterConfig.setDescription("Filters for " + categoryName); + mgtClient->addCategory(filterConfig, true); + mgtClient->addChildCategories(parentName, children); + vector children1; + children1.push_back(parentName); + mgtClient->addChildCategories(categoryName, children1); + return true; } catch (ConfigItemNotFound* e) { From 3f8bec190ed8e126d0b9a9338000d818fa386bd8 Mon Sep 17 00:00:00 2001 From: pintomax Date: Wed, 13 Feb 2019 13:25:49 +0100 Subject: [PATCH 14/86] FOGL-2343: return failure code when connections to north side services fail (#1399) FOGL-2343: return failure code when connections to north side services fail If at least one data block is sent (or no data to send) return success, return failure otherwise. --- .../north/sending_process/sending_process.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/C/tasks/north/sending_process/sending_process.cpp b/C/tasks/north/sending_process/sending_process.cpp index ee3ff2a8a7..b8ce5ae4ed 100644 --- a/C/tasks/north/sending_process/sending_process.cpp +++ b/C/tasks/north/sending_process/sending_process.cpp @@ -45,6 +45,12 @@ condition_variable cond_var; // Buffer max elements unsigned long memoryBufferSize; +// Exit code: +// 0 = success (some data sent) +// 1 = 100% failure sending data to north server +// 2 =internal errors +int exitCode = 1; + // Used to identifies logs const string LOG_SERVICE_NAME = "SendingProcess/sending_process"; @@ -80,7 +86,7 @@ int main(int argc, char** argv) { cerr << "Exception in " << argv[0] << " : " << e.what() << endl; // Return failure for class instance/configuration etc - exit(1); + exit(2); } // Catch all exceptions catch (...) @@ -88,11 +94,11 @@ int main(int argc, char** argv) std::exception_ptr p = std::current_exception(); string name = (p ? p.__cxa_exception_type()->name() : "null"); cerr << "Generic Exception in " << argv[0] << " : " << name << endl; - exit(1); + exit(2); } // Return success - exit(0); + exit(exitCode); } /** @@ -432,10 +438,12 @@ static void sendDataThread(SendingProcess *sendData) if (sentReadings) { processUpdate = true; + exitCode = 0; } } else { + exitCode = 0; // We have an empty readings set: check last id if (sendData->m_last_read_id.at(sendIdx) > 0) { @@ -445,6 +453,8 @@ static void sendDataThread(SendingProcess *sendData) if (processUpdate) { + exitCode = 0; + /** Sending done */ sendData->setUpdateDb(true); From 161edb8d5b9ef8ff03d38fccbdfc1bd2597ea500 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 14 Feb 2019 17:03:41 +0530 Subject: [PATCH 15/86] regex check added for host and port --- python/foglamp/common/process.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index d829184f35..daf3296070 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -9,6 +9,7 @@ from abc import ABC, abstractmethod import argparse import time +import re from foglamp.common.storage_client.storage_client import StorageClientAsync, ReadingsStorageClientAsync from foglamp.common import logger from foglamp.common.microservice_management_client.microservice_management_client import MicroserviceManagementClient @@ -84,6 +85,12 @@ def __init__(self): self._name = getattr(namespace, 'name') self._core_management_host = getattr(namespace, 'address') self._core_management_port = getattr(namespace, 'port') + if (re.match( + "(^[2][0-5][0-5]|^[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})$", + self._core_management_host) is None): + raise ArgumentParserError("Invalid Host: {}".format(self._core_management_host)) + if self._core_management_port < 1 or self._core_management_port > 65535: + raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port)) except ArgumentParserError as ex: _logger.error("Arg parser error: %s", str(ex)) raise From ba82e67232976261df5b92c278ec6f1efbd8a6d8 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Thu, 14 Feb 2019 11:34:23 +0000 Subject: [PATCH 16/86] FOGL-2483 Stack trace in FogLAMPProcess --- C/common/process.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/C/common/process.cpp b/C/common/process.cpp index a1282d1952..5266b249ba 100644 --- a/C/common/process.cpp +++ b/C/common/process.cpp @@ -15,11 +15,57 @@ #include #include #include +#include +#include +#include #define LOG_SERVICE_NAME "FogLAMP Process" using namespace std; +/** + * Signal handker to log stack trqaces on fatal signals + */ +static void handler(int sig) +{ +Logger *logger = Logger::getLogger(); +void *array[20]; +char buf[1024]; +int size; + + // get void*'s for all entries on the stack + size = backtrace(array, 20); + + // print out all the frames to stderr + logger->fatal("Signal %d (%s) trapped:\n", sig, strsignal(sig)); + char **messages = backtrace_symbols(array, size); + for (int i = 0; i < size; i++) + { + Dl_info info; + if (dladdr(array[i], &info) && info.dli_sname) + { + char *demangled = NULL; + int status = -1; + if (info.dli_sname[0] == '_') + demangled = abi::__cxa_demangle(info.dli_sname, NULL, 0, &status); + snprintf(buf, sizeof(buf), "%-3d %*p %s + %zd---------", + i, int(2 + sizeof(void*) * 2), array[i], + status == 0 ? demangled : + info.dli_sname == 0 ? messages[i] : info.dli_sname, + (char *)array[i] - (char *)info.dli_saddr); + free(demangled); + } + else + { + snprintf(buf, sizeof(buf), "%-3d %*p %s---------", + i, int(2 + sizeof(void*) * 2), array[i], messages[i]); + } + logger->fatal("(%d) %s", i, buf); + } + free(messages); + exit(1); +} + // Destructor FogLampProcess::~FogLampProcess() { @@ -34,6 +80,12 @@ FogLampProcess::FogLampProcess(int argc, char** argv) : m_argc(argc), m_arg_vals((const char**) argv) { + signal(SIGSEGV, handler); + signal(SIGILL, handler); + signal(SIGBUS, handler); + signal(SIGFPE, handler); + signal(SIGABRT, handler); + string myName = LOG_SERVICE_NAME; m_logger = new Logger(myName); From 3aadfc7c6ff63b7f21fd8675f7a1b24fe4693d30 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 14 Feb 2019 17:35:37 +0530 Subject: [PATCH 17/86] more checking for host --- python/foglamp/common/process.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index daf3296070..5c9d9dfb2c 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -85,10 +85,11 @@ def __init__(self): self._name = getattr(namespace, 'name') self._core_management_host = getattr(namespace, 'address') self._core_management_port = getattr(namespace, 'port') - if (re.match( - "(^[2][0-5][0-5]|^[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})$", - self._core_management_host) is None): - raise ArgumentParserError("Invalid Host: {}".format(self._core_management_host)) + if self._core_management_host.strip() != 'localhost': + if (re.match( + "(^[2][0-5][0-5]|^[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})$", + self._core_management_host) is None): + raise ArgumentParserError("Invalid Host: {}".format(self._core_management_host)) if self._core_management_port < 1 or self._core_management_port > 65535: raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port)) except ArgumentParserError as ex: From e3eef3009b5641f1178f5a6c00cdb6d89228e9b1 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 14 Feb 2019 18:10:27 +0530 Subject: [PATCH 18/86] argument checking refactored --- python/foglamp/common/process.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index 5c9d9dfb2c..542df25844 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -85,13 +85,15 @@ def __init__(self): self._name = getattr(namespace, 'name') self._core_management_host = getattr(namespace, 'address') self._core_management_port = getattr(namespace, 'port') - if self._core_management_host.strip() != 'localhost': - if (re.match( - "(^[2][0-5][0-5]|^[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})\.([0-2][0-5][0-5]|[1]{0,1}[0-9]{1,2})$", - self._core_management_host) is None): - raise ArgumentParserError("Invalid Host: {}".format(self._core_management_host)) if self._core_management_port < 1 or self._core_management_port > 65535: raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port)) + for item in args: + if item.startswith('--'): + kv = item.split('=') + if len(kv) == 2: + if len(kv[1].strip()) == 0: + raise ArgumentParserError("Invalid value {} for optional arg {}".format(kv[1], kv[0])) + except ArgumentParserError as ex: _logger.error("Arg parser error: %s", str(ex)) raise From 236db67568938fd11c8ada20cb6d3b97c1164e74 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Thu, 14 Feb 2019 14:35:54 +0000 Subject: [PATCH 19/86] FOGL-2483 Add stacktrace to storage service and FogLAMPProcess --- C/common/process.cpp | 4 ++- C/services/storage/storage.cpp | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/C/common/process.cpp b/C/common/process.cpp index 5266b249ba..d43b526088 100644 --- a/C/common/process.cpp +++ b/C/common/process.cpp @@ -16,15 +16,17 @@ #include #include #include +#include #include #include + #define LOG_SERVICE_NAME "FogLAMP Process" using namespace std; /** - * Signal handker to log stack trqaces on fatal signals + * Signal handler to log stack trqaces on fatal signals */ static void handler(int sig) { diff --git a/C/services/storage/storage.cpp b/C/services/storage/storage.cpp index 66a0f75fed..9062be9711 100644 --- a/C/services/storage/storage.cpp +++ b/C/services/storage/storage.cpp @@ -18,11 +18,59 @@ #include #include #include +#include +#include +#include +#include extern int makeDaemon(void); using namespace std; +/** + * Signal handler to log stack trqaces on fatal signals + */ +static void handler(int sig) +{ +Logger *logger = Logger::getLogger(); +void *array[20]; +char buf[1024]; +int size; + + // get void*'s for all entries on the stack + size = backtrace(array, 20); + + // print out all the frames to stderr + logger->fatal("Signal %d (%s) trapped:\n", sig, strsignal(sig)); + char **messages = backtrace_symbols(array, size); + for (int i = 0; i < size; i++) + { + Dl_info info; + if (dladdr(array[i], &info) && info.dli_sname) + { + char *demangled = NULL; + int status = -1; + if (info.dli_sname[0] == '_') + demangled = abi::__cxa_demangle(info.dli_sname, NULL, 0, &status); + snprintf(buf, sizeof(buf), "%-3d %*p %s + %zd---------", + i, int(2 + sizeof(void*) * 2), array[i], + status == 0 ? demangled : + info.dli_sname == 0 ? messages[i] : info.dli_sname, + (char *)array[i] - (char *)info.dli_saddr); + free(demangled); + } + else + { + snprintf(buf, sizeof(buf), "%-3d %*p %s---------", + i, int(2 + sizeof(void*) * 2), array[i], messages[i]); + } + logger->fatal("(%d) %s", i, buf); + } + free(messages); + exit(1); +} + + /** * Storage service main entry point */ @@ -122,6 +170,12 @@ unsigned short servicePort; config = new StorageConfiguration(); logger = new Logger(myName); + signal(SIGSEGV, handler); + signal(SIGILL, handler); + signal(SIGBUS, handler); + signal(SIGFPE, handler); + signal(SIGABRT, handler); + if (config->getValue("port") == NULL) { servicePort = 0; // default to a dynamic port From de7bf2ff700a3631b94d88aeeba0be3a726733f3 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Thu, 14 Feb 2019 15:47:43 +0000 Subject: [PATCH 20/86] Update process.cpp --- C/common/process.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/C/common/process.cpp b/C/common/process.cpp index d43b526088..66bc518951 100644 --- a/C/common/process.cpp +++ b/C/common/process.cpp @@ -26,7 +26,7 @@ using namespace std; /** - * Signal handler to log stack trqaces on fatal signals + * Signal handler to log stack traces on fatal signals */ static void handler(int sig) { From de593fb716938b8ba20ebda4f65489462287dcc5 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 15 Feb 2019 13:48:49 +0530 Subject: [PATCH 21/86] Tests fixed --- .../python/foglamp/common/test_process.py | 18 +++---- .../services/common/test_microservice.py | 8 ++-- .../tasks/north/test_sending_process.py | 48 +++++++++---------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/unit/python/foglamp/common/test_process.py b/tests/unit/python/foglamp/common/test_process.py index 0c96af45c4..4f1e6223e9 100644 --- a/tests/unit/python/foglamp/common/test_process.py +++ b/tests/unit/python/foglamp/common/test_process.py @@ -31,7 +31,7 @@ class FoglampProcessImp(FoglampProcess): @pytest.mark.parametrize('argslist', [(['pytest']), (['pytest, ''--address', 'corehost']), - (['pytest', '--address', 'corehost', '--port', 0]) + (['pytest', '--address', 'corehost', '--port', '32333']) ]) def test_constructor_missing_args(self, argslist): class FoglampProcessImp(FoglampProcess): @@ -47,16 +47,16 @@ def test_constructor_good(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: fp = FoglampProcessImp() - mmc_patch.assert_called_once_with('corehost', 0) - rsc_async_patch.assert_called_once_with('corehost', 0) - sc_async_patch.assert_called_once_with('corehost', 0) + mmc_patch.assert_called_once_with('corehost', 32333) + rsc_async_patch.assert_called_once_with('corehost', 32333) + sc_async_patch.assert_called_once_with('corehost', 32333) assert fp._core_management_host is 'corehost' - assert fp._core_management_port is 0 + assert fp._core_management_port == 32333 assert fp._name is 'sname' assert hasattr(fp, '_core_microservice_management_client') assert hasattr(fp, '_readings_storage_async') @@ -67,7 +67,7 @@ def test_get_services_from_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'get_services', return_value=None) as get_patch: with patch.object(ReadingsStorageClientAsync, '__init__', @@ -81,7 +81,7 @@ def test_register_service_with_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'register_service', return_value=None) as register_patch: with patch.object(ReadingsStorageClientAsync, '__init__', @@ -95,7 +95,7 @@ def test_unregister_service_with_core(self): class FoglampProcessImp(FoglampProcess): def run(self): pass - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'unregister_service', return_value=None) as unregister_patch: with patch.object(ReadingsStorageClientAsync, '__init__', diff --git a/tests/unit/python/foglamp/services/common/test_microservice.py b/tests/unit/python/foglamp/services/common/test_microservice.py index 318af9aa0f..2aa7307898 100644 --- a/tests/unit/python/foglamp/services/common/test_microservice.py +++ b/tests/unit/python/foglamp/services/common/test_microservice.py @@ -92,7 +92,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', @@ -109,7 +109,7 @@ async def add_track(self): fm = FoglampMicroserviceImp() # from FoglampProcess assert fm._core_management_host is 'corehost' - assert fm._core_management_port is 0 + assert fm._core_management_port == 32333 assert fm._name is 'sname' assert hasattr(fm, '_core_microservice_management_client') assert hasattr(fm, '_readings_storage_async') @@ -146,7 +146,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', @@ -184,7 +184,7 @@ async def add_track(self): pass with patch.object(asyncio, 'get_event_loop', return_value=loop): - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(MicroserviceManagementClient, 'create_configuration_category', return_value=None): with patch.object(MicroserviceManagementClient, 'create_child_category', diff --git a/tests/unit/python/foglamp/tasks/north/test_sending_process.py b/tests/unit/python/foglamp/tasks/north/test_sending_process.py index 18f9ee2896..854044ce9b 100644 --- a/tests/unit/python/foglamp/tasks/north/test_sending_process.py +++ b/tests/unit/python/foglamp/tasks/north/test_sending_process.py @@ -55,7 +55,7 @@ async def mock_audit_failure(): def fixture_sp(event_loop): """" Configures the sending process instance for the tests """ - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -77,7 +77,7 @@ def fixture_sp(event_loop): sp._task_fetch_data_sem = asyncio.Semaphore(0) sp._task_send_data_sem = asyncio.Semaphore(0) - + return sp @@ -367,7 +367,7 @@ async def test_is_stream_id_valid(self, async def test_is_north_valid(self, plugin_file, plugin_type, plugin_name, expected_result, event_loop): """Tests the possible cases of the function is_north_valid """ - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -392,7 +392,7 @@ async def mock_coroutine(): return True # Checks the Readings handling - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -472,7 +472,7 @@ async def mock_coroutine(): return p_rows # Checks the Readings handling - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -558,7 +558,7 @@ async def test_transform_in_memory_data_readings(self, """ Unit test for - _transform_in_memory_data_readings""" # Checks the Readings handling - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -729,7 +729,7 @@ async def test_load_data_into_memory_statistics(self, """Test _load_data_into_memory handling and transformations for the statistics """ # Checks the Statistics handling - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -822,7 +822,7 @@ async def test_transform_in_memory_data_statistics(self, """ Unit test for - _transform_in_memory_data_statistics""" # Checks the Statistics handling - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -860,7 +860,7 @@ async def mock_query_tbl_row_2(): rows = {"rows": [{"last_object": 10}, {"last_object": 11}]} return rows - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -927,7 +927,7 @@ async def mock_task(): return True - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1101,7 +1101,7 @@ async def retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1289,7 +1289,7 @@ async def retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1433,7 +1433,7 @@ async def mock_retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1602,7 +1602,7 @@ async def mock_retrieve_rows(idx): return p_rows[idx] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1786,7 +1786,7 @@ async def mock_send_rows(x): return p_send_result[x]["data_sent"], p_send_result[x]["new_last_object_id"], p_send_result[x]["num_sent"] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -1832,7 +1832,7 @@ async def mock_send_rows(x): # Lets the _task_fetch_data to run for a while await asyncio.sleep(3) - + # Tear down sp._task_send_data_run = False sp._task_fetch_data_sem.release() @@ -1968,7 +1968,7 @@ async def mock_send_rows(x): return p_send_result[x]["data_sent"], p_send_result[x]["new_last_object_id"], p_send_result[x]["num_sent"] # GIVEN - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2204,7 +2204,7 @@ async def mock_task(): """ Dummy async task """ return True - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2230,7 +2230,7 @@ async def mock_task(): async def test_standard_plugins(self, plugin_file, plugin_type, plugin_name, event_loop): """Tests if the standard plugins are available and loadable and if they have the required methods """ - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2290,7 +2290,7 @@ async def test_retrieve_configuration_good(self, expected_config): """ Unit tests - _retrieve_configuration - tests the transformations """ - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2313,7 +2313,7 @@ async def test_retrieve_configuration_good(self, async def test_start_stream_not_valid(self, event_loop): """ Unit tests - _start - stream_id is not valid """ - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2338,7 +2338,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2374,7 +2374,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: @@ -2414,7 +2414,7 @@ async def mock_stat_key(): async def mock_master_stat_key(): return 'Readings Sent' - with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', 0, '--name', 'sname']): + with patch.object(sys, 'argv', ['pytest', '--address', 'corehost', '--port', '32333', '--name', 'sname']): with patch.object(MicroserviceManagementClient, '__init__', return_value=None) as mmc_patch: with patch.object(ReadingsStorageClientAsync, '__init__', return_value=None) as rsc_async_patch: with patch.object(StorageClientAsync, '__init__', return_value=None) as sc_async_patch: From 05f586ce33a0e9a820b93c4a69e3e25be43b2e44 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 15 Feb 2019 13:52:58 +0530 Subject: [PATCH 22/86] Minor refactoring --- python/foglamp/common/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index 542df25844..e95596e037 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -9,7 +9,6 @@ from abc import ABC, abstractmethod import argparse import time -import re from foglamp.common.storage_client.storage_client import StorageClientAsync, ReadingsStorageClientAsync from foglamp.common import logger from foglamp.common.microservice_management_client.microservice_management_client import MicroserviceManagementClient From fbb2dd13bc4e9efbfb670715a9e7f8e63785c53e Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Fri, 15 Feb 2019 19:49:50 +0530 Subject: [PATCH 23/86] Initial commit --- tests/system/python/conftest.py | 24 ++- .../e2e/test_e2e_csv_multi_filter_pi.py | 168 ++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py diff --git a/tests/system/python/conftest.py b/tests/system/python/conftest.py index d3be62394d..6b4f56d844 100644 --- a/tests/system/python/conftest.py +++ b/tests/system/python/conftest.py @@ -225,6 +225,21 @@ def _add_filter(filter_plugin, filter_plugin_branch, filter_name, filter_config, data = {"name": "{}".format(filter_name), "plugin": "{}".format(filter_plugin), "filter_config": filter_config} conn = http.client.HTTPConnection(foglamp_url) + # Get filters in pipeline + existing_pipeline = [] + conn.request("GET", '/foglamp/filter/{}/pipeline'.format(filter_user_svc_task)) + r = conn.getresponse() + res = r.read().decode() + if r.status == 404: + print("No pipeline exist") + print( + "existing_pipeline={}, existing_pipeline_format={}".format(existing_pipeline, type(existing_pipeline))) + else: + print("Pipeline exist") + jdoc = json.loads(res) + existing_pipeline = jdoc["result"]["pipeline"] + print("existing_pipeline={}, existing_pipeline_format={}".format(existing_pipeline, type(existing_pipeline))) + conn.request("POST", '/foglamp/filter', json.dumps(data)) r = conn.getresponse() assert 200 == r.status @@ -233,8 +248,13 @@ def _add_filter(filter_plugin, filter_plugin_branch, filter_name, filter_config, assert filter_name == jdoc["filter"] uri = "{}/pipeline?allow_duplicates=true&append_filter=true".format(quote(filter_user_svc_task)) - filters_in_pipeline = [filter_name] - conn.request("PUT", '/foglamp/filter/' + uri, json.dumps({"pipeline": filters_in_pipeline})) + print("existing_pipeline={}, existing_pipeline_format={}, filter_to_Add={}".format(existing_pipeline, + type(existing_pipeline), + filter_name)) + existing_pipeline.append(filter_name) + # filters_in_pipeline = [filter_name] + print("filters_in_pipeline={}".format(existing_pipeline)) + conn.request("PUT", '/foglamp/filter/' + uri, json.dumps({"pipeline": existing_pipeline})) r = conn.getresponse() assert 200 == r.status res = r.read().decode() diff --git a/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py new file mode 100644 index 0000000000..aefe808750 --- /dev/null +++ b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +""" Test end to end flow with: + Playback south plugin + Delta, RMS, Rate, Scale, Asset & Metadata filter plugins + PI Server (C) plugin +""" + + +import http.client +import os +import json +import time +import pytest + + +__author__ = "Vaibhav Singhal" +__copyright__ = "Copyright (c) 2019 Dianomic Systems" +__license__ = "Apache 2.0" +__version__ = "${VERSION}" + + +SOUTH_PLUGIN = "Expression" +SOUTH_PLUGIN_LANGUAGE = "C" + +SVC_NAME = "playfilter" +ASSET_NAME = "Expression" + + +CSV_NAME = "sample.csv" +CSV_HEADERS = "ivalue" +CSV_DATA = [{'ivalue': 10}, + {'ivalue': 20}, + {'ivalue': 30}] + +NORTH_TASK_NAME = "NorthReadingsTo_PI" + +_data_str = {} + + +class TestE2eCsvMultiFltrPi: + + @pytest.fixture + def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, remove_directories, + south_branch, foglamp_url, add_filter, filter_branch, filter_name, + start_north_pi_server_c, pi_host, pi_port, pi_token, asset_name="e2e_csv_filter_pi"): + """ This fixture clone a south and north repo and starts both south and north instance + + reset_and_start_foglamp: Fixture that resets and starts foglamp, no explicit invocation, called at start + add_south: Fixture that adds a south service with given configuration with enabled or disabled mode + remove_directories: Fixture that remove directories created during the tests + """ + + # Define configuration of foglamp south playback service + south_config = {"assetName": {"value": "{}".format(asset_name)}, + "csvFilename": {"value": "{}".format(CSV_NAME)}, + "ingestMode": {"value": "batch"}} + + # Define the CSV data and create expected lists to be verified later + csv_file_path = os.path.join(os.path.expandvars('${FOGLAMP_ROOT}'), 'data/{}'.format(CSV_NAME)) + f = open(csv_file_path, "w") + f.write(CSV_HEADERS) + _heads = CSV_HEADERS.split(",") + for c_data in CSV_DATA: + temp_data = [] + for _head in _heads: + temp_data.append(str(c_data[_head])) + row = ','.join(temp_data) + f.write("\n{}".format(row)) + f.close() + + # Prepare list of values for each header + for _head in _heads: + tmp_list = [] + for c_data in CSV_DATA: + tmp_list.append(c_data[_head]) + _data_str[_head] = tmp_list + + south_plugin = "playback" + add_south(south_plugin, south_branch, foglamp_url, service_name=SVC_NAME, + config=south_config, start_service=False) + + filter_cfg = {"enable": "true"} + # I/P 10, 20, 30 -> O/P 1000, 2000, 3000 + add_filter("scale", filter_branch, "fscale", filter_cfg, foglamp_url, SVC_NAME) + + # "asset_name": "e2e_csv_filter_pi", "action": "rename", "new_asset_name": "e2e_filters" + # I/P e2e_csv_filter_pi > O/P e2e_filters + add_filter("asset", filter_branch, "fasset", filter_cfg, foglamp_url, SVC_NAME) + add_filter("metadata", filter_branch, "fmeta", filter_cfg, foglamp_url, SVC_NAME) + + enable_schedule(foglamp_url, SVC_NAME) + + # FIXME: FOGL-2417 + # We need to make north PI sending process to handle the case, to send and retrieve applied filter data + # in running service, so that we don't need to add south service in disabled mode And enable after applying + # filter pipeline + start_north_pi_server_c(foglamp_url, pi_host, pi_port, pi_token) + + yield self.start_south_north + + remove_directories("/tmp/foglamp-south-{}".format(SOUTH_PLUGIN.lower())) + remove_directories("/tmp/foglamp-filter-{}".format("metadata")) + + def test_end_to_end(self, start_south_north, disable_schedule, foglamp_url, read_data_from_pi, pi_host, pi_admin, + pi_passwd, pi_db, wait_time, retries): + """ Test that data is inserted in FogLAMP using expression south plugin & metadata filter, and sent to PI + start_south_north: Fixture that starts FogLAMP with south service, add filter and north instance + Assertions: + on endpoint GET /foglamp/asset + on endpoint GET /foglamp/asset/ with applied data processing filter value + data received from PI is same as data sent""" + + time.sleep(wait_time) + conn = http.client.HTTPConnection(foglamp_url) + # self._verify_ingest(conn) + + # disable schedule to stop the service and sending data + disable_schedule(foglamp_url, SVC_NAME) + + # self._verify_egress(read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries) + + def _verify_ingest(self, conn): + + conn.request("GET", '/foglamp/asset') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 1 == len(jdoc) + assert ASSET_NAME == jdoc[0]["assetCode"] + assert 0 < jdoc[0]["count"] + + conn.request("GET", '/foglamp/asset/{}'.format(ASSET_NAME)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 0 < len(jdoc) + + # read = jdoc[0]["reading"] + # assert 1.61977519054386 == read["Expression"] + # # verify filter is applied and we have {name: value} pair added by metadata filter + # assert "value" == read["name"] + + def _verify_egress(self, read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries): + + retry_count = 0 + data_from_pi = None + while (data_from_pi is None or data_from_pi == []) and retry_count < retries: + data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, ASSET_NAME, {"Expression", "name"}) + retry_count += 1 + time.sleep(wait_time * 2) + + if data_from_pi is None or retry_count == retries: + assert False, "Failed to read data from PI" + + assert len(data_from_pi) + assert "name" in data_from_pi + assert "Expression" in data_from_pi + assert isinstance(data_from_pi["name"], list) + assert isinstance(data_from_pi["Expression"], list) + assert "value" in data_from_pi["name"] + assert 1.61977519054386 in data_from_pi["Expression"] From f7ae83d1349df5604e55f714274a26aa341d0c1b Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Fri, 15 Feb 2019 15:13:50 +0000 Subject: [PATCH 24/86] FOGL-2481 Make filters not visible in GUI --- python/foglamp/services/core/api/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/foglamp/services/core/api/filters.py b/python/foglamp/services/core/api/filters.py index 60ac749fee..8451003eb5 100644 --- a/python/foglamp/services/core/api/filters.py +++ b/python/foglamp/services/core/api/filters.py @@ -241,7 +241,7 @@ async def add_filters_pipeline(request: web.Request) -> web.Response: # Config update for filter pipeline and a change callback after category children creation await cf_mgr.set_category_item_value_entry(user_name, config_item, {'pipeline': new_list}) else: # No existing filters, hence create new item 'config_item' and add the "pipeline" array as a string - new_item = dict({config_item: {'description': 'Filter pipeline', 'type': 'JSON', 'default': {}}}) + new_item = dict({config_item: {'description': 'Filter pipeline', 'type': 'JSON', 'default': {}, 'readonly':'true'}}) new_item[config_item]['default'] = json.dumps({'pipeline': filter_list}) await _add_child_filters(storage, cf_mgr, user_name, filter_list) await cf_mgr.create_category(category_name=user_name, category_value=new_item, keep_original_items=True) From 714173cdd55fdca533e393cba12e70d975b38152 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Fri, 15 Feb 2019 15:55:09 +0000 Subject: [PATCH 25/86] FOGL-2481 Unit test fix --- tests/unit/python/foglamp/services/core/api/test_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/python/foglamp/services/core/api/test_filters.py b/tests/unit/python/foglamp/services/core/api/test_filters.py index 8385565004..1c42fe1c2f 100644 --- a/tests/unit/python/foglamp/services/core/api/test_filters.py +++ b/tests/unit/python/foglamp/services/core/api/test_filters.py @@ -583,7 +583,7 @@ async def test_add_filter_pipeline_without_filter_config(self, client): args, kwargs = _add_child_patch.call_args assert user == args[2] assert ['AssetFilter'] == args[3] - create_cat_patch.assert_called_once_with(category_name='bench', category_value={'filter': {'description': 'Filter pipeline', 'type': 'JSON', 'default': '{"pipeline": ["AssetFilter"]}'}}, keep_original_items=True) + create_cat_patch.assert_called_once_with(category_name='bench', category_value={'filter': {'description': 'Filter pipeline', 'readonly' : 'true', 'type': 'JSON', 'default': '{"pipeline": ["AssetFilter"]}'}}, keep_original_items=True) query_tbl_patch.assert_called_once_with('filters', '{"where": {"column": "name", "condition": "=", "value": "AssetFilter"}}') get_cat_info_patch.assert_called_once_with(category_name=user) From 4f3fe681c9de7d2af901cb466cb6cc653c6bb4db Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 18 Feb 2019 12:55:12 +0530 Subject: [PATCH 26/86] FOGL-2432 - refactored common utils module --- python/foglamp/common/utils.py | 20 +++++++++ .../foglamp/services/core/api/notification.py | 14 +++--- python/foglamp/services/core/api/service.py | 6 +-- python/foglamp/services/core/api/task.py | 7 ++- .../services/core/scheduler/scheduler.py | 13 +++--- .../tasks/statistics/statistics_history.py | 4 +- .../foglamp/common/test_common_utils.py | 44 +++++++++++++++++++ .../statistics/test_statistics_history.py | 4 +- 8 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 tests/unit/python/foglamp/common/test_common_utils.py diff --git a/python/foglamp/common/utils.py b/python/foglamp/common/utils.py index 85e71b0af6..63e4332c25 100644 --- a/python/foglamp/common/utils.py +++ b/python/foglamp/common/utils.py @@ -6,6 +6,7 @@ """Common utilities""" +import datetime __author__ = "Amarendra K Sinha" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" @@ -34,3 +35,22 @@ def check_reserved(string): if s in reserved: return False return True + + +def get_diff(old, new): + diff = list() + for key in new: + if key in old: + if old[key] != new[key]: + diff.append(key) + else: + diff.append(key) + return diff + + +def local_timestamp(): + """ + :return: str - current time stamp with microseconds and machine timezone info + :example '2018-05-08 14:06:40.517313+05:30' + """ + return str(datetime.datetime.now(datetime.timezone.utc).astimezone()) diff --git a/python/foglamp/services/core/api/notification.py b/python/foglamp/services/core/api/notification.py index 7e0e7f3667..6a8a1b746f 100644 --- a/python/foglamp/services/core/api/notification.py +++ b/python/foglamp/services/core/api/notification.py @@ -9,7 +9,7 @@ import aiohttp from aiohttp import web -from foglamp.common import utils +from foglamp.common import utils as common_utils from foglamp.common import logger from foglamp.common.service_record import ServiceRecord from foglamp.common.storage_client.exceptions import StorageServerError @@ -163,11 +163,11 @@ async def post_notification(request): if notification_type is None: raise ValueError('Missing notification_type property in payload.') - if utils.check_reserved(name) is False: + if common_utils.check_reserved(name) is False: raise ValueError('Invalid name property in payload.') - if utils.check_reserved(rule) is False: + if common_utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') - if utils.check_reserved(channel) is False: + if common_utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') @@ -270,11 +270,11 @@ async def put_notification(request): rule_config = data.get('rule_config', {}) delivery_config = data.get('delivery_config', {}) - if utils.check_reserved(notif) is False: + if common_utils.check_reserved(notif) is False: raise ValueError('Invalid notification name parameter.') - if rule is not None and utils.check_reserved(rule) is False: + if rule is not None and common_utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') - if channel is not None and utils.check_reserved(channel) is False: + if channel is not None and common_utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type is not None and notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') diff --git a/python/foglamp/services/core/api/service.py b/python/foglamp/services/core/api/service.py index df296b3f13..c2f23689cf 100644 --- a/python/foglamp/services/core/api/service.py +++ b/python/foglamp/services/core/api/service.py @@ -10,7 +10,7 @@ from aiohttp import web from typing import Dict -from foglamp.common import utils +from foglamp.common import utils as common_utils from foglamp.common import logger from foglamp.common.service_record import ServiceRecord from foglamp.common.storage_client.payload_builder import PayloadBuilder @@ -137,7 +137,7 @@ async def add_service(request): if name is None: raise web.HTTPBadRequest(reason='Missing name property in payload.') - if utils.check_reserved(name) is False: + if common_utils.check_reserved(name) is False: raise web.HTTPBadRequest(reason='Invalid name property in payload.') if service_type is None: @@ -150,7 +150,7 @@ async def add_service(request): if plugin is None and service_type == 'south': raise web.HTTPBadRequest(reason='Missing plugin property for type south in payload.') - if plugin and utils.check_reserved(plugin) is False: + if plugin and common_utils.check_reserved(plugin) is False: raise web.HTTPBadRequest(reason='Invalid plugin property in payload.') if enabled is not None: diff --git a/python/foglamp/services/core/api/task.py b/python/foglamp/services/core/api/task.py index 00ea6c790d..611f4bdd64 100644 --- a/python/foglamp/services/core/api/task.py +++ b/python/foglamp/services/core/api/task.py @@ -6,10 +6,9 @@ import datetime import uuid - from aiohttp import web -from foglamp.common import utils +from foglamp.common import utils as common_utils from foglamp.common import logger from foglamp.common.configuration_manager import ConfigurationManager from foglamp.common.storage_client.payload_builder import PayloadBuilder @@ -87,9 +86,9 @@ async def add_task(request): raise web.HTTPBadRequest(reason='Missing plugin property in payload.') if task_type is None: raise web.HTTPBadRequest(reason='Missing type property in payload.') - if utils.check_reserved(name) is False: + if common_utils.check_reserved(name) is False: raise web.HTTPBadRequest(reason='Invalid name property in payload.') - if utils.check_reserved(plugin) is False: + if common_utils.check_reserved(plugin) is False: raise web.HTTPBadRequest(reason='Invalid plugin property in payload.') if task_type not in ['north']: raise web.HTTPBadRequest(reason='Only north type is supported.') diff --git a/python/foglamp/services/core/scheduler/scheduler.py b/python/foglamp/services/core/scheduler/scheduler.py index be2a94679b..060711271e 100644 --- a/python/foglamp/services/core/scheduler/scheduler.py +++ b/python/foglamp/services/core/scheduler/scheduler.py @@ -17,18 +17,19 @@ import subprocess import signal from typing import List -from foglamp.common.configuration_manager import ConfigurationManager + from foglamp.common import logger +from foglamp.common import utils as common_utils from foglamp.common.audit_logger import AuditLogger -from foglamp.services.core.scheduler.entities import * -from foglamp.services.core.scheduler.exceptions import * from foglamp.common.storage_client.exceptions import * from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common.storage_client.storage_client import StorageClientAsync +from foglamp.common.configuration_manager import ConfigurationManager +from foglamp.services.core.scheduler.entities import * +from foglamp.services.core.scheduler.exceptions import * from foglamp.services.core.service_registry.service_registry import ServiceRegistry from foglamp.services.core.service_registry import exceptions as service_registry_exceptions from foglamp.services.common import utils -from foglamp.plugins.common import utils as timedateUtils __author__ = "Terris Linenbach, Amarendra K Sinha, Massimiliano Pinto" __copyright__ = "Copyright (c) 2017-2018 OSIsoft, LLC" @@ -264,7 +265,7 @@ async def _wait_for_task_completion(self, task_process: _TaskProcess) -> None: update_payload = PayloadBuilder() \ .SET(exit_code=exit_code, state=int(state), - end_time=str(timedateUtils.local_timestamp())) \ + end_time=str(common_utils.local_timestamp())) \ .WHERE(['id', '=', str(task_process.task_id)]) \ .payload() try: @@ -337,7 +338,7 @@ async def _start_task(self, schedule: _ScheduleRow) -> None: schedule_name=schedule.name, process_name=schedule.process_name, state=int(Task.State.RUNNING), - start_time=str(timedateUtils.local_timestamp())) \ + start_time=str(common_utils.local_timestamp())) \ .payload() try: self._logger.debug('Database command: %s', insert_payload) diff --git a/python/foglamp/tasks/statistics/statistics_history.py b/python/foglamp/tasks/statistics/statistics_history.py index 76a166376d..ea17edd853 100644 --- a/python/foglamp/tasks/statistics/statistics_history.py +++ b/python/foglamp/tasks/statistics/statistics_history.py @@ -15,7 +15,7 @@ from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common import logger from foglamp.common.process import FoglampProcess -from foglamp.plugins.common import utils +from foglamp.common import utils as common_utils __author__ = "Ori Shadmon, Ashish Jabble" @@ -63,7 +63,7 @@ async def run(self): 1. INSERT the delta between `value` and `previous_value` into statistics_history 2. UPDATE the previous_value in statistics table to be equal to statistics.value at snapshot """ - current_time = utils.local_timestamp() + current_time = common_utils.local_timestamp() results = await self._storage_async.query_tbl("statistics") payload = {"updates": []} for r in results['rows']: diff --git a/tests/unit/python/foglamp/common/test_common_utils.py b/tests/unit/python/foglamp/common/test_common_utils.py new file mode 100644 index 0000000000..d221907b04 --- /dev/null +++ b/tests/unit/python/foglamp/common/test_common_utils.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +""" Unit tests for common utils """ + +import pytest +from foglamp.common import utils as common_utils +from collections import Counter + + +@pytest.allure.feature("unit") +@pytest.allure.story("common", "utils") +class TestCommonUtils: + @pytest.mark.parametrize("test_string, expected", [ + ("Gabbar&Gang", False), + ("with;Sambha", False), + ("andothers,halkats", False), + ("@Rampur", False), + ("triedloot/arson", False), + ("For$Gold", False), + ("Andlot{more", False), + ("Andmore}", False), + ("Veeru+Jai", False), + ("Gaonwale,Thakur", False), + ("=resisted", False), + ("successfully:", False), + ("any attack!", True), + ]) + def test_check_reserved(self, test_string, expected): + actual = common_utils.check_reserved(test_string) + assert expected == actual + + @pytest.mark.parametrize("test_input_old, test_input_new, expected", [ + ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22, 'c': 33}, ['a', 'b', 'c']), + ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22, 'd': 44}, ['a', 'b', 'd']), + ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22}, ['a', 'b']), + ({'a': 1, 'b': 2, 'c': 3}, {'d': 11, 'e': 22, 'c': 33}, ['d', 'e', 'c']) + ]) + def test_get_diff(self, test_input_old, test_input_new, expected): + actual = common_utils.get_diff(test_input_old, test_input_new) + assert Counter(expected) == Counter(actual) diff --git a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py index e8729bc4d5..ae420dd2ec 100644 --- a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py +++ b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py @@ -15,7 +15,7 @@ from foglamp.common.storage_client.storage_client import StorageClientAsync from foglamp.tasks.statistics.statistics_history import StatisticsHistory from foglamp.common.process import FoglampProcess -from foglamp.plugins.common import utils +from foglamp.common import utils as common_utils __author__ = "Vaibhav Singhal" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" @@ -55,7 +55,7 @@ async def test_insert_into_stats_history(self): sh = StatisticsHistory() sh._storage_async = MagicMock(spec=StorageClientAsync) with patch.object(sh._storage_async, "insert_into_tbl", return_value=mock_coro(None)) as patch_storage: - ts = utils.local_timestamp() + ts = common_utils.local_timestamp() await sh._insert_into_stats_history(key='Bla', value=1, history_ts=ts) args, kwargs = patch_storage.call_args assert "statistics_history" == args[0] From af069c5bbdc5ea19079dfeef5c423558ac167007 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 18 Feb 2019 13:25:48 +0530 Subject: [PATCH 27/86] Feedback changes --- python/foglamp/common/process.py | 3 ++- tests/unit/python/foglamp/common/test_process.py | 2 +- tests/unit/python/foglamp/services/common/test_microservice.py | 2 +- tests/unit/python/foglamp/tasks/north/test_sending_process.py | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index e95596e037..38332dc05e 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -84,7 +84,8 @@ def __init__(self): self._name = getattr(namespace, 'name') self._core_management_host = getattr(namespace, 'address') self._core_management_port = getattr(namespace, 'port') - if self._core_management_port < 1 or self._core_management_port > 65535: + r = range(1, 65536) + if self._core_management_port not in r: raise ArgumentParserError("Invalid Port: {}".format(self._core_management_port)) for item in args: if item.startswith('--'): diff --git a/tests/unit/python/foglamp/common/test_process.py b/tests/unit/python/foglamp/common/test_process.py index 4f1e6223e9..2a56fdcd23 100644 --- a/tests/unit/python/foglamp/common/test_process.py +++ b/tests/unit/python/foglamp/common/test_process.py @@ -6,7 +6,7 @@ from unittest.mock import patch from foglamp.common.storage_client.storage_client import ReadingsStorageClientAsync, StorageClientAsync -from foglamp.common.process import FoglampProcess, SilentArgParse, ArgumentParserError +from foglamp.common.process import FoglampProcess, ArgumentParserError from foglamp.common.microservice_management_client.microservice_management_client import MicroserviceManagementClient diff --git a/tests/unit/python/foglamp/services/common/test_microservice.py b/tests/unit/python/foglamp/services/common/test_microservice.py index 2aa7307898..8592e198a8 100644 --- a/tests/unit/python/foglamp/services/common/test_microservice.py +++ b/tests/unit/python/foglamp/services/common/test_microservice.py @@ -2,7 +2,7 @@ import pytest import time -from unittest.mock import patch, MagicMock +from unittest.mock import patch from aiohttp import web import asyncio import sys diff --git a/tests/unit/python/foglamp/tasks/north/test_sending_process.py b/tests/unit/python/foglamp/tasks/north/test_sending_process.py index 854044ce9b..65ce3c76c5 100644 --- a/tests/unit/python/foglamp/tasks/north/test_sending_process.py +++ b/tests/unit/python/foglamp/tasks/north/test_sending_process.py @@ -18,7 +18,6 @@ from foglamp.common.audit_logger import AuditLogger from foglamp.common.storage_client.storage_client import StorageClientAsync, ReadingsStorageClientAsync from foglamp.tasks.north.sending_process import SendingProcess -from foglamp.common.process import FoglampProcess, SilentArgParse, ArgumentParserError from foglamp.common.microservice_management_client.microservice_management_client import MicroserviceManagementClient __author__ = "Stefano Simonelli" From a68065e2c40a1c2e62c1e62196465f57790ec668 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 18 Feb 2019 14:28:15 +0530 Subject: [PATCH 28/86] Feedback changes --- python/foglamp/common/process.py | 4 ++-- .../unit/python/foglamp/services/common/test_microservice.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/foglamp/common/process.py b/python/foglamp/common/process.py index 38332dc05e..14a394773a 100644 --- a/python/foglamp/common/process.py +++ b/python/foglamp/common/process.py @@ -28,8 +28,8 @@ def __init__(self, message): self.message = message def __str__(self): - format = '%(message)s' - return format % dict(message=self.message) + fmt = '%(message)s' + return fmt % dict(message=self.message) class SilentArgParse(argparse.ArgumentParser): diff --git a/tests/unit/python/foglamp/services/common/test_microservice.py b/tests/unit/python/foglamp/services/common/test_microservice.py index 8592e198a8..ca78182ddb 100644 --- a/tests/unit/python/foglamp/services/common/test_microservice.py +++ b/tests/unit/python/foglamp/services/common/test_microservice.py @@ -7,7 +7,7 @@ import asyncio import sys from foglamp.common.storage_client.storage_client import ReadingsStorageClientAsync, StorageClientAsync -from foglamp.common.process import FoglampProcess, SilentArgParse, ArgumentParserError +from foglamp.common.process import FoglampProcess from foglamp.services.common.microservice import FoglampMicroservice, _logger from foglamp.common.microservice_management_client.microservice_management_client import MicroserviceManagementClient From b22f603c2cce1fb98e118e5cab7fb765260e6eda Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 18 Feb 2019 14:56:35 +0530 Subject: [PATCH 29/86] Reverted refactoring in notification.py, service.py, task.py and statistics_history.py --- python/foglamp/services/core/api/notification.py | 14 +++++++------- python/foglamp/services/core/api/service.py | 6 +++--- python/foglamp/services/core/api/task.py | 7 ++++--- .../foglamp/tasks/statistics/statistics_history.py | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/python/foglamp/services/core/api/notification.py b/python/foglamp/services/core/api/notification.py index 6a8a1b746f..7e0e7f3667 100644 --- a/python/foglamp/services/core/api/notification.py +++ b/python/foglamp/services/core/api/notification.py @@ -9,7 +9,7 @@ import aiohttp from aiohttp import web -from foglamp.common import utils as common_utils +from foglamp.common import utils from foglamp.common import logger from foglamp.common.service_record import ServiceRecord from foglamp.common.storage_client.exceptions import StorageServerError @@ -163,11 +163,11 @@ async def post_notification(request): if notification_type is None: raise ValueError('Missing notification_type property in payload.') - if common_utils.check_reserved(name) is False: + if utils.check_reserved(name) is False: raise ValueError('Invalid name property in payload.') - if common_utils.check_reserved(rule) is False: + if utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') - if common_utils.check_reserved(channel) is False: + if utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') @@ -270,11 +270,11 @@ async def put_notification(request): rule_config = data.get('rule_config', {}) delivery_config = data.get('delivery_config', {}) - if common_utils.check_reserved(notif) is False: + if utils.check_reserved(notif) is False: raise ValueError('Invalid notification name parameter.') - if rule is not None and common_utils.check_reserved(rule) is False: + if rule is not None and utils.check_reserved(rule) is False: raise ValueError('Invalid rule property in payload.') - if channel is not None and common_utils.check_reserved(channel) is False: + if channel is not None and utils.check_reserved(channel) is False: raise ValueError('Invalid channel property in payload.') if notification_type is not None and notification_type not in NOTIFICATION_TYPE: raise ValueError('Invalid notification_type property in payload.') diff --git a/python/foglamp/services/core/api/service.py b/python/foglamp/services/core/api/service.py index c2f23689cf..df296b3f13 100644 --- a/python/foglamp/services/core/api/service.py +++ b/python/foglamp/services/core/api/service.py @@ -10,7 +10,7 @@ from aiohttp import web from typing import Dict -from foglamp.common import utils as common_utils +from foglamp.common import utils from foglamp.common import logger from foglamp.common.service_record import ServiceRecord from foglamp.common.storage_client.payload_builder import PayloadBuilder @@ -137,7 +137,7 @@ async def add_service(request): if name is None: raise web.HTTPBadRequest(reason='Missing name property in payload.') - if common_utils.check_reserved(name) is False: + if utils.check_reserved(name) is False: raise web.HTTPBadRequest(reason='Invalid name property in payload.') if service_type is None: @@ -150,7 +150,7 @@ async def add_service(request): if plugin is None and service_type == 'south': raise web.HTTPBadRequest(reason='Missing plugin property for type south in payload.') - if plugin and common_utils.check_reserved(plugin) is False: + if plugin and utils.check_reserved(plugin) is False: raise web.HTTPBadRequest(reason='Invalid plugin property in payload.') if enabled is not None: diff --git a/python/foglamp/services/core/api/task.py b/python/foglamp/services/core/api/task.py index 611f4bdd64..00ea6c790d 100644 --- a/python/foglamp/services/core/api/task.py +++ b/python/foglamp/services/core/api/task.py @@ -6,9 +6,10 @@ import datetime import uuid + from aiohttp import web -from foglamp.common import utils as common_utils +from foglamp.common import utils from foglamp.common import logger from foglamp.common.configuration_manager import ConfigurationManager from foglamp.common.storage_client.payload_builder import PayloadBuilder @@ -86,9 +87,9 @@ async def add_task(request): raise web.HTTPBadRequest(reason='Missing plugin property in payload.') if task_type is None: raise web.HTTPBadRequest(reason='Missing type property in payload.') - if common_utils.check_reserved(name) is False: + if utils.check_reserved(name) is False: raise web.HTTPBadRequest(reason='Invalid name property in payload.') - if common_utils.check_reserved(plugin) is False: + if utils.check_reserved(plugin) is False: raise web.HTTPBadRequest(reason='Invalid plugin property in payload.') if task_type not in ['north']: raise web.HTTPBadRequest(reason='Only north type is supported.') diff --git a/python/foglamp/tasks/statistics/statistics_history.py b/python/foglamp/tasks/statistics/statistics_history.py index ea17edd853..76a166376d 100644 --- a/python/foglamp/tasks/statistics/statistics_history.py +++ b/python/foglamp/tasks/statistics/statistics_history.py @@ -15,7 +15,7 @@ from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common import logger from foglamp.common.process import FoglampProcess -from foglamp.common import utils as common_utils +from foglamp.plugins.common import utils __author__ = "Ori Shadmon, Ashish Jabble" @@ -63,7 +63,7 @@ async def run(self): 1. INSERT the delta between `value` and `previous_value` into statistics_history 2. UPDATE the previous_value in statistics table to be equal to statistics.value at snapshot """ - current_time = common_utils.local_timestamp() + current_time = utils.local_timestamp() results = await self._storage_async.query_tbl("statistics") payload = {"updates": []} for r in results['rows']: From 5b31f3c6582ba9a384204a19c18c76e24889eee3 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 18 Feb 2019 14:58:46 +0530 Subject: [PATCH 30/86] Reverted refactoring in test_statistics_history.py --- .../foglamp/tasks/statistics/test_statistics_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py index ae420dd2ec..e8729bc4d5 100644 --- a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py +++ b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py @@ -15,7 +15,7 @@ from foglamp.common.storage_client.storage_client import StorageClientAsync from foglamp.tasks.statistics.statistics_history import StatisticsHistory from foglamp.common.process import FoglampProcess -from foglamp.common import utils as common_utils +from foglamp.plugins.common import utils __author__ = "Vaibhav Singhal" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" @@ -55,7 +55,7 @@ async def test_insert_into_stats_history(self): sh = StatisticsHistory() sh._storage_async = MagicMock(spec=StorageClientAsync) with patch.object(sh._storage_async, "insert_into_tbl", return_value=mock_coro(None)) as patch_storage: - ts = common_utils.local_timestamp() + ts = utils.local_timestamp() await sh._insert_into_stats_history(key='Bla', value=1, history_ts=ts) args, kwargs = patch_storage.call_args assert "statistics_history" == args[0] From 6be9bf692f11ae8bfb8bf13a68ff94426f0ed30e Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Mon, 18 Feb 2019 19:35:10 +0530 Subject: [PATCH 31/86] test with multiple filters and csv plugin added --- tests/system/python/conftest.py | 28 +---- .../e2e/test_e2e_csv_multi_filter_pi.py | 101 +++++++++--------- 2 files changed, 53 insertions(+), 76 deletions(-) diff --git a/tests/system/python/conftest.py b/tests/system/python/conftest.py index 6b4f56d844..509c71bf14 100644 --- a/tests/system/python/conftest.py +++ b/tests/system/python/conftest.py @@ -225,21 +225,6 @@ def _add_filter(filter_plugin, filter_plugin_branch, filter_name, filter_config, data = {"name": "{}".format(filter_name), "plugin": "{}".format(filter_plugin), "filter_config": filter_config} conn = http.client.HTTPConnection(foglamp_url) - # Get filters in pipeline - existing_pipeline = [] - conn.request("GET", '/foglamp/filter/{}/pipeline'.format(filter_user_svc_task)) - r = conn.getresponse() - res = r.read().decode() - if r.status == 404: - print("No pipeline exist") - print( - "existing_pipeline={}, existing_pipeline_format={}".format(existing_pipeline, type(existing_pipeline))) - else: - print("Pipeline exist") - jdoc = json.loads(res) - existing_pipeline = jdoc["result"]["pipeline"] - print("existing_pipeline={}, existing_pipeline_format={}".format(existing_pipeline, type(existing_pipeline))) - conn.request("POST", '/foglamp/filter', json.dumps(data)) r = conn.getresponse() assert 200 == r.status @@ -248,19 +233,14 @@ def _add_filter(filter_plugin, filter_plugin_branch, filter_name, filter_config, assert filter_name == jdoc["filter"] uri = "{}/pipeline?allow_duplicates=true&append_filter=true".format(quote(filter_user_svc_task)) - print("existing_pipeline={}, existing_pipeline_format={}, filter_to_Add={}".format(existing_pipeline, - type(existing_pipeline), - filter_name)) - existing_pipeline.append(filter_name) - # filters_in_pipeline = [filter_name] - print("filters_in_pipeline={}".format(existing_pipeline)) - conn.request("PUT", '/foglamp/filter/' + uri, json.dumps({"pipeline": existing_pipeline})) + filters_in_pipeline = [filter_name] + conn.request("PUT", '/foglamp/filter/' + uri, json.dumps({"pipeline": filters_in_pipeline})) r = conn.getresponse() assert 200 == r.status res = r.read().decode() - expected = "Filter pipeline {{'pipeline': ['{}']}} updated successfully".format(filter_name) jdoc = json.loads(res) - assert expected == jdoc["result"] + # Asset newly added filter exist in request's response + assert filter_name in jdoc["result"] return _add_filter diff --git a/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py index aefe808750..2864ae5da9 100644 --- a/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py +++ b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py @@ -24,35 +24,26 @@ __version__ = "${VERSION}" -SOUTH_PLUGIN = "Expression" -SOUTH_PLUGIN_LANGUAGE = "C" - SVC_NAME = "playfilter" -ASSET_NAME = "Expression" - - CSV_NAME = "sample.csv" CSV_HEADERS = "ivalue" -CSV_DATA = [{'ivalue': 10}, - {'ivalue': 20}, - {'ivalue': 30}] +CSV_DATA = "10,20,21,40" NORTH_TASK_NAME = "NorthReadingsTo_PI" -_data_str = {} - class TestE2eCsvMultiFltrPi: @pytest.fixture def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, remove_directories, - south_branch, foglamp_url, add_filter, filter_branch, filter_name, + remove_data_file, south_branch, foglamp_url, add_filter, filter_branch, start_north_pi_server_c, pi_host, pi_port, pi_token, asset_name="e2e_csv_filter_pi"): """ This fixture clone a south and north repo and starts both south and north instance reset_and_start_foglamp: Fixture that resets and starts foglamp, no explicit invocation, called at start add_south: Fixture that adds a south service with given configuration with enabled or disabled mode remove_directories: Fixture that remove directories created during the tests + remove_data_file: Fixture that remove data file created during the tests """ # Define configuration of foglamp south playback service @@ -64,51 +55,60 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, csv_file_path = os.path.join(os.path.expandvars('${FOGLAMP_ROOT}'), 'data/{}'.format(CSV_NAME)) f = open(csv_file_path, "w") f.write(CSV_HEADERS) - _heads = CSV_HEADERS.split(",") - for c_data in CSV_DATA: - temp_data = [] - for _head in _heads: - temp_data.append(str(c_data[_head])) - row = ','.join(temp_data) - f.write("\n{}".format(row)) + for _items in CSV_DATA.split(","): + f.write("\n{}".format(_items)) f.close() - # Prepare list of values for each header - for _head in _heads: - tmp_list = [] - for c_data in CSV_DATA: - tmp_list.append(c_data[_head]) - _data_str[_head] = tmp_list - south_plugin = "playback" add_south(south_plugin, south_branch, foglamp_url, service_name=SVC_NAME, config=south_config, start_service=False) filter_cfg = {"enable": "true"} - # I/P 10, 20, 30 -> O/P 1000, 2000, 3000 + # I/P 10, 20, 21, 40 -> O/P 1000, 2000, 2100, 4000 add_filter("scale", filter_branch, "fscale", filter_cfg, foglamp_url, SVC_NAME) - # "asset_name": "e2e_csv_filter_pi", "action": "rename", "new_asset_name": "e2e_filters" - # I/P e2e_csv_filter_pi > O/P e2e_filters - add_filter("asset", filter_branch, "fasset", filter_cfg, foglamp_url, SVC_NAME) + # I/P asset_name : e2e_csv_filter_pi > O/P e2e_filters + filter_cfg_asset = {"config": {"rules": [{"new_asset_name": "e2e_filters", + "action": "rename", + "asset_name": asset_name}]}, + "enable": "true"} + add_filter("asset", filter_branch, "fasset", filter_cfg_asset, foglamp_url, SVC_NAME) + + # I/P 1000, 2000, 2100, 4000 -> O/P 2000, 2100, 4000 + filter_cfg_rate = {"trigger": "ivalue > 1200", "untrigger": "ivalue < 1100", "preTrigger": "0", "enable": "true"} + add_filter("rate", filter_branch, "frate", filter_cfg_rate, foglamp_url, SVC_NAME) + + # I/P 1000, 2000, 2100, 40000 -> O/P 2000, 4000 + filter_cfg_delta = {"tolerance": "20", "enable": "true"} + add_filter("delta", filter_branch, "fdelta", filter_cfg_delta , foglamp_url, SVC_NAME) + + # I/P 2000, 4000 -> O/P rms=3162.2776601684, rms_peak=2000 + filter_cfg_rms = {"assetName": "%a_RMS", "samples": "2", "peak": "true", "enable": "true"} + add_filter("rms", filter_branch, "frms", filter_cfg_rms, foglamp_url, SVC_NAME) + add_filter("metadata", filter_branch, "fmeta", filter_cfg, foglamp_url, SVC_NAME) + # Since playback plugin reads all csv data at once, we cant keep it in enable mode before filter add + # enable service when all filters all applied enable_schedule(foglamp_url, SVC_NAME) - # FIXME: FOGL-2417 - # We need to make north PI sending process to handle the case, to send and retrieve applied filter data - # in running service, so that we don't need to add south service in disabled mode And enable after applying - # filter pipeline start_north_pi_server_c(foglamp_url, pi_host, pi_port, pi_token) yield self.start_south_north - remove_directories("/tmp/foglamp-south-{}".format(SOUTH_PLUGIN.lower())) + remove_directories("/tmp/foglamp-south-{}".format(south_plugin)) + remove_directories("/tmp/foglamp-filter-{}".format("scale")) + remove_directories("/tmp/foglamp-filter-{}".format("asset")) + remove_directories("/tmp/foglamp-filter-{}".format("rate")) + remove_directories("/tmp/foglamp-filter-{}".format("delta")) + remove_directories("/tmp/foglamp-filter-{}".format("rms")) remove_directories("/tmp/foglamp-filter-{}".format("metadata")) + remove_data_file(csv_file_path) def test_end_to_end(self, start_south_north, disable_schedule, foglamp_url, read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries): - """ Test that data is inserted in FogLAMP using expression south plugin & metadata filter, and sent to PI + """ Test that data is inserted in FogLAMP using playback south plugin & + Delta, RMS, Rate, Scale, Asset & Metadata filters, and sent to PI start_south_north: Fixture that starts FogLAMP with south service, add filter and north instance Assertions: on endpoint GET /foglamp/asset @@ -117,12 +117,12 @@ def test_end_to_end(self, start_south_north, disable_schedule, foglamp_url, read time.sleep(wait_time) conn = http.client.HTTPConnection(foglamp_url) - # self._verify_ingest(conn) + self._verify_ingest(conn) # disable schedule to stop the service and sending data disable_schedule(foglamp_url, SVC_NAME) - # self._verify_egress(read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries) + self._verify_egress(read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries) def _verify_ingest(self, conn): @@ -132,37 +132,34 @@ def _verify_ingest(self, conn): r = r.read().decode() jdoc = json.loads(r) assert 1 == len(jdoc) - assert ASSET_NAME == jdoc[0]["assetCode"] + assert "e2e_filters_RMS" == jdoc[0]["assetCode"] assert 0 < jdoc[0]["count"] - conn.request("GET", '/foglamp/asset/{}'.format(ASSET_NAME)) + conn.request("GET", '/foglamp/asset/{}'.format("e2e_filters_RMS")) r = conn.getresponse() assert 200 == r.status r = r.read().decode() jdoc = json.loads(r) assert 0 < len(jdoc) - # read = jdoc[0]["reading"] - # assert 1.61977519054386 == read["Expression"] - # # verify filter is applied and we have {name: value} pair added by metadata filter - # assert "value" == read["name"] + read = jdoc[0]["reading"] + assert 2000.0 == read["ivaluepeak"] + assert 3162.2776601684 == read["ivalue"] + assert "value" == read["name"] def _verify_egress(self, read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, wait_time, retries): retry_count = 0 data_from_pi = None while (data_from_pi is None or data_from_pi == []) and retry_count < retries: - data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, ASSET_NAME, {"Expression", "name"}) + data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, + "e2e_filters_RMS", {"ivalue", "ivaluepeak", "name"}) retry_count += 1 time.sleep(wait_time * 2) if data_from_pi is None or retry_count == retries: assert False, "Failed to read data from PI" - assert len(data_from_pi) - assert "name" in data_from_pi - assert "Expression" in data_from_pi - assert isinstance(data_from_pi["name"], list) - assert isinstance(data_from_pi["Expression"], list) - assert "value" in data_from_pi["name"] - assert 1.61977519054386 in data_from_pi["Expression"] + assert 3162.2776601684 == data_from_pi["ivalue"][-1] + assert 2000 == data_from_pi["ivaluepeak"][-1] + assert "value" == data_from_pi["name"][-1] From 035cd11d79066c6a66042adb431cf3621ea2a306 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Tue, 19 Feb 2019 11:15:20 +0530 Subject: [PATCH 32/86] Feedback changes --- python/foglamp/tasks/statistics/statistics_history.py | 5 ++--- .../foglamp/tasks/statistics/test_statistics_history.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/foglamp/tasks/statistics/statistics_history.py b/python/foglamp/tasks/statistics/statistics_history.py index 76a166376d..f6343c2880 100644 --- a/python/foglamp/tasks/statistics/statistics_history.py +++ b/python/foglamp/tasks/statistics/statistics_history.py @@ -15,8 +15,7 @@ from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common import logger from foglamp.common.process import FoglampProcess -from foglamp.plugins.common import utils - +from foglamp.common import utils as common_utils __author__ = "Ori Shadmon, Ashish Jabble" __copyright__ = "Copyright (c) 2017 OSI Soft, LLC" @@ -63,7 +62,7 @@ async def run(self): 1. INSERT the delta between `value` and `previous_value` into statistics_history 2. UPDATE the previous_value in statistics table to be equal to statistics.value at snapshot """ - current_time = utils.local_timestamp() + current_time = common_utils.local_timestamp() results = await self._storage_async.query_tbl("statistics") payload = {"updates": []} for r in results['rows']: diff --git a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py index e8729bc4d5..ae420dd2ec 100644 --- a/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py +++ b/tests/unit/python/foglamp/tasks/statistics/test_statistics_history.py @@ -15,7 +15,7 @@ from foglamp.common.storage_client.storage_client import StorageClientAsync from foglamp.tasks.statistics.statistics_history import StatisticsHistory from foglamp.common.process import FoglampProcess -from foglamp.plugins.common import utils +from foglamp.common import utils as common_utils __author__ = "Vaibhav Singhal" __copyright__ = "Copyright (c) 2017 OSIsoft, LLC" @@ -55,7 +55,7 @@ async def test_insert_into_stats_history(self): sh = StatisticsHistory() sh._storage_async = MagicMock(spec=StorageClientAsync) with patch.object(sh._storage_async, "insert_into_tbl", return_value=mock_coro(None)) as patch_storage: - ts = utils.local_timestamp() + ts = common_utils.local_timestamp() await sh._insert_into_stats_history(key='Bla', value=1, history_ts=ts) args, kwargs = patch_storage.call_args assert "statistics_history" == args[0] From 57b1f266e0bbbd6c68ac20314ff8c5803aaaf708 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Tue, 19 Feb 2019 12:53:26 +0530 Subject: [PATCH 33/86] Feedback changes --- python/foglamp/common/utils.py | 11 ----------- tests/unit/python/foglamp/common/test_common_utils.py | 10 ---------- 2 files changed, 21 deletions(-) diff --git a/python/foglamp/common/utils.py b/python/foglamp/common/utils.py index 63e4332c25..6a17b105a3 100644 --- a/python/foglamp/common/utils.py +++ b/python/foglamp/common/utils.py @@ -37,17 +37,6 @@ def check_reserved(string): return True -def get_diff(old, new): - diff = list() - for key in new: - if key in old: - if old[key] != new[key]: - diff.append(key) - else: - diff.append(key) - return diff - - def local_timestamp(): """ :return: str - current time stamp with microseconds and machine timezone info diff --git a/tests/unit/python/foglamp/common/test_common_utils.py b/tests/unit/python/foglamp/common/test_common_utils.py index d221907b04..9e841797e8 100644 --- a/tests/unit/python/foglamp/common/test_common_utils.py +++ b/tests/unit/python/foglamp/common/test_common_utils.py @@ -32,13 +32,3 @@ class TestCommonUtils: def test_check_reserved(self, test_string, expected): actual = common_utils.check_reserved(test_string) assert expected == actual - - @pytest.mark.parametrize("test_input_old, test_input_new, expected", [ - ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22, 'c': 33}, ['a', 'b', 'c']), - ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22, 'd': 44}, ['a', 'b', 'd']), - ({'a': 1, 'b': 2, 'c': 3}, {'a': 11, 'b': 22}, ['a', 'b']), - ({'a': 1, 'b': 2, 'c': 3}, {'d': 11, 'e': 22, 'c': 33}, ['d', 'e', 'c']) - ]) - def test_get_diff(self, test_input_old, test_input_new, expected): - actual = common_utils.get_diff(test_input_old, test_input_new) - assert Counter(expected) == Counter(actual) From bdf91e96f8d0eeaade9efb6a968c9bcb5b96fa48 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 20 Feb 2019 16:25:39 +0530 Subject: [PATCH 34/86] FOGL-2404 service notification system tests added (#1407) * service notification system tests added * feedback fixes * restructuring fixes * sent and receive notification tests are added --- tests/system/python/conftest.py | 53 +-- ...t_e2e_notification_service_with_plugins.py | 301 ++++++++++++++++++ tests/system/python/scripts/install_c_service | 38 +++ 3 files changed, 374 insertions(+), 18 deletions(-) create mode 100644 tests/system/python/e2e/test_e2e_notification_service_with_plugins.py create mode 100755 tests/system/python/scripts/install_c_service diff --git a/tests/system/python/conftest.py b/tests/system/python/conftest.py index d3be62394d..3152d8a70d 100644 --- a/tests/system/python/conftest.py +++ b/tests/system/python/conftest.py @@ -274,14 +274,35 @@ def _disable_sch(foglamp_url, sch_name): def pytest_addoption(parser): - parser.addoption("--south-branch", action="store", default="develop", - help="south branch name") - parser.addoption("--north-branch", action="store", default="develop", - help="north branch name") parser.addoption("--foglamp-url", action="store", default="localhost:8081", help="FogLAMP client api url") parser.addoption("--use-pip-cache", action="store", default=False, help="use pip cache is requirement is available") + parser.addoption("--wait-time", action="store", default=5, type=int, + help="Generic wait time between processes to run") + parser.addoption("--retries", action="store", default=3, type=int, + help="Number of tries for polling") + + # South/North Args + parser.addoption("--south-branch", action="store", default="develop", + help="south branch name") + parser.addoption("--north-branch", action="store", default="develop", + help="north branch name") + parser.addoption("--south-service-name", action="store", default="southSvc #1", + help="Name of the South Service") + parser.addoption("--asset-name", action="store", default="SystemTest", + help="Name of asset") + + # Filter Args + parser.addoption("--filter-branch", action="store", default="develop", help="Filter plugin repo branch") + parser.addoption("--filter-name", action="store", default="Meta #1", help="Filter name to be added to pipeline") + + # External Services Arg foglamp-service-* e.g. foglamp-service-notification + parser.addoption("--service-branch", action="store", default="develop", + help="service branch name") + # Notify Arg + parser.addoption("--notify-branch", action="store", default="develop", help="Notify plugin repo branch") + # PI Config parser.addoption("--pi-host", action="store", default="pi-server", help="PI Server Host Name/IP") @@ -308,20 +329,6 @@ def pytest_addoption(parser): parser.addoption("--ocs-token", action="store", default="ocs_north_0001", help="Token of OCS account") - parser.addoption("--south-service-name", action="store", default="southSvc #1", - help="Name of the South Service") - parser.addoption("--asset-name", action="store", default="SystemTest", - help="Name of asset") - - parser.addoption("--wait-time", action="store", default=5, type=int, - help="Generic wait time between processes to run") - parser.addoption("--retries", action="store", default=3, type=int, - help="Number of tries for polling") - - # Filter Args - parser.addoption("--filter-branch", action="store", default="develop", help="Filter plugin repo branch") - parser.addoption("--filter-name", action="store", default="Meta #1", help="Filter name to be added to pipeline") - # Kafka Config parser.addoption("--kafka-host", action="store", default="localhost", help="Kafka Server Host Name/IP") @@ -341,11 +348,21 @@ def north_branch(request): return request.config.getoption("--north-branch") +@pytest.fixture +def service_branch(request): + return request.config.getoption("--service-branch") + + @pytest.fixture def filter_branch(request): return request.config.getoption("--filter-branch") +@pytest.fixture +def notify_branch(request): + return request.config.getoption("--notify-branch") + + @pytest.fixture def use_pip_cache(request): return request.config.getoption("--use-pip-cache") diff --git a/tests/system/python/e2e/test_e2e_notification_service_with_plugins.py b/tests/system/python/e2e/test_e2e_notification_service_with_plugins.py new file mode 100644 index 0000000000..d81d5fc4b1 --- /dev/null +++ b/tests/system/python/e2e/test_e2e_notification_service_with_plugins.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +""" Test end to end flow with: + Notification service And OverMaxRule in built rule plugin + notify-python35 delivery plugin +""" + +import os +import time +import subprocess +import http.client +import json +import pytest + + +__author__ = "Ashish Jabble" +__copyright__ = "Copyright (c) 2019 Dianomic Systems" +__license__ = "Apache 2.0" +__version__ = "${VERSION}" + + +SERVICE = "notification" +SERVICE_NAME = "NotificationServer #1" +NOTIFY_PLUGIN = "python35" +NOTIFY_RULE = "OverMaxRule" + + +def _configure_and_start_service(service_branch, foglamp_url, remove_directories): + try: + subprocess.run(["$FOGLAMP_ROOT/tests/system/python/scripts/install_c_service {} {}" + .format(service_branch, SERVICE)], shell=True, check=True) + except subprocess.CalledProcessError: + assert False, "{} installation failed".format(SERVICE) + finally: + remove_directories("/tmp/foglamp-service-{}".format(SERVICE)) + + # Start service + conn = http.client.HTTPConnection(foglamp_url) + data = {"name": SERVICE_NAME, + "type": "notification", + "enabled": "true" + } + conn.request("POST", '/foglamp/service', json.dumps(data)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 2 == len(jdoc) + assert SERVICE_NAME == jdoc['name'] + + +def _install_notify_plugin(notify_branch, plugin_name, remove_directories): + try: + subprocess.run(["$FOGLAMP_ROOT/tests/system/python/scripts/install_c_plugin {} notify {}".format( + notify_branch, plugin_name)], shell=True, check=True) + except subprocess.CalledProcessError: + assert False, "{} installation failed".format(plugin_name) + finally: + remove_directories("/tmp/foglamp-notify-{}".format(plugin_name)) + + +def _get_result(foglamp_url, path): + conn = http.client.HTTPConnection(foglamp_url) + conn.request("GET", path) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + return jdoc + + +def _verify_service(foglamp_url, status): + jdoc = _get_result(foglamp_url, '/foglamp/service') + svc = jdoc['services'][2] + assert SERVICE_NAME == svc['name'] + assert SERVICE.capitalize() == svc['type'] + assert status == svc['status'] + + +def _verify_audit_log_entry(foglamp_url, path, name, severity='INFORMATION'): + jdoc = _get_result(foglamp_url, path) + audit_detail = jdoc['audit'][0] + assert 1 == jdoc['totalCount'] + assert severity == audit_detail['severity'] + assert name == audit_detail['details']['name'] + + +def _add_notification_instance(foglamp_url, payload): + conn = http.client.HTTPConnection(foglamp_url) + conn.request("POST", '/foglamp/notification', json.dumps(payload)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert "Notification {} created successfully".format(payload['name']) == jdoc['result'] + + +class TestNotificationService: + + def test_service(self, reset_and_start_foglamp, service_branch, foglamp_url, wait_time, retries, remove_directories): + _configure_and_start_service(service_branch, foglamp_url, remove_directories) + + retry_count = 0 + # only 2 services is being up by default i.e core and storage + default_registry_count = 2 + service_registry = default_registry_count + while service_registry != 3 and retry_count < retries: + svc = _get_result(foglamp_url, '/foglamp/service') + service_registry = svc['services'] + retry_count += 1 + time.sleep(wait_time * 2) + + if len(service_registry) == default_registry_count: + assert False, "Failed to start the {} service".format(SERVICE) + + _verify_service(foglamp_url, status='running') + + _verify_audit_log_entry(foglamp_url, '/foglamp/audit?source=NTFST', name=SERVICE_NAME) + + def test_get_default_notification_plugins(self, foglamp_url, remove_directories): + remove_directories(os.environ['FOGLAMP_ROOT'] + '/plugins/notificationDelivery') + remove_directories(os.environ['FOGLAMP_ROOT'] + 'cmake_build/C/plugins/notificationDelivery') + jdoc = _get_result(foglamp_url, '/foglamp/notification/plugin') + assert [] == jdoc['delivery'] + assert 1 == len(jdoc['rules']) + assert NOTIFY_RULE == jdoc['rules'][0]['name'] + + +class TestNotificationCRUD: + + # FIXME: FOGL-2434 Add name with some special character + @pytest.mark.parametrize("data", [ + {"name": "Test1", "description": "Test 1 notification", "rule": NOTIFY_RULE, + "channel": NOTIFY_PLUGIN, "enabled": "false", "notification_type": "retriggered"}, + {"name": "Test2", "description": "Test 2 notification", "rule": NOTIFY_RULE, + "channel": NOTIFY_PLUGIN, "enabled": "false", "notification_type": "toggled"}, + {"name": "Test3", "description": "Test 3 notification", "rule": NOTIFY_RULE, + "channel": NOTIFY_PLUGIN, "enabled": "false", "notification_type": "one shot"} + ]) + def test_create_notification_instances_with_default_rule_and_channel_python35(self, foglamp_url, notify_branch, + data, + remove_directories): + # FIXME: Handle in a better way; we need below code once for a test + if data['name'] == 'Test1': + _install_notify_plugin(notify_branch, NOTIFY_PLUGIN, remove_directories) + _add_notification_instance(foglamp_url, data) + + def test_inbuilt_rule_plugin_and_notify_python35_delivery(self, foglamp_url): + jdoc = _get_result(foglamp_url, '/foglamp/notification/plugin') + assert 1 == len(jdoc['delivery']) + assert NOTIFY_PLUGIN == jdoc['delivery'][0]['name'] + assert 1 == len(jdoc['rules']) + assert NOTIFY_RULE == jdoc['rules'][0]['name'] + + def test_get_notifications_and_audit_entry(self, foglamp_url): + jdoc = _get_result(foglamp_url, '/foglamp/notification') + assert 3 == len(jdoc['notifications']) + + jdoc = _get_result(foglamp_url, '/foglamp/audit?source=NTFAD') + assert 3 == jdoc['totalCount'] + + def test_update_notification(self, foglamp_url, name="Test1"): + conn = http.client.HTTPConnection(foglamp_url) + data = {"notification_type": "toggled"} + conn.request("PUT", '/foglamp/notification/{}'.format(name), json.dumps(data)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert "Notification {} updated successfully".format(name) == jdoc["result"] + + # Verify updated notification info + jdoc = _get_result(foglamp_url, '/foglamp/notification/{}'.format(name)) + assert "toggled" == jdoc['notification']['notificationType'] + + def test_delete_notification(self, foglamp_url, name="Test3"): + conn = http.client.HTTPConnection(foglamp_url) + conn.request("DELETE", '/foglamp/notification/{}'.format(name)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert "Notification {} deleted successfully.".format(name) == jdoc["result"] + + # Verify only two notifications should exist NOT 3 + jdoc = _get_result(foglamp_url, '/foglamp/notification') + notifications = jdoc['notifications'] + assert 2 == len(notifications) + assert "Notification Test1" == notifications[0]['name'] + assert "Notification Test2" == notifications[1]['name'] + + +class TestSentAndReceiveNotification: + FOGBENCH_TEMPLATE = "fogbench-template.json" + SENSOR_VALUE = 20 + SOUTH_PLUGIN_NAME = "coap" + ASSET_NAME = "{}".format(SOUTH_PLUGIN_NAME) + + @pytest.fixture + def start_south(self, add_south, remove_data_file, remove_directories, south_branch, foglamp_url, wait_time): + """ This fixture clone a south repo and starts south instance + add_south: Fixture that starts any south service with given configuration + remove_data_file: Fixture that remove data file created during the tests + remove_directories: Fixture that remove directories created during the tests """ + + fogbench_template_path = self.prepare_template_reading_from_fogbench() + + add_south(self.SOUTH_PLUGIN_NAME, south_branch, foglamp_url, service_name=self.SOUTH_PLUGIN_NAME) + + yield self.start_south + + # Cleanup code that runs after the test is over + remove_data_file(fogbench_template_path) + remove_directories("/tmp/foglamp-south-{}".format(self.SOUTH_PLUGIN_NAME)) + + def prepare_template_reading_from_fogbench(self): + """ Define the template file for fogbench readings """ + + fogbench_template_path = os.path.join( + os.path.expandvars('${FOGLAMP_ROOT}'), 'data/{}'.format(self.FOGBENCH_TEMPLATE)) + with open(fogbench_template_path, "w") as f: + f.write( + '[{"name": "%s", "sensor_values": ' + '[{"name": "sensor", "type": "number", "min": %d, "max": %d, "precision": 0}]}]' % ( + self.ASSET_NAME, self.SENSOR_VALUE, self.SENSOR_VALUE)) + + return fogbench_template_path + + def ingest_readings_from_fogbench(self, foglamp_url, wait_time): + conn = http.client.HTTPConnection(foglamp_url) + subprocess.run(["cd $FOGLAMP_ROOT/extras/python; python3 -m fogbench -t ../../data/{}; cd -" + .format(self.FOGBENCH_TEMPLATE)], shell=True, check=True) + time.sleep(wait_time) + conn.request("GET", '/foglamp/asset') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + val = json.loads(r) + assert 1 == len(val) + assert self.ASSET_NAME == val[0]["assetCode"] + assert 1 == val[0]["count"] + + conn.request("GET", '/foglamp/asset/{}'.format(self.ASSET_NAME)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + val = json.loads(r) + assert 1 == len(val) + assert {'sensor': self.SENSOR_VALUE} == val[0]["reading"] + + def configure_rule_with_latest_eval_type(self, foglamp_url, cat_name): + conn = http.client.HTTPConnection(foglamp_url) + data = {"asset": self.ASSET_NAME, + "datapoint": "sensor", + "evaluation_type": "latest", + "trigger_value": str(self.SENSOR_VALUE), + } + conn.request("PUT", '/foglamp/category/rule{}'.format(cat_name), json.dumps(data)) + r = conn.getresponse() + assert 200 == r.status + + def enable_notification(self, foglamp_url, cat_name, is_enabled=True): + _enabled = "true" if is_enabled else "false" + data = {"value": _enabled} + conn = http.client.HTTPConnection(foglamp_url) + conn.request("PUT", '/foglamp/category/{}/enable'.format(cat_name), json.dumps(data)) + r = conn.getresponse() + assert 200 == r.status + + def test_sent_and_receive_notification(self, foglamp_url, start_south, wait_time): + data = {"name": "Test4", "description": "Test4_Notification", "rule": NOTIFY_RULE, "channel": NOTIFY_PLUGIN, + "enabled": False, "notification_type": "retriggered"} + name = data['name'] + _add_notification_instance(foglamp_url, data) + self.configure_rule_with_latest_eval_type(foglamp_url, name) + self.enable_notification(foglamp_url, name) + + time.sleep(wait_time) + self.ingest_readings_from_fogbench(foglamp_url, wait_time) + + _verify_audit_log_entry(foglamp_url, '/foglamp/audit?source=NTFSN', name=name) + + +class TestStartStopNotificationService: + + def test_shutdown_service_with_schedule_disable(self, foglamp_url, disable_schedule, wait_time): + disable_schedule(foglamp_url, SERVICE_NAME) + + _verify_service(foglamp_url, status='shutdown') + time.sleep(wait_time) + _verify_audit_log_entry(foglamp_url, '/foglamp/audit?source=NTFSD', name=SERVICE_NAME) + + def test_restart_notification_service(self, foglamp_url, enable_schedule): + enable_schedule(foglamp_url, SERVICE_NAME) + + _verify_service(foglamp_url, status='running') + _verify_audit_log_entry(foglamp_url, '/foglamp/audit?source=NTFST', name=SERVICE_NAME) diff --git a/tests/system/python/scripts/install_c_service b/tests/system/python/scripts/install_c_service new file mode 100755 index 0000000000..26254c8416 --- /dev/null +++ b/tests/system/python/scripts/install_c_service @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -e + +__author__="Ashish Jabble" +__copyright__="Copyright (c) 2019 Dianomic Systems" +__license__="Apache 2.0" +__version__="1.0.0" + +########################################################################################## +# Usage text for this script +# $FOGLAMP_ROOT/tests/system/python/scripts/install_c_service {BRANCH_NAME} {SERVICE_NAME} +########################################################################################## + +BRANCH_NAME=$1 +SERVICE_NAME=$2 + +[[ -z "${BRANCH_NAME}" ]] && echo "Branch name not found." && exit 1 +[[ -z "${SERVICE_NAME}" ]] && echo "Service name not found." && exit 1 + +REPO_NAME=foglamp-service-${SERVICE_NAME} + +clean () { + rm -rf /tmp/${REPO_NAME} + rm -rf ${FOGLAMP_ROOT}/services/foglamp.services.${SERVICE_NAME} +} + +clone_repo () { + git clone -b ${BRANCH_NAME} --single-branch https://github.com/foglamp/${REPO_NAME}.git /tmp/${REPO_NAME} +} + +install_binary_file () { + mkdir -p /tmp/${REPO_NAME}/build; cd /tmp/${REPO_NAME}/build; cmake -DFOGLAMP_INSTALL=${FOGLAMP_ROOT} ..; make -j4 && make install; cd - +} + +clean +clone_repo +install_binary_file +echo "${SERVICE_NAME} service is installed." From 712557e99680828b7457b9dfa6717e794f5237d1 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 20 Feb 2019 17:13:57 +0530 Subject: [PATCH 35/86] requirement.sh added --- README.rst | 2 +- requirement.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 requirement.sh diff --git a/README.rst b/README.rst index 26e39e6cee..98a3b5b844 100644 --- a/README.rst +++ b/README.rst @@ -69,7 +69,7 @@ FogLAMP is currently based on C/C++ and Python code. The packages needed to buil - sqlite3 - libsqlite3-dev -On Ubuntu-based Linux distributions the packages can be installed with *apt-get*: +On Ubuntu-based Linux distributions the packages can be installed with given `requirement.sh `_ or manual *apt-get*: :: apt-get install avahi-daemon curl apt-get install cmake g++ make build-essential autoconf automake uuid-dev diff --git a/requirement.sh b/requirement.sh new file mode 100755 index 0000000000..aea36960a4 --- /dev/null +++ b/requirement.sh @@ -0,0 +1,31 @@ +#!//usr/bin/env bash + +##-------------------------------------------------------------------- +## Copyright (c) 2019 Dianomic Systems +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +##-------------------------------------------------------------------- + +## +## Author: Ashish Jabble +## + + +set -e + +sudo apt-get install avahi-daemon curl +sudo apt-get install cmake g++ make build-essential autoconf automake uuid-dev +sudo apt-get install libtool libboost-dev libboost-system-dev libboost-thread-dev libpq-dev libssl-dev libz-dev +sudo apt-get install python-dbus python-dev python3-dev python3-pip +sudo apt-get install postgresql +sudo apt-get install sqlite3 libsqlite3-dev From aeb28ac1e3d2a6eea7cc91d6caea34ac0d6653b8 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 20 Feb 2019 17:55:40 +0530 Subject: [PATCH 36/86] feedback fixes --- README.rst | 2 +- requirement.sh => requirements.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename requirement.sh => requirements.sh (97%) diff --git a/README.rst b/README.rst index 98a3b5b844..3f85e22ae9 100644 --- a/README.rst +++ b/README.rst @@ -69,7 +69,7 @@ FogLAMP is currently based on C/C++ and Python code. The packages needed to buil - sqlite3 - libsqlite3-dev -On Ubuntu-based Linux distributions the packages can be installed with given `requirement.sh `_ or manual *apt-get*: +On Ubuntu-based Linux distributions the packages can be installed with given `requirements.sh `_ or manual *apt-get*: :: apt-get install avahi-daemon curl apt-get install cmake g++ make build-essential autoconf automake uuid-dev diff --git a/requirement.sh b/requirements.sh similarity index 97% rename from requirement.sh rename to requirements.sh index aea36960a4..5a8865ca8e 100755 --- a/requirement.sh +++ b/requirements.sh @@ -27,5 +27,5 @@ sudo apt-get install avahi-daemon curl sudo apt-get install cmake g++ make build-essential autoconf automake uuid-dev sudo apt-get install libtool libboost-dev libboost-system-dev libboost-thread-dev libpq-dev libssl-dev libz-dev sudo apt-get install python-dbus python-dev python3-dev python3-pip -sudo apt-get install postgresql sudo apt-get install sqlite3 libsqlite3-dev +# sudo apt-get install postgresql From cb79fc96609655ed9f75ce6afdef5add718b194d Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 21 Feb 2019 12:33:36 +0530 Subject: [PATCH 37/86] FOGL-2413 - Fixed python_plugin_interface.cpp -> logErrorMessage() to correctly display exception messages --- .../python/python_plugin_interface.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp index 191301b9a9..749713d9fc 100644 --- a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp +++ b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp @@ -611,15 +611,15 @@ static void logErrorMessage() //Get error message PyObject *pType, *pValue, *pTraceback; PyErr_Fetch(&pType, &pValue, &pTraceback); + PyErr_NormalizeException(&pType, &pValue, &pTraceback); - // NOTE from : - // https://docs.python.org/2/c-api/exceptions.html - // - // The value and traceback object may be NULL - // even when the type object is not. - const char* pErrorMessage = pValue ? - PyBytes_AsString(pValue) : - "no error description."; + PyObject* str_exc_type = PyObject_Repr(pType); + PyObject* pyStr = PyUnicode_AsEncodedString(str_exc_type, "utf-8", "Error ~"); + const char *strExcType = PyBytes_AS_STRING(pyStr); + + PyObject* str_exc_value = PyObject_Repr(pValue); + PyObject* pyExcValueStr = PyUnicode_AsEncodedString(str_exc_value, "utf-8", "Error ~"); + const char *pErrorMessage = PyBytes_AS_STRING(pyExcValueStr); Logger::getLogger()->fatal("logErrorMessage: Error '%s' ", pErrorMessage ? From 9efb57593887ad67059b01eb199beaf850039359 Mon Sep 17 00:00:00 2001 From: Ashwin Gopalakrishnan Date: Thu, 21 Feb 2019 06:35:12 -0800 Subject: [PATCH 38/86] Updated Version History Yfor 1.5.1 --- docs/91_version_history.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/91_version_history.rst b/docs/91_version_history.rst index 00f7369a65..4a7ded010d 100644 --- a/docs/91_version_history.rst +++ b/docs/91_version_history.rst @@ -30,6 +30,14 @@ v1.5.1 Release Date: 2019-03-12 +- **FogLAMP Core** + + - Bug Fix: plugin loading errors + + +- **GUI** + + - Bug Fix: uptime shows up to 24 hour clock only v1.5.0 From 9b2c5a39778580763fd8e9d8299ec273297009dd Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 22 Feb 2019 13:11:56 +0530 Subject: [PATCH 39/86] Feedback changes --- .../python/python_plugin_interface.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp index 749713d9fc..ac3f145341 100644 --- a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp +++ b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp @@ -613,10 +613,6 @@ static void logErrorMessage() PyErr_Fetch(&pType, &pValue, &pTraceback); PyErr_NormalizeException(&pType, &pValue, &pTraceback); - PyObject* str_exc_type = PyObject_Repr(pType); - PyObject* pyStr = PyUnicode_AsEncodedString(str_exc_type, "utf-8", "Error ~"); - const char *strExcType = PyBytes_AS_STRING(pyStr); - PyObject* str_exc_value = PyObject_Repr(pValue); PyObject* pyExcValueStr = PyUnicode_AsEncodedString(str_exc_value, "utf-8", "Error ~"); const char *pErrorMessage = PyBytes_AS_STRING(pyExcValueStr); From 42e4360a96565844a29a11058c3c7664d4ab7703 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 22 Feb 2019 13:22:37 +0530 Subject: [PATCH 40/86] Feedback changes --- .../python/python_plugin_interface.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp index ac3f145341..b937a6e609 100644 --- a/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp +++ b/C/services/south-plugin-interfaces/python/python_plugin_interface.cpp @@ -615,12 +615,10 @@ static void logErrorMessage() PyObject* str_exc_value = PyObject_Repr(pValue); PyObject* pyExcValueStr = PyUnicode_AsEncodedString(str_exc_value, "utf-8", "Error ~"); - const char *pErrorMessage = PyBytes_AS_STRING(pyExcValueStr); - - Logger::getLogger()->fatal("logErrorMessage: Error '%s' ", - pErrorMessage ? - pErrorMessage : - "no description"); + const char* pErrorMessage = pValue ? + PyBytes_AsString(pyExcValueStr) : + "no error description."; + Logger::getLogger()->fatal("logErrorMessage: Error '%s' ", pErrorMessage); // Reset error PyErr_Clear(); @@ -629,6 +627,8 @@ static void logErrorMessage() Py_CLEAR(pType); Py_CLEAR(pValue); Py_CLEAR(pTraceback); + Py_CLEAR(str_exc_value); + Py_CLEAR(pyExcValueStr); } }; From cd7ca8efb30a2e79a3676eda67d372a3b2486341 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Fri, 22 Feb 2019 16:53:33 +0530 Subject: [PATCH 41/86] Fix shim.py->plugin_init() to retain JSON type items as str and not dict --- python/foglamp/plugins/common/shim/shim.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/foglamp/plugins/common/shim/shim.py b/python/foglamp/plugins/common/shim/shim.py index 5bef308239..e239ed5713 100644 --- a/python/foglamp/plugins/common/shim/shim.py +++ b/python/foglamp/plugins/common/shim/shim.py @@ -43,7 +43,18 @@ def plugin_info(): def plugin_init(config): _LOGGER.info("plugin_init called") handle = _plugin.plugin_init(json.loads(config)) - return handle + # South C server sends "config" argument as string in which all JSON type items' components, + # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore + # them to str, which is the required format for configuration items. + revised_handle = {} + for k, v in handle.items(): + if v['type'] == 'JSON': + if isinstance(v['default'], dict): + v['default'] = json.dumps(v['default']) + if isinstance(v['value'], dict): + v['value'] = json.dumps(v['value']) + revised_handle.update({k: v}) + return revised_handle def plugin_poll(handle): From 4edd388c3aaeb916b40da2fe57c054d595a24c8b Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Fri, 22 Feb 2019 18:08:31 +0530 Subject: [PATCH 42/86] -y added to apt install --- requirements.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.sh b/requirements.sh index 5a8865ca8e..3dea715be3 100755 --- a/requirements.sh +++ b/requirements.sh @@ -23,9 +23,9 @@ set -e -sudo apt-get install avahi-daemon curl -sudo apt-get install cmake g++ make build-essential autoconf automake uuid-dev -sudo apt-get install libtool libboost-dev libboost-system-dev libboost-thread-dev libpq-dev libssl-dev libz-dev -sudo apt-get install python-dbus python-dev python3-dev python3-pip -sudo apt-get install sqlite3 libsqlite3-dev -# sudo apt-get install postgresql +sudo apt install -y avahi-daemon curl +sudo apt install -y cmake g++ make build-essential autoconf automake uuid-dev +sudo apt install -y libtool libboost-dev libboost-system-dev libboost-thread-dev libpq-dev libssl-dev libz-dev +sudo apt install -y python-dbus python-dev python3-dev python3-pip +sudo apt install -y sqlite3 libsqlite3-dev +# sudo apt install -y postgresql From 406ee67684d6d3dff536e4408599a0fc1eb23ef5 Mon Sep 17 00:00:00 2001 From: Amarendra Date: Fri, 22 Feb 2019 21:19:26 +0530 Subject: [PATCH 43/86] Update shim.py --- python/foglamp/plugins/common/shim/shim.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/foglamp/plugins/common/shim/shim.py b/python/foglamp/plugins/common/shim/shim.py index e239ed5713..08bfa04628 100644 --- a/python/foglamp/plugins/common/shim/shim.py +++ b/python/foglamp/plugins/common/shim/shim.py @@ -46,6 +46,7 @@ def plugin_init(config): # South C server sends "config" argument as string in which all JSON type items' components, # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore # them to str, which is the required format for configuration items. + # TODO: FOGL-1827 - Config item value must be respected as per type given revised_handle = {} for k, v in handle.items(): if v['type'] == 'JSON': From 314b36fe6a6b4b3689a19b41e5dbf7474b9a947e Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 25 Feb 2019 15:21:47 +0530 Subject: [PATCH 44/86] Fix JSON type configuration items for plugin_reconfigure() in shim layer --- python/foglamp/plugins/common/shim/shim.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python/foglamp/plugins/common/shim/shim.py b/python/foglamp/plugins/common/shim/shim.py index 08bfa04628..78eb3d099a 100644 --- a/python/foglamp/plugins/common/shim/shim.py +++ b/python/foglamp/plugins/common/shim/shim.py @@ -66,7 +66,19 @@ def plugin_poll(handle): def plugin_reconfigure(handle, new_config): _LOGGER.info("plugin_reconfigure") new_handle = _plugin.plugin_reconfigure(handle, json.loads(new_config)) - return new_handle + # South C server sends "config" argument as string in which all JSON type items' components, + # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore + # them to str, which is the required format for configuration items. + # TODO: FOGL-1827 - Config item value must be respected as per type given + revised_handle = {} + for k, v in new_handle.items(): + if v['type'] == 'JSON': + if isinstance(v['default'], dict): + v['default'] = json.dumps(v['default']) + if isinstance(v['value'], dict): + v['value'] = json.dumps(v['value']) + revised_handle.update({k: v}) + return revised_handle def plugin_shutdown(handle): From 56276cbabbaeb36d7b3b2ed3dc0aae14a36069cb Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Mon, 25 Feb 2019 15:43:16 +0530 Subject: [PATCH 45/86] Common code refactored --- python/foglamp/plugins/common/shim/shim.py | 39 ++++++++++------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/python/foglamp/plugins/common/shim/shim.py b/python/foglamp/plugins/common/shim/shim.py index 78eb3d099a..c156b564fb 100644 --- a/python/foglamp/plugins/common/shim/shim.py +++ b/python/foglamp/plugins/common/shim/shim.py @@ -43,18 +43,8 @@ def plugin_info(): def plugin_init(config): _LOGGER.info("plugin_init called") handle = _plugin.plugin_init(json.loads(config)) - # South C server sends "config" argument as string in which all JSON type items' components, - # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore - # them to str, which is the required format for configuration items. # TODO: FOGL-1827 - Config item value must be respected as per type given - revised_handle = {} - for k, v in handle.items(): - if v['type'] == 'JSON': - if isinstance(v['default'], dict): - v['default'] = json.dumps(v['default']) - if isinstance(v['value'], dict): - v['value'] = json.dumps(v['value']) - revised_handle.update({k: v}) + revised_handle = _revised_config_for_json_item(handle) return revised_handle @@ -66,18 +56,8 @@ def plugin_poll(handle): def plugin_reconfigure(handle, new_config): _LOGGER.info("plugin_reconfigure") new_handle = _plugin.plugin_reconfigure(handle, json.loads(new_config)) - # South C server sends "config" argument as string in which all JSON type items' components, - # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore - # them to str, which is the required format for configuration items. # TODO: FOGL-1827 - Config item value must be respected as per type given - revised_handle = {} - for k, v in new_handle.items(): - if v['type'] == 'JSON': - if isinstance(v['default'], dict): - v['default'] = json.dumps(v['default']) - if isinstance(v['value'], dict): - v['value'] = json.dumps(v['value']) - revised_handle.update({k: v}) + revised_handle = _revised_config_for_json_item(new_handle) return revised_handle @@ -94,3 +74,18 @@ def plugin_start(handle): def plugin_register_ingest(handle, callback, ingest_ref): _LOGGER.info("plugin_register_ingest") return _plugin.plugin_register_ingest(handle, callback, ingest_ref) + + +def _revised_config_for_json_item(config): + # South C server sends "config" argument as string in which all JSON type items' components, + # 'default' and 'value', gets converted to dict during json.loads(). Hence we need to restore + # them to str, which is the required format for configuration items. + revised_config_handle = {} + for k, v in config.items(): + if v['type'] == 'JSON': + if isinstance(v['default'], dict): + v['default'] = json.dumps(v['default']) + if isinstance(v['value'], dict): + v['value'] = json.dumps(v['value']) + revised_config_handle.update({k: v}) + return revised_config_handle From 96c87866097ef6f0cdd52301d3f411f0375e78f6 Mon Sep 17 00:00:00 2001 From: Praveen Garg Date: Tue, 26 Feb 2019 13:05:41 +0530 Subject: [PATCH 46/86] added update upgrade commands to requirements.sh --- requirements.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.sh b/requirements.sh index 3dea715be3..1330f6ad45 100755 --- a/requirements.sh +++ b/requirements.sh @@ -23,6 +23,9 @@ set -e +sudo apt update +sudo apt -y upgrade + sudo apt install -y avahi-daemon curl sudo apt install -y cmake g++ make build-essential autoconf automake uuid-dev sudo apt install -y libtool libboost-dev libboost-system-dev libboost-thread-dev libpq-dev libssl-dev libz-dev From ac63c27e4286cee0c3137112851e53bde367ad04 Mon Sep 17 00:00:00 2001 From: stefano Date: Wed, 27 Feb 2019 16:02:14 +0100 Subject: [PATCH 47/86] FOGL-2331: applied microseconds and TZ changes to sqlite in memory --- C/plugins/storage/sqlite/connection.cpp | 4 + C/plugins/storage/sqlitememory/connection.cpp | 667 ++++++++++++++---- .../storage/sqlitememory/include/connection.h | 3 +- 3 files changed, 531 insertions(+), 143 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index eed12bf546..ae572df06c 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -1773,6 +1773,8 @@ bool add_row = false; * Fetch a block of readings from the reading table * It might not work with SQLite 3 * + * Fetch, used by the north side, returns timestamp in UTC. + * * NOTE : it expects to handle a date having a fixed format * with milliseconds, microseconds and timezone expressed, * like for example : @@ -1854,6 +1856,8 @@ int retrieve; /** * Perform a query against the readings table * + * retrieveReadings, used by the API, returns timestamp in localtime. + * */ bool Connection::retrieveReadings(const string& condition, string& resultSet) { diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index 2fb752cbfe..ea84c89080 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -39,13 +39,18 @@ using namespace rapidjson; #define _DB_NAME "/foglamp.sqlite" -#define F_TIMEH24_S "%H:%M:%S" -#define F_DATEH24_S "%Y-%m-%d %H:%M:%S" -#define F_DATEH24_M "%Y-%m-%d %H:%M" -#define F_DATEH24_H "%Y-%m-%d %H" +#define LEN_BUFFER_DATE 100 +#define F_TIMEH24_S "%H:%M:%S" +#define F_DATEH24_S "%Y-%m-%d %H:%M:%S" +#define F_DATEH24_M "%Y-%m-%d %H:%M" +#define F_DATEH24_H "%Y-%m-%d %H" // This is the default datetime format in FogLAMP: 2018-05-03 18:15:00.622 -#define F_DATEH24_MS "%Y-%m-%d %H:%M:%f" -#define SQLITE3_NOW "strftime('%Y-%m-%d %H:%M:%f', 'now', 'localtime')" +#define F_DATEH24_MS "%Y-%m-%d %H:%M:%f" +// Format up to seconds +#define F_DATEH24_SEC "%Y-%m-%d %H:%M:%S" +#define SQLITE3_NOW "strftime('%Y-%m-%d %H:%M:%f', 'now', 'localtime')" +// The default precision is milliseconds, it adds microseconds and timezone +#define SQLITE3_NOW_READING "strftime('%Y-%m-%d %H:%M:%f000+00:00', 'now')" #define SQLITE3_FOGLAMP_DATETIME_TYPE "DATETIME" static time_t connectErrorTime = 0; map sqliteDateFormat = { @@ -106,82 +111,118 @@ bool Connection::applyColumnDateTimeFormat(sqlite3_stmt *pStmt, int i, string& newDate) { - /** - * Handle here possible unformatted DATETIME column type - * If (column_name == column_original_name) AND - * (sqlite3_column_table_name() == "DATETIME") - * we assume the column has not been formatted - * by any datetime() or strftime() SQLite function. - * Thus we apply default FOGLAMP formatting: - * "%Y-%m-%d %H:%M:%f" with 'localtime' - */ + + bool apply_format = false; + string formatStmt = {}; if (sqlite3_column_database_name(pStmt, i) != NULL && - sqlite3_column_table_name(pStmt, i) != NULL && - (strcmp(sqlite3_column_origin_name(pStmt, i), - sqlite3_column_name(pStmt, i)) == 0)) + sqlite3_column_table_name(pStmt, i) != NULL) { - const char* pzDataType; - int retType = sqlite3_table_column_metadata(inMemory, - sqlite3_column_database_name(pStmt, i), - sqlite3_column_table_name(pStmt, i), - sqlite3_column_name(pStmt, i), - &pzDataType, - NULL, NULL, NULL, NULL); - - // Check whether to Apply dateformat - if (pzDataType != NULL && - retType == SQLITE_OK && - strcmp(pzDataType, SQLITE3_FOGLAMP_DATETIME_TYPE) == 0 && - strcmp(sqlite3_column_origin_name(pStmt, i), - sqlite3_column_name(pStmt, i)) == 0) - { - // Column metadata found and column datatype is "pzDataType" - string formatStmt = string("SELECT strftime('"); - formatStmt += string(F_DATEH24_MS); - formatStmt += "', '" + string((char *)sqlite3_column_text(pStmt, i)); - formatStmt += "')"; - char* zErrMsg = NULL; - // New formatted data - char formattedData[100] = ""; + if ((strcmp(sqlite3_column_origin_name(pStmt, i), "user_ts") == 0) && + (strcmp(sqlite3_column_table_name(pStmt, i), "readings") == 0) && + (strlen((char *) sqlite3_column_text(pStmt, i)) == 32)) + { - // Exec the format SQL - int rc = SQLexec(inMemory, - formatStmt.c_str(), - dateCallback, - formattedData, - &zErrMsg); + // Extract milliseconds and microseconds for the user_ts field of the readings table + formatStmt = string("SELECT strftime('"); + formatStmt += string(F_DATEH24_SEC); + formatStmt += "', '" + string((char *) sqlite3_column_text(pStmt, i)); + formatStmt += "')"; + formatStmt += " || substr('" + string((char *) sqlite3_column_text(pStmt, i)); + formatStmt += "', instr('" + string((char *) sqlite3_column_text(pStmt, i)); + formatStmt += "', '.'), 7)"; - if (rc == SQLITE_OK ) + apply_format = true; + } + else + { + /** + * Handle here possible unformatted DATETIME column type + * If (column_name == column_original_name) AND + * (sqlite3_column_table_name() == "DATETIME") + * we assume the column has not been formatted + * by any datetime() or strftime() SQLite function. + * Thus we apply default FOGLAMP formatting: + * "%Y-%m-%d %H:%M:%f" + */ + if (sqlite3_column_database_name(pStmt, i) != NULL && + sqlite3_column_table_name(pStmt, i) != NULL && + (strcmp(sqlite3_column_origin_name(pStmt, i), + sqlite3_column_name(pStmt, i)) == 0)) { - // Use new formatted datetime value - newDate.assign(formattedData); + const char *pzDataType; + int retType = sqlite3_table_column_metadata(inMemory, + sqlite3_column_database_name(pStmt, i), + sqlite3_column_table_name(pStmt, i), + sqlite3_column_name(pStmt, i), + &pzDataType, + NULL, NULL, NULL, NULL); + + // Check whether to Apply dateformat + if (pzDataType != NULL && + retType == SQLITE_OK && + strcmp(pzDataType, SQLITE3_FOGLAMP_DATETIME_TYPE) == 0 && + strcmp(sqlite3_column_origin_name(pStmt, i), + sqlite3_column_name(pStmt, i)) == 0) + { + // Column metadata found and column datatype is "pzDataType" + formatStmt = string("SELECT strftime('"); + formatStmt += string(F_DATEH24_MS); + formatStmt += "', '" + string((char *) sqlite3_column_text(pStmt, i)); + formatStmt += "')"; - return true; - } - else - { - Logger::getLogger()->error("SELECT dateformat '%s': error %s", - formatStmt.c_str(), - zErrMsg); + apply_format = true; - sqlite3_free(zErrMsg); + } + else + { + // Format not done + // Just log the error if present + if (retType != SQLITE_OK) + { + Logger::getLogger()->error("SQLite3 failed " \ + "to call sqlite3_table_column_metadata() " \ + "for column '%s'", + sqlite3_column_name(pStmt, i)); + } + } } } + } + + if (apply_format) + { + + char* zErrMsg = NULL; + // New formatted data + char formattedData[100] = ""; + + // Exec the format SQL + int rc = SQLexec(inMemory, + formatStmt.c_str(), + dateCallback, + formattedData, + &zErrMsg); + + if (rc == SQLITE_OK ) + { + // Use new formatted datetime value + newDate.assign(formattedData); + + return true; + } else { - // Format not done - // Just log the error if present - if (retType != SQLITE_OK) - { - Logger::getLogger()->error("SQLite3 memory failed " \ - "to call sqlite3_table_column_metadata() " \ - "for column '%s'", - sqlite3_column_name(pStmt, i)); - } + Logger::getLogger()->error("SELECT dateformat '%s': error %s", + formatStmt.c_str(), + zErrMsg); + + sqlite3_free(zErrMsg); } + } + return false; } @@ -233,7 +274,68 @@ bool retCode; outFormat.append(colName); } - outFormat.append(")"); + outFormat.append(", 'localtime')"); // MR TRY THIS + retCode = true; + } + else + { + // Use column as is + outFormat.append(colName); + retCode = false; + } + + return retCode; +} + +/** + * Apply the specified date format + * using the available formats in SQLite3 + * for a specific column + * + * If the requested format is not availble + * the input column is used as is. + * Additionally milliseconds could be rounded + * upon request. + * The routine return false if datwe format is not + * found and the caller might decide to raise an error + * or use the non formatted value + * + * @param inFormat Input date format from application + * @param colName The column name to format + * @param outFormat The formatted column + * @return True if format has been applied or + * false id no format is in use. + */ +static bool applyColumnDateFormatLocaltime(const string& inFormat, + const string& colName, + string& outFormat, + bool roundMs = false) + +{ +bool retCode; + // Get format, if any, from the supported formats map + const string format = sqliteDateFormat[inFormat]; + if (!format.empty()) + { + // Apply found format via SQLite3 strftime() + outFormat.append("strftime('"); + outFormat.append(format); + outFormat.append("', "); + + // Check whether we have to round milliseconds + if (roundMs == true && + format.back() == 'f') + { + outFormat.append("cast(round((julianday("); + outFormat.append(colName); + outFormat.append(") - 2440587.5)*86400 -0.00005, 3) AS FLOAT), 'unixepoch'"); + } + else + { + outFormat.append(colName); + } + + outFormat.append(", 'localtime')"); // MR force localtime retCode = true; } else @@ -287,13 +389,14 @@ Connection::Connection() */ const char *inMemoryConn = "file:?cache=shared"; + // UTC time as default const char * createReadings = "CREATE TABLE foglamp.readings (" \ "id INTEGER PRIMARY KEY AUTOINCREMENT," \ "asset_code character varying(50) NOT NULL," \ "read_key uuid UNIQUE," \ "reading JSON NOT NULL DEFAULT '{}'," \ - "user_ts DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime'))," \ - "ts DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW', 'localtime'))" \ + "user_ts DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f+00:00', 'NOW' ))," \ + "ts DATETIME DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f+00:00', 'NOW' ))" \ ");"; const char * createReadingsFk = "CREATE INDEX fki_readings_fk1 ON readings (asset_code);"; @@ -398,7 +501,7 @@ unsigned long nRows = 0, nCols = 0; // Iterate over all the rows in the resultSet while ((rc = sqlite3_step(pStmt)) == SQLITE_ROW) { - // Get number of columns foir current row + // Get number of columns for current row nCols = sqlite3_column_count(pStmt); // Create the 'row' object Value row(kObjectType); @@ -549,11 +652,12 @@ int *nRows = (int *)data; } /** - * Perform a query against a common table + * Perform a query against the readings table + * + * retrieveReadings, used by the API, returns timestamp in localtime. * */ -bool Connection::retrieveReadings(const string& condition, - string& resultSet) +bool Connection::retrieveReadings(const string& condition, string& resultSet) { // Default template parameter uses UTF8 and MemoryPoolAllocator. Document document; @@ -561,6 +665,11 @@ SQLBuffer sql; // Extra constraints to add to where clause SQLBuffer jsonConstraints; + // FIXME: + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG retrieveReadings"); + + try { if (inMemory == NULL) { @@ -570,7 +679,21 @@ SQLBuffer jsonConstraints; if (condition.empty()) { - sql.append("SELECT * FROM readings"); + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG condition empty"); + + const char *sql_cmd = R"( + SELECT + id, + asset_code, + read_key, + reading, + strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, + strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + FROM readings)"; + + sql.append(sql_cmd); } else { @@ -652,24 +775,95 @@ SQLBuffer jsonConstraints; return false; } // SQLite3 doesnt support time zone formatting - if (strcasecmp((*itr)["timezone"].GetString(), "utc") != 0) + const char *tz = (*itr)["timezone"].GetString(); + + if (strncasecmp(tz, "utc", 3) == 0) + { + if (strcmp((*itr)["column"].GetString() ,"user_ts") == 0) + { + // Extract milliseconds and microseconds for the user_ts fields + + sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'utc') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } + } + else + { + sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append((*itr)["column"].GetString()); + sql.append(", 'utc')"); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } + } + } + else if (strncasecmp(tz, "localtime", 9) == 0) + { + if (strcmp((*itr)["column"].GetString() ,"user_ts") == 0) + { + // Extract milliseconds and microseconds for the user_ts fields + + sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } + } + else + { + sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append((*itr)["column"].GetString()); + sql.append(", 'localtime')"); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } + } + } + else { raiseError("retrieve", - "SQLite3 Memory plugin does not support timezones in qeueries"); + "SQLite3 plugin does not support timezones in queries"); return false; } + + } + else + { + + if (strcmp((*itr)["column"].GetString() ,"user_ts") == 0) + { + // Extract milliseconds and microseconds for the user_ts fields + + sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } + } else { sql.append("strftime('%Y-%m-%d %H:%M:%f', "); sql.append((*itr)["column"].GetString()); - sql.append(", 'utc')"); - sql.append(" AS "); - sql.append((*itr)["column"].GetString()); + sql.append(", 'localtime')"); + if (! itr->HasMember("alias")) + { + sql.append(" AS "); + sql.append((*itr)["column"].GetString()); + } } - } - else - { - sql.append((*itr)["column"].GetString()); + } sql.append(' '); } @@ -699,13 +893,27 @@ SQLBuffer jsonConstraints; } else { + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG condition NO"); + + sql.append("SELECT "); if (document.HasMember("modifier")) { sql.append(document["modifier"].GetString()); sql.append(' '); } - sql.append(" * FROM foglamp."); + const char *sql_cmd = R"( + id, + asset_code, + read_key, + reading, + strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, + strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + FROM foglamp.)"; + + sql.append(sql_cmd); } sql.append("readings"); if (document.HasMember("where")) @@ -778,6 +986,123 @@ SQLBuffer jsonConstraints; } } + +/** + * Format a date to a fixed format with milliseconds, microseconds and + * timezone expressed, examples : + * + * case - formatted |2019-01-01 10:01:01.000000+00:00| date |2019-01-01 10:01:01| + * case - formatted |2019-02-01 10:02:01.000000+00:00| date |2019-02-01 10:02:01.0| + * case - formatted |2019-02-02 10:02:02.841000+00:00| date |2019-02-02 10:02:02.841| + * case - formatted |2019-02-03 10:02:03.123456+00:00| date |2019-02-03 10:02:03.123456| + * case - formatted |2019-03-01 10:03:01.100000+00:00| date |2019-03-01 10:03:01.1+00:00| + * case - formatted |2019-03-02 10:03:02.123000+00:00| date |2019-03-02 10:03:02.123+00:00| + * case - formatted |2019-03-03 10:03:03.123456+00:00| date |2019-03-03 10:03:03.123456+00:00| + * case - formatted |2019-03-04 10:03:04.123456+01:00| date |2019-03-04 10:03:04.123456+01:00| + * case - formatted |2019-03-05 10:03:05.123456-01:00| date |2019-03-05 10:03:05.123456-01:00| + * case - formatted |2019-03-04 10:03:04.123456+02:30| date |2019-03-04 10:03:04.123456+02:30| + * case - formatted |2019-03-05 10:03:05.123456-02:30| date |2019-03-05 10:03:05.123456-02:30| + * + * @param out false if the date is invalid + * + */ +bool Connection::formatDate(char *formatted_date, size_t buffer_size, const char *date) { + + struct timeval tv = {0}; + struct tm tm = {0}; + char *valid_date = nullptr; + + // Extract up to seconds + memset(&tm, 0, sizeof(tm)); + valid_date = strptime(date, "%Y-%m-%d %H:%M:%S", &tm); + + if (! valid_date) + { + return (false); + } + + strftime (formatted_date, buffer_size, "%Y-%m-%d %H:%M:%S", &tm); + + // Work out the microseconds from the fractional part of the seconds + char fractional[10] = {0}; + sscanf(date, "%*d-%*d-%*d %*d:%*d:%*d.%[0-9]*", fractional); + // Truncate to max 6 digits + fractional[6] = 0; + int multiplier = 6 - (int)strlen(fractional); + if (multiplier < 0) + multiplier = 0; + while (multiplier--) + strcat(fractional, "0"); + + strcat(formatted_date ,"."); + strcat(formatted_date ,fractional); + + // Handles timezone + char timezone_hour[5] = {0}; + char timezone_min[5] = {0}; + char sign[2] = {0}; + + sscanf(date, "%*d-%*d-%*d %*d:%*d:%*d.%*d-%2[0-9]:%2[0-9]", timezone_hour, timezone_min); + if (timezone_hour[0] != 0) + { + strcat(sign, "-"); + } + else + { + memset(timezone_hour, 0, sizeof(timezone_hour)); + memset(timezone_min, 0, sizeof(timezone_min)); + + sscanf(date, "%*d-%*d-%*d %*d:%*d:%*d.%*d+%2[0-9]:%2[0-9]", timezone_hour, timezone_min); + if (timezone_hour[0] != 0) + { + strcat(sign, "+"); + } + else + { + // No timezone is expressed in the source date + // the default UTC is added + strcat(formatted_date, "+00:00"); + } + } + + if (sign[0] != 0) + { + if (timezone_hour[0] != 0) + { + strcat(formatted_date, sign); + + // Pad with 0 if an hour having only 1 digit was provided + // +1 -> +01 + if (strlen(timezone_hour) == 1) + strcat(formatted_date, "0"); + + strcat(formatted_date, timezone_hour); + strcat(formatted_date, ":"); + } + + if (timezone_min[0] != 0) + { + strcat(formatted_date, timezone_min); + + // Pad with 0 if minutes having only 1 digit were provided + // 3 -> 30 + if (strlen(timezone_min) == 1) + strcat(formatted_date, "0"); + + } + else + { + // Minutes aren't expressed in the source date + strcat(formatted_date, "00"); + } + } + + + return (true); + + +} + /** * Append a set of readings to the readings table */ @@ -787,6 +1112,7 @@ int Connection::appendReadings(const char *readings) Document doc; SQLBuffer sql; int row = 0; +bool add_row = false; ParseResult ok = doc.Parse(readings); if (!ok) @@ -795,7 +1121,7 @@ int row = 0; return -1; } - sql.append("INSERT INTO foglamp.readings ( asset_code, read_key, reading, user_ts ) VALUES "); + sql.append("INSERT INTO foglamp.readings ( user_ts, asset_code, read_key, reading ) VALUES "); if (!doc.HasMember("readings")) { @@ -816,48 +1142,81 @@ int row = 0; "Each reading in the readings array must be an object"); return -1; } - if (row) - { - sql.append(", ("); - } - else - { - sql.append('('); - } - row++; - sql.append('\''); - sql.append((*itr)["asset_code"].GetString()); - // Python code is passing the string None when here is no read_key in the payload - if (itr->HasMember("read_key") && strcmp((*itr)["read_key"].GetString(), "None") != 0) - { - sql.append("', \'"); - sql.append((*itr)["read_key"].GetString()); - sql.append("', \'"); - } - else - { - // No "read_key" in this reading, insert NULL - sql.append("', NULL, '"); - } - StringBuffer buffer; - Writer writer(buffer); - (*itr)["reading"].Accept(writer); - sql.append(buffer.GetString()); - sql.append("\', "); + add_row = true; + + // Handles - user_ts const char *str = (*itr)["user_ts"].GetString(); if (strcmp(str, "now()") == 0) { - sql.append(SQLITE3_NOW); + if (row) + { + sql.append(", ("); + } + else + { + sql.append('('); + } + + sql.append(SQLITE3_NOW_READING); } else { + char formatted_date[LEN_BUFFER_DATE] = {0}; + if (! formatDate(formatted_date, sizeof(formatted_date), str) ) + { + raiseError("appendReadings", "Invalid date |%s|", str); + add_row = false; + } + else + { + if (row) + { + sql.append(", ("); + } + else + { + sql.append('('); + } + + sql.append('\''); + sql.append(formatted_date); + sql.append('\''); + } + } + + if (add_row) + { + row++; + + // Handles - asset_code + sql.append(",\'"); + sql.append((*itr)["asset_code"].GetString()); + + // Handles - read_key + // Python code is passing the string None when here is no read_key in the payload + if (itr->HasMember("read_key") && strcmp((*itr)["read_key"].GetString(), "None") != 0) + { + sql.append("', \'"); + sql.append((*itr)["read_key"].GetString()); + sql.append("', \'"); + } + else + { + // No "read_key" in this reading, insert NULL + sql.append("', NULL, '"); + } + + // Handles - reading + StringBuffer buffer; + Writer writer(buffer); + (*itr)["reading"].Accept(writer); + sql.append(buffer.GetString()); sql.append('\''); - sql.append(escape(str)); - sql.append('\''); + + sql.append(')'); } - sql.append(')'); } sql.append(';'); @@ -894,32 +1253,50 @@ int row = 0; /** * Fetch a block of readings from the reading table + * + * Fetch, used by the north side, returns timestamp in UTC. + * + * NOTE : it expects to handle a date having a fixed format + * with milliseconds, microseconds and timezone expressed, + * like for example : + * + * 2019-01-11 15:45:01.123456+01:00 */ bool Connection::fetchReadings(unsigned long id, unsigned int blksize, std::string& resultSet) { -char sqlbuffer[300]; +char sqlbuffer[1100]; char *zErrMsg = NULL; int rc; int retrieve; + // FIXME: + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG fetchReadings"); + + // SQL command to extract the data from the foglamp.readings + const char *sql_cmd = R"( + SELECT + id, + asset_code, + read_key, + reading, + strftime('%%Y-%%m-%%d %%H:%%M:%%S', user_ts, 'utc') || + substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, + strftime('%%Y-%%m-%%d %%H:%%M:%%f', ts, 'utc') AS ts + FROM foglamp.readings + WHERE id >= %lu + ORDER BY id ASC + LIMIT %u; + )"; + /* * This query assumes datetime values are in 'localtime' */ - snprintf(sqlbuffer, sizeof(sqlbuffer), - "SELECT id, " \ - "asset_code, " \ - "read_key, " \ - "reading, " \ - "strftime('%%Y-%%m-%%d %%H:%%M:%%f', user_ts, 'utc') AS \"user_ts\", " \ - "strftime('%%Y-%%m-%%d %%H:%%M:%%f', ts, 'utc') AS \"ts\" " \ - "FROM foglamp.readings " \ - "WHERE id >= %lu " \ - "ORDER BY id ASC " \ - "LIMIT %u;", + sql_cmd, id, blksize); @@ -1540,7 +1917,7 @@ bool Connection::jsonAggregates(const Value& payload, } /** - * Process the modifers for limit, skip, sort and group + * Process the modifiers for limit, skip, sort and group */ bool Connection::jsonModifiers(const Value& payload, SQLBuffer& sql) { @@ -1718,7 +2095,7 @@ bool Connection::jsonModifiers(const Value& payload, SQLBuffer& sql) * */ bool Connection::jsonWhereClause(const Value& whereClause, - SQLBuffer& sql) + SQLBuffer& sql, bool convertLocaltime) { if (!whereClause.IsObject()) { @@ -1755,7 +2132,10 @@ bool Connection::jsonWhereClause(const Value& whereClause, } sql.append("< datetime('now', '-"); sql.append(whereClause["value"].GetInt()); - sql.append(" seconds', 'localtime')"); + if (convertLocaltime) + sql.append(" seconds', 'localtime')"); // Get value in localtime + else + sql.append(" seconds')"); // Get value in UTC by asking for no timezone } else if (!cond.compare("newer")) { @@ -1767,7 +2147,10 @@ bool Connection::jsonWhereClause(const Value& whereClause, } sql.append("> datetime('now', '-"); sql.append(whereClause["value"].GetInt()); - sql.append(" seconds', 'localtime')"); + if (convertLocaltime) + sql.append(" seconds', 'localtime')"); // Get value in localtime + else + sql.append(" seconds')"); // Get value in UTC by asking for no timezone } else if (!cond.compare("in") || !cond.compare("not in")) { @@ -1779,8 +2162,8 @@ bool Connection::jsonWhereClause(const Value& whereClause, sql.append(" ( "); int field = 0; for (Value::ConstValueIterator itr = whereClause["value"].Begin(); - itr != whereClause["value"].End(); - ++itr) + itr != whereClause["value"].End(); + ++itr) { if (field) { @@ -1811,9 +2194,9 @@ bool Connection::jsonWhereClause(const Value& whereClause, else { string message("The \"value\" of a \"" + \ - cond + \ - "\" condition array element must be " \ - "a string, integer or double."); + cond + \ + "\" condition array element must be " \ + "a string, integer or double."); raiseError("where clause", message.c_str()); return false; } @@ -1839,7 +2222,7 @@ bool Connection::jsonWhereClause(const Value& whereClause, } else if (whereClause["value"].IsString()) { sql.append('\''); - sql.append(whereClause["value"].GetString()); + sql.append(escape(whereClause["value"].GetString())); sql.append('\''); } } @@ -1847,7 +2230,7 @@ bool Connection::jsonWhereClause(const Value& whereClause, if (whereClause.HasMember("and")) { sql.append(" AND "); - if (!jsonWhereClause(whereClause["and"], sql)) + if (!jsonWhereClause(whereClause["and"], sql, convertLocaltime)) { return false; } @@ -1855,7 +2238,7 @@ bool Connection::jsonWhereClause(const Value& whereClause, if (whereClause.HasMember("or")) { sql.append(" OR "); - if (!jsonWhereClause(whereClause["or"], sql)) + if (!jsonWhereClause(whereClause["or"], sql, convertLocaltime)) { return false; } diff --git a/C/plugins/storage/sqlitememory/include/connection.h b/C/plugins/storage/sqlitememory/include/connection.h index a50e5c9bfe..4b29eba097 100644 --- a/C/plugins/storage/sqlitememory/include/connection.h +++ b/C/plugins/storage/sqlitememory/include/connection.h @@ -36,7 +36,7 @@ class Connection { void raiseError(const char *operation, const char *reason,...); sqlite3 *inMemory; // Handle for :memory: database int mapResultSet(void *res, std::string& resultSet); - bool jsonWhereClause(const rapidjson::Value& whereClause, SQLBuffer&); + bool jsonWhereClause(const rapidjson::Value& whereClause, SQLBuffer&, bool convertLocaltime = false); bool jsonModifiers(const rapidjson::Value&, SQLBuffer&); bool jsonAggregates(const rapidjson::Value&, const rapidjson::Value&, @@ -50,5 +50,6 @@ class Connection { int i, std::string& newDate); void logSQL(const char *, const char *); + bool formatDate(char *formatted_date, size_t formatted_date_size, const char *date); }; #endif From 682667745d65453f6ee09531104b11edaec8d11b Mon Sep 17 00:00:00 2001 From: stefano Date: Wed, 27 Feb 2019 17:14:44 +0100 Subject: [PATCH 48/86] FOGL-2331: fixed aggregate handling. --- C/plugins/storage/sqlitememory/include/connection.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/C/plugins/storage/sqlitememory/include/connection.h b/C/plugins/storage/sqlitememory/include/connection.h index 4b29eba097..0c222e6fec 100644 --- a/C/plugins/storage/sqlitememory/include/connection.h +++ b/C/plugins/storage/sqlitememory/include/connection.h @@ -38,10 +38,11 @@ class Connection { int mapResultSet(void *res, std::string& resultSet); bool jsonWhereClause(const rapidjson::Value& whereClause, SQLBuffer&, bool convertLocaltime = false); bool jsonModifiers(const rapidjson::Value&, SQLBuffer&); - bool jsonAggregates(const rapidjson::Value&, - const rapidjson::Value&, - SQLBuffer&, - SQLBuffer&); + bool jsonAggregates(const rapidjson::Value&, + const rapidjson::Value&, + SQLBuffer&, + SQLBuffer&, + bool isTableReading = false); bool returnJson(const rapidjson::Value&, SQLBuffer&, SQLBuffer&); char *trim(char *str); const char *escape(const char *); From fb5700547dac79191bea8fe49cd0f5eaeba49966 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 28 Feb 2019 13:14:34 +0530 Subject: [PATCH 49/86] Minor fix in shim.py when custom fields are added to config in plugin_init() --- python/foglamp/plugins/common/shim/shim.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/foglamp/plugins/common/shim/shim.py b/python/foglamp/plugins/common/shim/shim.py index c156b564fb..511b13d779 100644 --- a/python/foglamp/plugins/common/shim/shim.py +++ b/python/foglamp/plugins/common/shim/shim.py @@ -82,10 +82,11 @@ def _revised_config_for_json_item(config): # them to str, which is the required format for configuration items. revised_config_handle = {} for k, v in config.items(): - if v['type'] == 'JSON': - if isinstance(v['default'], dict): - v['default'] = json.dumps(v['default']) - if isinstance(v['value'], dict): - v['value'] = json.dumps(v['value']) + if isinstance(v, dict): + if 'type' in v and v['type'] == 'JSON': + if isinstance(v['default'], dict): + v['default'] = json.dumps(v['default']) + if isinstance(v['value'], dict): + v['value'] = json.dumps(v['value']) revised_config_handle.update({k: v}) return revised_config_handle From f996309f7eff52adc5008cd753777616a864afcf Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 28 Feb 2019 13:18:28 +0530 Subject: [PATCH 50/86] FOGL-2528 - description field added in post method --- python/foglamp/services/core/api/notification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/foglamp/services/core/api/notification.py b/python/foglamp/services/core/api/notification.py index 7e0e7f3667..c348d49468 100644 --- a/python/foglamp/services/core/api/notification.py +++ b/python/foglamp/services/core/api/notification.py @@ -217,6 +217,7 @@ async def post_notification(request): storage = connect.get_storage_async() config_mgr = ConfigurationManager(storage) notification_config = { + "description": description, "rule": rule, "channel": channel, "notification_type": notification_type, From 99e360a1e706b31c79923ff02fdddab019bb0a56 Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 28 Feb 2019 13:27:44 +0530 Subject: [PATCH 51/86] Tests fixed --- .../foglamp/services/core/api/test_notification.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/python/foglamp/services/core/api/test_notification.py b/tests/unit/python/foglamp/services/core/api/test_notification.py index 4af3ae1585..0cbc33ccbb 100644 --- a/tests/unit/python/foglamp/services/core/api/test_notification.py +++ b/tests/unit/python/foglamp/services/core/api/test_notification.py @@ -389,7 +389,7 @@ async def test_post_notification(self, mocker, client): assert 200 == resp.status result = await resp.json() assert result['result'].endswith("Notification {} created successfully".format("Test Notification")) - update_configuration_item_bulk_calls = [call('Test Notification', {'enable': 'false', 'rule': 'threshold', 'notification_type': 'one shot', 'channel': 'email'})] + update_configuration_item_bulk_calls = [call('Test Notification', {'enable': 'false', 'rule': 'threshold', 'description': 'Test Notification', 'channel': 'email', 'notification_type': 'one shot'})] update_configuration_item_bulk.assert_has_calls(update_configuration_item_bulk_calls, any_order=True) async def test_post_notification2(self, mocker, client): @@ -413,9 +413,10 @@ async def test_post_notification2(self, mocker, client): assert 200 == resp.status result = await resp.json() assert result['result'].endswith("Notification {} created successfully".format("Test Notification")) - update_configuration_item_bulk_calls = [call('Test Notification', {'notification_type': 'one shot', 'enable': 'false', 'channel': 'email', 'rule': 'threshold'}), - call('ruleTest Notification', {'window': '100'}), - call('deliveryTest Notification', {'server': 'pop'})] + update_configuration_item_bulk_calls = [call('Test Notification', {'description': 'Test Notification', 'rule': 'threshold', 'channel': 'email', + 'notification_type': 'one shot', 'enable': 'false'}), + call('ruleTest Notification', {'window': '100'}), + call('deliveryTest Notification', {'server': 'pop'})] update_configuration_item_bulk.assert_has_calls(update_configuration_item_bulk_calls, any_order=True) async def test_post_notification_exception(self, mocker, client): From 31104fa831236e6bd26e44ab7b770bc315e51d2f Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 28 Feb 2019 10:00:28 +0100 Subject: [PATCH 52/86] FOGL-2331: fixed 'Simple column name' handling --- C/plugins/storage/sqlitememory/connection.cpp | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index ea84c89080..ccdbaf4046 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -710,7 +710,7 @@ SQLBuffer jsonConstraints; sql.append(document["modifier"].GetString()); sql.append(' '); } - if (!jsonAggregates(document, document["aggregate"], sql, jsonConstraints)) + if (!jsonAggregates(document, document["aggregate"], sql, jsonConstraints, true)) { return false; } @@ -733,11 +733,39 @@ SQLBuffer jsonConstraints; } for (Value::ConstValueIterator itr = columns.Begin(); itr != columns.End(); ++itr) { + + // FIXME: + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG retrieveReadings - loop"); + if (col) sql.append(", "); if (!itr->IsObject()) // Simple column name { - sql.append(itr->GetString()); + + // FIXME: + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug("DBG retrieveReadings - simple column"); + + ///-- + if (strcmp(itr->GetString() ,"user_ts") == 0) + { + // Display without TZ expression and microseconds also + sql.append(" strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + sql.append(" as user_ts "); + } + else if (strcmp(itr->GetString() ,"ts") == 0) + { + // Display without TZ expression and microseconds also + sql.append(" strftime('" F_DATEH24_MS "', ts, 'localtime') "); + sql.append(" as ts "); + } + else + { + sql.append(itr->GetString()); + } + } else { @@ -1520,7 +1548,8 @@ long numReadings = 0; bool Connection::jsonAggregates(const Value& payload, const Value& aggregates, SQLBuffer& sql, - SQLBuffer& jsonConstraint) + SQLBuffer& jsonConstraint, + bool isTableReading) { if (aggregates.IsObject()) { @@ -1536,11 +1565,32 @@ bool Connection::jsonAggregates(const Value& payload, "Missing property \"column\" or \"json\""); return false; } + string column_name = aggregates["column"].GetString(); + sql.append(aggregates["operation"].GetString()); sql.append('('); if (aggregates.HasMember("column")) { - sql.append(aggregates["column"].GetString()); + if (strcmp(aggregates["operation"].GetString(), "count") != 0) + { + // an operation different from the 'count' is requested + if (isTableReading && (column_name.compare("user_ts") == 0) ) + { + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + } + else + { + sql.append("\""); + sql.append(column_name); + sql.append("\""); + } + } + else + { + // 'count' operation is requested + sql.append(column_name); + } } else if (aggregates.HasMember("json")) { @@ -1672,7 +1722,19 @@ bool Connection::jsonAggregates(const Value& payload, sql.append('('); if (itr->HasMember("column")) { - sql.append((*itr)["column"].GetString()); + string column_name= (*itr)["column"].GetString(); + if (isTableReading && (column_name.compare("user_ts") == 0) ) + { + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + } + else + { + sql.append("\""); + sql.append(column_name); + sql.append("\""); + } + } else if (itr->HasMember("json")) { From 698361c4bd4fb300693919c9251bb0c6427a857a Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 28 Feb 2019 16:10:11 +0100 Subject: [PATCH 53/86] FOGL-2331: extended C uni test SQLite --- .../C/services/storage/sqlite/expected/103 | 2 +- .../C/services/storage/sqlite/expected/104 | 2 +- .../C/services/storage/sqlite/expected/105 | 1 + .../C/services/storage/sqlite/expected/106 | 1 + .../C/services/storage/sqlite/expected/107 | 1 + .../C/services/storage/sqlite/expected/108 | 1 + .../payloads/msec_add_readings_user_ts.json | 21 +++++++++++++------ .../payloads/msec_query_asset_aggmin.json | 14 +++++++++++++ .../msec_query_asset_aggminarray.json | 20 ++++++++++++++++++ .../payloads/msec_query_asset_alias.json | 15 +++++++++++++ .../payloads/msec_query_asset_noalias.json | 15 +++++++++++++ tests/unit/C/services/storage/sqlite/testset | 4 ++++ 12 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 tests/unit/C/services/storage/sqlite/expected/105 create mode 100644 tests/unit/C/services/storage/sqlite/expected/106 create mode 100644 tests/unit/C/services/storage/sqlite/expected/107 create mode 100644 tests/unit/C/services/storage/sqlite/expected/108 create mode 100644 tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggmin.json create mode 100644 tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggminarray.json create mode 100644 tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_alias.json create mode 100644 tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_noalias.json diff --git a/tests/unit/C/services/storage/sqlite/expected/103 b/tests/unit/C/services/storage/sqlite/expected/103 index 7bbe797ef1..d84da94821 100644 --- a/tests/unit/C/services/storage/sqlite/expected/103 +++ b/tests/unit/C/services/storage/sqlite/expected/103 @@ -1 +1 @@ -{ "response" : "appended", "readings_added" : 4 } \ No newline at end of file +{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/104 b/tests/unit/C/services/storage/sqlite/expected/104 index 3beac32be3..b0e610c297 100644 --- a/tests/unit/C/services/storage/sqlite/expected/104 +++ b/tests/unit/C/services/storage/sqlite/expected/104 @@ -1 +1 @@ -{"count":4,"rows":[{"asset_code":"msec__001_OK","read_key":"f1cfff7a-3769-4f47-9ded-00000000001","reading":{"value":1},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec__002_OK","read_key":"f1cfff7a-3769-4f47-9ded-00000000002","reading":{"value":1},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec__003_OK","read_key":"f1cfff7a-3769-4f47-9ded-00000000003","reading":{"value":1},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec__004_OK","read_key":"f1cfff7a-3769-4f47-9ded-00000000004","reading":{"value":1},"user_ts":"2019-01-04 11:03:05.123456"}]} \ No newline at end of file +{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 10:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 10:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 10:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 09:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 11:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 07:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 12:33:05.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/105 b/tests/unit/C/services/storage/sqlite/expected/105 new file mode 100644 index 0000000000..d5ef46e060 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected/105 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/106 b/tests/unit/C/services/storage/sqlite/expected/106 new file mode 100644 index 0000000000..36b92e7724 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected/106 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts_alias":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/107 b/tests/unit/C/services/storage/sqlite/expected/107 new file mode 100644 index 0000000000..761333c73d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected/107 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/108 b/tests/unit/C/services/storage/sqlite/expected/108 new file mode 100644 index 0000000000..96421c5796 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected/108 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456","user_ts_max":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/payloads/msec_add_readings_user_ts.json b/tests/unit/C/services/storage/sqlite/payloads/msec_add_readings_user_ts.json index 4116a5fb1b..31c62f9761 100644 --- a/tests/unit/C/services/storage/sqlite/payloads/msec_add_readings_user_ts.json +++ b/tests/unit/C/services/storage/sqlite/payloads/msec_add_readings_user_ts.json @@ -1,12 +1,21 @@ { "readings" : [ - { "user_ts" : "xxx", "asset_code": "msec__001_BAD", "read_key" : "f1cfff7a-3769-4f47-9ded-10000000001", "reading" : { "value" : 1 }}, - { "user_ts" : "2019-30-07 10:17:17.123456+00", "asset_code": "msec__002_BAD", "read_key" : "f1cfff7a-3769-4f47-9ded-10000000001", "reading" : { "value" : 1 }}, + { "user_ts" : "xxx", "asset_code": "msec_001_BAD", "read_key" : "f1cfff7a-3769-4f47-9ded-100000000001", "reading" : { "value" : 1 }}, + { "user_ts" : "2019-30-07 10:17:17.123456+00", "asset_code": "msec_002_BAD", "read_key" : "f1cfff7a-3769-4f47-9ded-100000000002", "reading" : { "value" : 2 }}, - { "user_ts" : "2019-01-01 10:01:01", "asset_code": "msec__001_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-00000000001", "reading" : { "value" : 1 }}, - { "user_ts" : "2019-01-02 10:02:01.0", "asset_code": "msec__002_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-00000000002", "reading" : { "value" : 1 }}, - { "user_ts" : "2019-01-03 10:02:02.841", "asset_code": "msec__003_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-00000000003", "reading" : { "value" : 1 }}, - { "user_ts" : "2019-01-04 10:03:05.123456-01:00","asset_code": "msec__004_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-00000000004", "reading" : { "value" : 1 }} + { "user_ts" : "2019-01-01 10:01:01", "asset_code": "msec_003_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000003", "reading" : { "value" : 3 }}, + { "user_ts" : "2019-01-02 10:02:01.0", "asset_code": "msec_004_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000004", "reading" : { "value" : 4 }}, + { "user_ts" : "2019-01-03 10:02:02.841", "asset_code": "msec_005_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000005", "reading" : { "value" : 5 }}, + { "user_ts" : "2019-01-04 10:03:05.123456", "asset_code": "msec_006_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000006", "reading" : { "value" : 6 }}, + + { "user_ts" : "2019-01-04 10:03:05.1+00:00", "asset_code": "msec_007_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000007", "reading" : { "value" : 7 }}, + { "user_ts" : "2019-01-04 10:03:05.123+00:00", "asset_code": "msec_008_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000008", "reading" : { "value" : 8 }}, + + { "user_ts" : "2019-03-03 10:03:03.123456+00:00","asset_code": "msec_009_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000009", "reading" : { "value" : 9 }}, + { "user_ts" : "2019-03-04 10:03:04.123456+01:00","asset_code": "msec_010_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000010", "reading" : { "value" :10 }}, + { "user_ts" : "2019-03-05 10:03:05.123456-01:00","asset_code": "msec_011_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000011", "reading" : { "value" :11 }}, + { "user_ts" : "2019-03-04 10:03:04.123456+02:30","asset_code": "msec_012_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000012", "reading" : { "value" :12 }}, + { "user_ts" : "2019-03-05 10:03:05.123456-02:30","asset_code": "msec_013_OK", "read_key" : "f1cfff7a-3769-4f47-9ded-000000000013", "reading" : { "value" :13 }} ] } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggmin.json b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggmin.json new file mode 100644 index 0000000000..06ac638b2a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggmin.json @@ -0,0 +1,14 @@ +{ + "aggregate": + { + "operation": "min", + "column": "user_ts", + "alias": "user_ts_min" + }, + "where": + { + "column": "asset_code", + "condition": "=", + "value": "msec_009_OK" + } +} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggminarray.json b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggminarray.json new file mode 100644 index 0000000000..ae838350f1 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_aggminarray.json @@ -0,0 +1,20 @@ +{ + "aggregate": [ + { + "operation": "min", + "column": "user_ts", + "alias": "user_ts_min" + }, + { + "operation": "max", + "column": "user_ts", + "alias": "user_ts_max" + } + ], + "where": { + "column": "asset_code", + "condition": "=", + "value": "msec_009_OK" + } +} + diff --git a/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_alias.json b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_alias.json new file mode 100644 index 0000000000..d99088d7b1 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_alias.json @@ -0,0 +1,15 @@ +{ + "where" : { + "column" : "asset_code", + "condition" : "=", + "value" : "msec_009_OK" + }, + + "return" : [ + "reading", + { + "column" : "user_ts", + "alias" : "user_ts_alias" + } + ] +} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_noalias.json b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_noalias.json new file mode 100644 index 0000000000..df3a033c8d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/payloads/msec_query_asset_noalias.json @@ -0,0 +1,15 @@ +{ + "where" : { + "column" : "asset_code", + "condition" : "=", + "value" : "msec_009_OK" + }, + + "return" : [ + "reading", + { + "column" : "user_ts" + } + + ] +} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/testset b/tests/unit/C/services/storage/sqlite/testset index 18daf92982..fa672be38f 100644 --- a/tests/unit/C/services/storage/sqlite/testset +++ b/tests/unit/C/services/storage/sqlite/testset @@ -102,4 +102,8 @@ Query Readings IN operator bad values,PUT,http://localhost:8080/storage/reading/ microseconds - Purge Readings,PUT,http://localhost:8080/storage/reading/purge?age=1&sent=0&flags=purge, microseconds - Add Readings,POST,http://localhost:8080/storage/reading,msec_add_readings_user_ts.json microseconds - Query Readings,PUT,http://localhost:8080/storage/reading/query,msec_query_readings.json +microseconds - Query asset NO alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_noalias.json +microseconds - Query asset alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_alias.json +microseconds - Query asset aggregate min,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggmin.json +microseconds - Query asset aggregate min array,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggminarray.json Shutdown,POST,http://localhost:1081/foglamp/service/shutdown,,checkstate From e7ed8979345eb7b95c2c5cd7f6975483bc6dd202 Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 28 Feb 2019 16:26:10 +0100 Subject: [PATCH 54/86] FOGL-2331: code cleanup --- C/plugins/storage/sqlitememory/connection.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index ccdbaf4046..7281c8f0b1 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -665,11 +665,6 @@ SQLBuffer sql; // Extra constraints to add to where clause SQLBuffer jsonConstraints; - // FIXME: - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG retrieveReadings"); - - try { if (inMemory == NULL) { @@ -734,20 +729,10 @@ SQLBuffer jsonConstraints; for (Value::ConstValueIterator itr = columns.Begin(); itr != columns.End(); ++itr) { - // FIXME: - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG retrieveReadings - loop"); - if (col) sql.append(", "); if (!itr->IsObject()) // Simple column name { - - // FIXME: - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG retrieveReadings - simple column"); - - ///-- if (strcmp(itr->GetString() ,"user_ts") == 0) { // Display without TZ expression and microseconds also @@ -1299,10 +1284,6 @@ char *zErrMsg = NULL; int rc; int retrieve; - // FIXME: - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG fetchReadings"); - // SQL command to extract the data from the foglamp.readings const char *sql_cmd = R"( SELECT From e6599a380b649486b2b5fa33e66fe87a765452f6 Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 28 Feb 2019 17:13:52 +0100 Subject: [PATCH 55/86] FOGL-2331: reduce sqlbuffer in fetchReadings --- C/plugins/storage/sqlite/connection.cpp | 2 +- C/plugins/storage/sqlitememory/connection.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index ae572df06c..a1cdd5df45 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -1785,7 +1785,7 @@ bool Connection::fetchReadings(unsigned long id, unsigned int blksize, std::string& resultSet) { -char sqlbuffer[1100]; +char sqlbuffer[512]; char *zErrMsg = NULL; int rc; int retrieve; diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index 7281c8f0b1..27312cf5a4 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -1279,7 +1279,7 @@ bool Connection::fetchReadings(unsigned long id, unsigned int blksize, std::string& resultSet) { -char sqlbuffer[1100]; +char sqlbuffer[512]; char *zErrMsg = NULL; int rc; int retrieve; @@ -1309,6 +1309,11 @@ int retrieve; id, blksize); + // FIXME_I: + Logger::getLogger()->setMinLevel("debug"); + Logger::getLogger()->debug( + "DBG fetchReadings strlen |%d| ",strlen(sqlbuffer)); + logSQL("ReadingsFetch", sqlbuffer); sqlite3_stmt *stmt; // Prepare the SQL statement and get the result set From 1b5c2041e315cd935b37db17243fac9a969ec6ca Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 28 Feb 2019 17:18:35 +0100 Subject: [PATCH 56/86] FOGL-2331: code cleanup --- C/plugins/storage/sqlite/connection.cpp | 1 - C/plugins/storage/sqlitememory/connection.cpp | 5 ----- 2 files changed, 6 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index a1cdd5df45..b44417b6ca 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -1814,7 +1814,6 @@ int retrieve; sql_cmd, id, blksize); - logSQL("ReadingsFetch", sqlbuffer); sqlite3_stmt *stmt; // Prepare the SQL statement and get the result set diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index 27312cf5a4..87b586e6dc 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -1309,11 +1309,6 @@ int retrieve; id, blksize); - // FIXME_I: - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug( - "DBG fetchReadings strlen |%d| ",strlen(sqlbuffer)); - logSQL("ReadingsFetch", sqlbuffer); sqlite3_stmt *stmt; // Prepare the SQL statement and get the result set From cbf4cc9a25a0bfa7b961f13a53bd2db5dccdc61f Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Mar 2019 10:13:51 +0100 Subject: [PATCH 57/86] FOGL-2331: code cleanup. switch to use F_DATEH24_* --- C/plugins/storage/sqlitememory/connection.cpp | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/C/plugins/storage/sqlitememory/connection.cpp b/C/plugins/storage/sqlitememory/connection.cpp index 87b586e6dc..72afc94908 100644 --- a/C/plugins/storage/sqlitememory/connection.cpp +++ b/C/plugins/storage/sqlitememory/connection.cpp @@ -674,18 +674,15 @@ SQLBuffer jsonConstraints; if (condition.empty()) { - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG condition empty"); - const char *sql_cmd = R"( SELECT id, asset_code, read_key, reading, - strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + strftime(')" F_DATEH24_SEC R"(', user_ts, 'localtime') || substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, - strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + strftime(')" F_DATEH24_MS R"(', ts, 'localtime') AS ts FROM readings)"; sql.append(sql_cmd); @@ -796,7 +793,7 @@ SQLBuffer jsonConstraints; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'utc') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'utc') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -806,7 +803,7 @@ SQLBuffer jsonConstraints; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'utc')"); if (! itr->HasMember("alias")) @@ -822,7 +819,7 @@ SQLBuffer jsonConstraints; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -832,7 +829,7 @@ SQLBuffer jsonConstraints; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'localtime')"); if (! itr->HasMember("alias")) @@ -857,7 +854,7 @@ SQLBuffer jsonConstraints; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -867,7 +864,7 @@ SQLBuffer jsonConstraints; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'localtime')"); if (! itr->HasMember("alias")) @@ -906,10 +903,6 @@ SQLBuffer jsonConstraints; } else { - Logger::getLogger()->setMinLevel("debug"); - Logger::getLogger()->debug("DBG condition NO"); - - sql.append("SELECT "); if (document.HasMember("modifier")) { @@ -921,9 +914,9 @@ SQLBuffer jsonConstraints; asset_code, read_key, reading, - strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + strftime(')" F_DATEH24_SEC R"(', user_ts, 'localtime') || substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, - strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + strftime(')" F_DATEH24_MS R"(', ts, 'localtime') AS ts FROM foglamp.)"; sql.append(sql_cmd); @@ -1027,14 +1020,14 @@ bool Connection::formatDate(char *formatted_date, size_t buffer_size, const char // Extract up to seconds memset(&tm, 0, sizeof(tm)); - valid_date = strptime(date, "%Y-%m-%d %H:%M:%S", &tm); + valid_date = strptime(date, F_DATEH24_SEC, &tm); if (! valid_date) { return (false); } - strftime (formatted_date, buffer_size, "%Y-%m-%d %H:%M:%S", &tm); + strftime (formatted_date, buffer_size, F_DATEH24_SEC, &tm); // Work out the microseconds from the fractional part of the seconds char fractional[10] = {0}; From cfdda95d2520736f5bdd88070db1e291a94c5eff Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Mon, 4 Mar 2019 17:26:02 +0530 Subject: [PATCH 58/86] Feedback changes --- .../e2e/test_e2e_csv_multi_filter_pi.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py index 2864ae5da9..e1235efdee 100644 --- a/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py +++ b/tests/system/python/e2e/test_e2e_csv_multi_filter_pi.py @@ -53,19 +53,18 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, # Define the CSV data and create expected lists to be verified later csv_file_path = os.path.join(os.path.expandvars('${FOGLAMP_ROOT}'), 'data/{}'.format(CSV_NAME)) - f = open(csv_file_path, "w") - f.write(CSV_HEADERS) - for _items in CSV_DATA.split(","): - f.write("\n{}".format(_items)) - f.close() + with open(csv_file_path, 'w') as f: + f.write(CSV_HEADERS) + for _items in CSV_DATA.split(","): + f.write("\n{}".format(_items)) south_plugin = "playback" add_south(south_plugin, south_branch, foglamp_url, service_name=SVC_NAME, config=south_config, start_service=False) - filter_cfg = {"enable": "true"} + filter_cfg_scale = {"enable": "true"} # I/P 10, 20, 21, 40 -> O/P 1000, 2000, 2100, 4000 - add_filter("scale", filter_branch, "fscale", filter_cfg, foglamp_url, SVC_NAME) + add_filter("scale", filter_branch, "fscale", filter_cfg_scale, foglamp_url, SVC_NAME) # I/P asset_name : e2e_csv_filter_pi > O/P e2e_filters filter_cfg_asset = {"config": {"rules": [{"new_asset_name": "e2e_filters", @@ -78,7 +77,10 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, filter_cfg_rate = {"trigger": "ivalue > 1200", "untrigger": "ivalue < 1100", "preTrigger": "0", "enable": "true"} add_filter("rate", filter_branch, "frate", filter_cfg_rate, foglamp_url, SVC_NAME) - # I/P 1000, 2000, 2100, 40000 -> O/P 2000, 4000 + # I/P 1000, 2000, 2100, 4000 -> O/P 2000, 4000 + # Delta in 1st pair (2000-1000) = 1000 (> 20% of 1000) so 2000 is output + # Delta in second pair (2100-2000) = 100 (<20% of 2000) so 2100 not in output + # Delta in third pair (4000-2100) = 1900 (>20% of 2100) so 4000 in output filter_cfg_delta = {"tolerance": "20", "enable": "true"} add_filter("delta", filter_branch, "fdelta", filter_cfg_delta , foglamp_url, SVC_NAME) @@ -86,7 +88,8 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, filter_cfg_rms = {"assetName": "%a_RMS", "samples": "2", "peak": "true", "enable": "true"} add_filter("rms", filter_branch, "frms", filter_cfg_rms, foglamp_url, SVC_NAME) - add_filter("metadata", filter_branch, "fmeta", filter_cfg, foglamp_url, SVC_NAME) + filter_cfg_meta = {"enable": "true"} + add_filter("metadata", filter_branch, "fmeta", filter_cfg_meta, foglamp_url, SVC_NAME) # Since playback plugin reads all csv data at once, we cant keep it in enable mode before filter add # enable service when all filters all applied @@ -97,12 +100,10 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, yield self.start_south_north remove_directories("/tmp/foglamp-south-{}".format(south_plugin)) - remove_directories("/tmp/foglamp-filter-{}".format("scale")) - remove_directories("/tmp/foglamp-filter-{}".format("asset")) - remove_directories("/tmp/foglamp-filter-{}".format("rate")) - remove_directories("/tmp/foglamp-filter-{}".format("delta")) - remove_directories("/tmp/foglamp-filter-{}".format("rms")) - remove_directories("/tmp/foglamp-filter-{}".format("metadata")) + filters = ["scale", "asset", "rate", "delta", "rms", "metadata"] + for fltr in filters: + remove_directories("/tmp/foglamp-filter-{}".format(fltr)) + remove_data_file(csv_file_path) def test_end_to_end(self, start_south_north, disable_schedule, foglamp_url, read_data_from_pi, pi_host, pi_admin, From b3ccea9332980023d35352ce4f07963ab394e8ab Mon Sep 17 00:00:00 2001 From: pintomax Date: Mon, 4 Mar 2019 13:43:14 +0100 Subject: [PATCH 59/86] FOGL-2482: return quoted value for TEXT column with a numeric value (#1434) FOGL-2482: return quoted value for TEXT column with a numeric value --- C/plugins/storage/sqlite/connection.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index eed12bf546..a199676c36 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -600,8 +600,16 @@ unsigned long nRows = 0, nCols = 0; Value value; if (!d.Parse(str).HasParseError()) { - // JSON parsing ok, use the document - value = Value(d, allocator); + if (d.IsNumber()) + { + // Set string + value = Value(str, allocator); + } + else + { + // JSON parsing ok, use the document + value = Value(d, allocator); + } } else { From 470cd802f220c0880076dabb0eab0633269d8ec3 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Mar 2019 14:14:03 +0100 Subject: [PATCH 60/86] FOGL-2331: sqlitememory C unit tests created - formatDate --- .../storage/sqlitememory/include/connection.h | 2 +- tests/unit/C/CMakeLists.txt | 24 ++++++ tests/unit/C/Findsqlite3.cmake | 26 +++++++ .../storage/sqlitememory/CMakeLists.txt | 45 +++++++++++ .../C/plugins/storage/sqlitememory/README.rst | 21 ++++++ .../C/plugins/storage/sqlitememory/tests.cpp | 75 +++++++++++++++++++ 6 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 tests/unit/C/Findsqlite3.cmake create mode 100644 tests/unit/C/plugins/storage/sqlitememory/CMakeLists.txt create mode 100644 tests/unit/C/plugins/storage/sqlitememory/README.rst create mode 100644 tests/unit/C/plugins/storage/sqlitememory/tests.cpp diff --git a/C/plugins/storage/sqlitememory/include/connection.h b/C/plugins/storage/sqlitememory/include/connection.h index 0c222e6fec..7cf538abf5 100644 --- a/C/plugins/storage/sqlitememory/include/connection.h +++ b/C/plugins/storage/sqlitememory/include/connection.h @@ -28,6 +28,7 @@ class Connection { unsigned long sent, std::string& results); long tableSize(const std::string& table); void setTrace(bool flag) { m_logSQL = flag; }; + static bool formatDate(char *formatted_date, size_t formatted_date_size, const char *date); private: int SQLexec(sqlite3 *db, const char *sql, int (*callback)(void*,int,char**,char**), @@ -51,6 +52,5 @@ class Connection { int i, std::string& newDate); void logSQL(const char *, const char *); - bool formatDate(char *formatted_date, size_t formatted_date_size, const char *date); }; #endif diff --git a/tests/unit/C/CMakeLists.txt b/tests/unit/C/CMakeLists.txt index 15ddc09821..4459ce1b3c 100644 --- a/tests/unit/C/CMakeLists.txt +++ b/tests/unit/C/CMakeLists.txt @@ -23,6 +23,9 @@ include_directories(../../../C/plugins/common/include) include_directories(../../../C/services/common/include) include_directories(../../../C/thirdparty/rapidjson/include) include_directories(../../../C/thirdparty/Simple-Web-Server) +# sqlitememory plugin +include_directories(../../../C/plugins/storage/common/include) +include_directories(../../../C/plugins/storage/sqlitememory/include) # Add Python 3.x header files include_directories(${PYTHON_INCLUDE_DIRS}) @@ -57,3 +60,24 @@ add_library(plugins-common-lib SHARED ${PLUGINS_COMMON_LIB_SOURCES}) target_link_libraries(plugins-common-lib ${Boost_LIBRARIES} common-lib services-common-lib z ssl crypto) set_target_properties(plugins-common-lib PROPERTIES SOVERSION 1) + + +# +# sqlitememory plugin +# +set(LIB_NAME sqlitememory) + +# Check Sqlite3 required version +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}") +find_package(sqlite3) + +# Find source files +file(GLOB PG_PLUGIN_SOURCES ../../../C/plugins/storage/sqlitememory/*.cpp) + +# Create shared library +add_library(${LIB_NAME} SHARED ${PG_PLUGIN_SOURCES}) +target_link_libraries(${LIB_NAME} ${UUIDLIB}) +target_link_libraries(${LIB_NAME} ${Boost_LIBRARIES}) +target_link_libraries(${LIB_NAME} -lsqlite3) +set_target_properties(${LIB_NAME} PROPERTIES SOVERSION 1) + diff --git a/tests/unit/C/Findsqlite3.cmake b/tests/unit/C/Findsqlite3.cmake new file mode 100644 index 0000000000..c20a28888a --- /dev/null +++ b/tests/unit/C/Findsqlite3.cmake @@ -0,0 +1,26 @@ +# This CMake file locates the SQLite3 development libraries +# +# The following variables are set: +# SQLITE_FOUND - If the SQLite library was found +# SQLITE_LIBRARIES - Path to the static library +# SQLITE_INCLUDE_DIR - Path to SQLite headers +# SQLITE_VERSION - Library version + +set(SQLITE_MIN_VERSION "3.11.0") +find_path(SQLITE_INCLUDE_DIR sqlite3.h) +find_library(SQLITE_LIBRARIES NAMES libsqlite3.so) + +if (SQLITE_INCLUDE_DIR AND SQLITE_LIBRARIES) + execute_process(COMMAND grep ".*#define.*SQLITE_VERSION " ${SQLITE_INCLUDE_DIR}/sqlite3.h + COMMAND sed "s/.*\"\\(.*\\)\".*/\\1/" + OUTPUT_VARIABLE SQLITE_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + if ("${SQLITE_VERSION}" VERSION_LESS "${SQLITE_MIN_VERSION}") + message(FATAL_ERROR "SQLite3 version >= ${SQLITE_MIN_VERSION} required, found version ${SQLITE_VERSION}") + else() + message(STATUS "Found SQLite version ${SQLITE_VERSION}: ${SQLITE_LIBRARIES}") + set(SQLITE_FOUND TRUE) + endif() +else() + message(FATAL_ERROR "Could not find SQLite") +endif() diff --git a/tests/unit/C/plugins/storage/sqlitememory/CMakeLists.txt b/tests/unit/C/plugins/storage/sqlitememory/CMakeLists.txt new file mode 100644 index 0000000000..d4557c0c18 --- /dev/null +++ b/tests/unit/C/plugins/storage/sqlitememory/CMakeLists.txt @@ -0,0 +1,45 @@ +# Project configuration +project(RunTests) +cmake_minimum_required(VERSION 2.6) +set(CMAKE_CXX_FLAGS "-std=c++11 -O3") + +# libraries +set(PG_LIB pq) + +# FogLAMP libraries +set(COMMON_LIB common-lib) +set(SERVICE_COMMON_LIB services-common-lib) +set(PLUGINS_COMMON_LIB plugins-common-lib) +set(PLUGIN_SQLITEMEMORY sqlitememory) +set(STORAGE_COMMON_LIB storage-common-lib) + +# Locate GTest +find_package(GTest REQUIRED) + +# Include files +include_directories(${GTEST_INCLUDE_DIRS}) +include_directories(../../../../../../C/common/include) +include_directories(../../../../../../C/services/common/include) +include_directories(../../../../../../C/plugins/storage/common/include) +include_directories(../../../../../../C/plugins/storage/sqlitememory/include) +include_directories(../../../../../../C/thirdparty/rapidjson/include) + +# Source files +file(GLOB test_sources tests.cpp) + +# Exe creation +link_directories( + ${PROJECT_BINARY_DIR}/../../../../lib +) + +add_executable(${PROJECT_NAME} ${test_sources}) + +target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) +target_link_libraries(${PROJECT_NAME} ${SERVICE_COMMON_LIB}) +target_link_libraries(${PROJECT_NAME} ${PLUGINS_COMMON_LIB}) + +target_link_libraries(${PROJECT_NAME} ${PLUGIN_SQLITEMEMORY}) +target_link_libraries(${PROJECT_NAME} ${STORAGE_COMMON_LIB}) +target_link_libraries(${PROJECT_NAME} ${PG_LIB}) + +target_link_libraries(${PROJECT_NAME} ${GTEST_LIBRARIES} pthread) \ No newline at end of file diff --git a/tests/unit/C/plugins/storage/sqlitememory/README.rst b/tests/unit/C/plugins/storage/sqlitememory/README.rst new file mode 100644 index 0000000000..4423a3ef32 --- /dev/null +++ b/tests/unit/C/plugins/storage/sqlitememory/README.rst @@ -0,0 +1,21 @@ +***************************************************** +Unit Test for Postgres Storage Plugin +***************************************************** + +Require Google Unit Test framework + +Install with: +:: + sudo apt-get install libgtest-dev + cd /usr/src/gtest + cmake CMakeLists.txt + sudo make + sudo make install + +To build the unit test: +:: + mkdir build + cd build + cmake .. + make + ./runTests diff --git a/tests/unit/C/plugins/storage/sqlitememory/tests.cpp b/tests/unit/C/plugins/storage/sqlitememory/tests.cpp new file mode 100644 index 0000000000..b3f84160b3 --- /dev/null +++ b/tests/unit/C/plugins/storage/sqlitememory/tests.cpp @@ -0,0 +1,75 @@ +#include +#include +#include "gtest/gtest.h" +#include +#include +#include + +using namespace std; + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + +class RowFormatDate { + public: + const char *test_case; + const char *expected; + bool result; + + RowFormatDate(const char *p1, const char *p2, bool p3) { + test_case = p1; + expected = p2; + result = p3; + }; +}; + +class TestFormatDate : public ::testing::TestWithParam { +}; + +TEST_P(TestFormatDate, TestConversions) +{ + Logger::getLogger()->setMinLevel("debug"); + + RowFormatDate const& p = GetParam(); + + char formatted_date[50] = {0}; + memset (formatted_date,0 , sizeof (formatted_date)); + bool result = Connection::formatDate(formatted_date, sizeof(formatted_date), p.test_case); + + string test_case = formatted_date; + string expected = p.expected; + + ASSERT_EQ(test_case, expected); + ASSERT_EQ(result, p.result); +} + +INSTANTIATE_TEST_CASE_P( + TestConversions, + TestFormatDate, + ::testing::Values( + // Test cases Expected + RowFormatDate("2019-01-01 10:01:01" ,"2019-01-01 10:01:01.000000+00:00", true), + RowFormatDate("2019-02-01 10:02:01.0" ,"2019-02-01 10:02:01.000000+00:00", true), + RowFormatDate("2019-02-02 10:02:02.841" ,"2019-02-02 10:02:02.841000+00:00", true), + RowFormatDate("2019-02-03 10:02:03.123456" ,"2019-02-03 10:02:03.123456+00:00", true), + + RowFormatDate("2019-03-01 10:03:01.1+00:00" ,"2019-03-01 10:03:01.100000+00:00", true), + RowFormatDate("2019-03-02 10:03:02.123+00:00" ,"2019-03-02 10:03:02.123000+00:00", true), + + RowFormatDate("2019-03-03 10:03:03.123456+00:00" ,"2019-03-03 10:03:03.123456+00:00", true), + RowFormatDate("2019-03-04 10:03:04.123456+01:00" ,"2019-03-04 10:03:04.123456+01:00", true), + RowFormatDate("2019-03-05 10:03:05.123456-01:00" ,"2019-03-05 10:03:05.123456-01:00", true), + RowFormatDate("2019-03-04 10:03:04.123456+02:30" ,"2019-03-04 10:03:04.123456+02:30", true), + RowFormatDate("2019-03-05 10:03:05.123456-02:30" ,"2019-03-05 10:03:05.123456-02:30", true), + + // Timestamp truncated + RowFormatDate("2017-10-11 15:10:51.927191906" ,"2017-10-11 15:10:51.927191+00:00", true), + + // Bad cases + RowFormatDate("xxx", "", false), + RowFormatDate("2019-50-50 10:01:01.0", "", false) + ) +); From 16cb5778b0bc7549d0d81deb6398f7ad51a14c17 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Mar 2019 14:45:20 +0100 Subject: [PATCH 61/86] FOGL-2331: fixing - sqlitememory C unit tests created - formatDate --- tests/unit/C/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit/C/CMakeLists.txt b/tests/unit/C/CMakeLists.txt index 4459ce1b3c..120f5376ca 100644 --- a/tests/unit/C/CMakeLists.txt +++ b/tests/unit/C/CMakeLists.txt @@ -61,6 +61,20 @@ target_link_libraries(plugins-common-lib ${Boost_LIBRARIES} common-lib services- set_target_properties(plugins-common-lib PROPERTIES SOVERSION 1) +# +# storage-common-lib +# +set(LIB_NAME storage-common-lib) +set(DLLIB -ldl) + +# Find source files +file(GLOB STORAGE_COMMON_LIB_SOURCE ../../../C/plugins/storage/common/*.cpp) + +# Create shared library +add_library(${LIB_NAME} SHARED ${STORAGE_COMMON_LIB_SOURCE}) +target_link_libraries(${LIB_NAME} ${DLLIB}) +set_target_properties(${LIB_NAME} PROPERTIES SOVERSION 1) + # # sqlitememory plugin From 67cadf9a0e96cd45f1aeecc2e0a4c4c0daf8fb87 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Tue, 5 Mar 2019 12:00:23 +0530 Subject: [PATCH 62/86] command line for db selection added --- tests/system/python/conftest.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/system/python/conftest.py b/tests/system/python/conftest.py index 148fd71b45..4c39040c97 100644 --- a/tests/system/python/conftest.py +++ b/tests/system/python/conftest.py @@ -25,12 +25,19 @@ @pytest.fixture -def reset_and_start_foglamp(): - """Fixture that kills foglamp, reset database and starts foglamp again""" +def reset_and_start_foglamp(storage_plugin): + """Fixture that kills foglamp, reset database and starts foglamp again + storage_plugin: Fixture that defines the storage plugin to be used for tests + """ - # TODO: allow to sed storage.json and use postgres database plugin assert os.environ.get('FOGLAMP_ROOT') is not None + subprocess.run(["$FOGLAMP_ROOT/scripts/foglamp kill"], shell=True, check=True) + if storage_plugin == 'postgres': + subprocess.run(["sed -i 's/sqlite/postgres/g' $FOGLAMP_ROOT/data/etc/storage.json"], shell=True, check=True) + else: + subprocess.run(["sed -i 's/postgres/sqlite/g' $FOGLAMP_ROOT/data/etc/storage.json"], shell=True, check=True) + subprocess.run(["echo YES | $FOGLAMP_ROOT/scripts/foglamp reset"], shell=True, check=True) subprocess.run(["$FOGLAMP_ROOT/scripts/foglamp start"], shell=True) stat = subprocess.run(["$FOGLAMP_ROOT/scripts/foglamp status"], shell=True, stdout=subprocess.PIPE) @@ -274,6 +281,8 @@ def _disable_sch(foglamp_url, sch_name): def pytest_addoption(parser): + parser.addoption("--storage-plugin", action="store", default="sqlite", + help="Database plugin to use for tests") parser.addoption("--foglamp-url", action="store", default="localhost:8081", help="FogLAMP client api url") parser.addoption("--use-pip-cache", action="store", default=False, @@ -338,6 +347,11 @@ def pytest_addoption(parser): parser.addoption("--kafka-rest-port", action="store", default="8082", help="Kafka Rest Proxy Port") +@pytest.fixture +def storage_plugin(request): + return request.config.getoption("--storage-plugin") + + @pytest.fixture def south_branch(request): return request.config.getoption("--south-branch") From 189040848984ea1e4f2d5644c8ee176abfca5c56 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Tue, 5 Mar 2019 12:00:41 +0530 Subject: [PATCH 63/86] Doc changes --- tests/system/python/README.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system/python/README.rst b/tests/system/python/README.rst index d4586eadc2..b20e195379 100644 --- a/tests/system/python/README.rst +++ b/tests/system/python/README.rst @@ -85,7 +85,8 @@ custom options :: $ pytest --help ... custom options: - + --storage-plugin=STORAGE_PLUGIN + Database plugin to use for tests --south-branch=SOUTH_BRANCH south branch name --north-branch=NORTH_BRANCH @@ -139,7 +140,13 @@ custom options :: --kafka-rest-port=KAFKA_REST_PORT Kafka REST Proxy Port +Using different storage engine +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default system tests runs with sqlite database. If you want, you can use postgres storage plugin and tests will be +executed using postgres database and postgres storage engine:: + $ pytest test_smoke.py --storage-plugin=postgres Test test_e2e_coap_PI and test_e2e_csv_PI ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 932b59c1840d8aa7afc16894fb742490d647b506 Mon Sep 17 00:00:00 2001 From: stefano Date: Tue, 5 Mar 2019 12:28:51 +0100 Subject: [PATCH 64/86] FOGL-2545: working stage --- .../unit/C/services/storage/sqlite/expected/1 | 1 - .../C/services/storage/sqlite/expected/10 | 1 - .../C/services/storage/sqlite/expected/100 | 1 - .../C/services/storage/sqlite/expected/101 | 0 .../C/services/storage/sqlite/expected/105 | 1 - .../C/services/storage/sqlite/expected/106 | 1 - .../C/services/storage/sqlite/expected/107 | 1 - .../C/services/storage/sqlite/expected/108 | 1 - .../C/services/storage/sqlite/expected/11 | 1 - .../C/services/storage/sqlite/expected/12 | 1 - .../C/services/storage/sqlite/expected/13 | 1 - .../C/services/storage/sqlite/expected/14 | 2 - .../C/services/storage/sqlite/expected/15 | 1 - .../C/services/storage/sqlite/expected/16 | 1 - .../C/services/storage/sqlite/expected/18 | 1 - .../C/services/storage/sqlite/expected/19 | 1 - .../unit/C/services/storage/sqlite/expected/2 | 1 - .../C/services/storage/sqlite/expected/20 | 1 - .../C/services/storage/sqlite/expected/21 | 1 - .../C/services/storage/sqlite/expected/22 | 1 - .../C/services/storage/sqlite/expected/23 | 1 - .../C/services/storage/sqlite/expected/24 | 1 - .../C/services/storage/sqlite/expected/25 | 1 - .../C/services/storage/sqlite/expected/26 | 1 - .../C/services/storage/sqlite/expected/27 | 1 - .../C/services/storage/sqlite/expected/28 | 1 - .../C/services/storage/sqlite/expected/29 | 1 - .../unit/C/services/storage/sqlite/expected/3 | 1 - .../C/services/storage/sqlite/expected/30 | 1 - .../C/services/storage/sqlite/expected/31 | 1 - .../C/services/storage/sqlite/expected/32 | 1 - .../C/services/storage/sqlite/expected/33 | 1 - .../C/services/storage/sqlite/expected/34 | 1 - .../C/services/storage/sqlite/expected/35 | 1 - .../C/services/storage/sqlite/expected/37 | 1 - .../C/services/storage/sqlite/expected/38 | 1 - .../C/services/storage/sqlite/expected/39 | 1 - .../unit/C/services/storage/sqlite/expected/4 | 1 - .../C/services/storage/sqlite/expected/40 | 1 - .../C/services/storage/sqlite/expected/41 | 1 - .../C/services/storage/sqlite/expected/42 | 1 - .../C/services/storage/sqlite/expected/43 | 1 - .../C/services/storage/sqlite/expected/44 | 1 - .../C/services/storage/sqlite/expected/45 | 1 - .../C/services/storage/sqlite/expected/46 | 1 - .../C/services/storage/sqlite/expected/47 | 1 - .../C/services/storage/sqlite/expected/48 | 1 - .../C/services/storage/sqlite/expected/49 | 1 - .../unit/C/services/storage/sqlite/expected/5 | 1 - .../C/services/storage/sqlite/expected/50 | 1 - .../C/services/storage/sqlite/expected/51 | 1 - .../C/services/storage/sqlite/expected/52 | 1 - .../C/services/storage/sqlite/expected/53 | 1 - .../C/services/storage/sqlite/expected/54 | 1 - .../C/services/storage/sqlite/expected/55 | 1 - .../C/services/storage/sqlite/expected/56 | 1 - .../C/services/storage/sqlite/expected/57 | 1 - .../C/services/storage/sqlite/expected/58 | 1 - .../C/services/storage/sqlite/expected/59 | 1 - .../unit/C/services/storage/sqlite/expected/6 | 1 - .../C/services/storage/sqlite/expected/60 | 1 - .../C/services/storage/sqlite/expected/61 | 1 - .../C/services/storage/sqlite/expected/62 | 1 - .../C/services/storage/sqlite/expected/63 | 1 - .../C/services/storage/sqlite/expected/64 | 1 - .../C/services/storage/sqlite/expected/65 | 1 - .../C/services/storage/sqlite/expected/66 | 1 - .../C/services/storage/sqlite/expected/67 | 1 - .../C/services/storage/sqlite/expected/68 | 1 - .../C/services/storage/sqlite/expected/69 | 1 - .../unit/C/services/storage/sqlite/expected/7 | 1 - .../C/services/storage/sqlite/expected/70 | 1 - .../C/services/storage/sqlite/expected/71 | 1 - .../C/services/storage/sqlite/expected/72 | 1 - .../C/services/storage/sqlite/expected/73 | 1 - .../C/services/storage/sqlite/expected/74 | 1 - .../C/services/storage/sqlite/expected/75 | 1 - .../C/services/storage/sqlite/expected/76 | 1 - .../C/services/storage/sqlite/expected/77 | 1 - .../C/services/storage/sqlite/expected/78 | 1 - .../C/services/storage/sqlite/expected/79 | 1 - .../unit/C/services/storage/sqlite/expected/8 | 1 - .../C/services/storage/sqlite/expected/80 | 1 - .../C/services/storage/sqlite/expected/81 | 1 - .../C/services/storage/sqlite/expected/82 | 1 - .../C/services/storage/sqlite/expected/83 | 1 - .../C/services/storage/sqlite/expected/84 | 1 - .../C/services/storage/sqlite/expected/85 | 1 - .../C/services/storage/sqlite/expected/86 | 1 - .../C/services/storage/sqlite/expected/87 | 1 - .../C/services/storage/sqlite/expected/88 | 1 - .../C/services/storage/sqlite/expected/89 | 1 - .../unit/C/services/storage/sqlite/expected/9 | 1 - .../C/services/storage/sqlite/expected/90 | 1 - .../C/services/storage/sqlite/expected/91 | 1 - .../C/services/storage/sqlite/expected/92 | 1 - .../C/services/storage/sqlite/expected/93 | 1 - .../C/services/storage/sqlite/expected/94 | 1 - .../C/services/storage/sqlite/expected/95 | 1 - .../C/services/storage/sqlite/expected/96 | 1 - .../C/services/storage/sqlite/expected/97 | 1 - .../C/services/storage/sqlite/expected/98 | 1 - .../C/services/storage/sqlite/expected/99 | 1 - .../{expected/102 => expected_ETC_UTC/1} | 0 .../{expected/103 => expected_ETC_UTC/2} | 0 .../{expected/104 => expected_ETC_UTC/3} | 0 .../storage/sqlite/expected_EUROPE_ROME/1 | 1 + .../storage/sqlite/expected_EUROPE_ROME/2 | 1 + .../storage/sqlite/expected_EUROPE_ROME/3 | 1 + .../C/services/storage/sqlite/testRunner.sh | 73 +++++++++--- tests/unit/C/services/storage/sqlite/testset | 107 +----------------- 111 files changed, 62 insertions(+), 224 deletions(-) delete mode 100644 tests/unit/C/services/storage/sqlite/expected/1 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/10 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/100 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/101 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/105 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/106 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/107 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/108 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/11 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/12 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/13 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/14 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/15 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/16 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/18 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/19 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/2 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/20 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/21 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/22 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/23 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/24 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/25 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/26 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/27 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/28 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/29 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/3 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/30 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/31 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/32 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/33 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/34 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/35 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/37 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/38 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/39 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/4 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/40 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/41 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/42 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/43 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/44 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/45 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/46 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/47 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/48 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/49 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/5 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/50 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/51 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/52 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/53 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/54 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/55 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/56 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/57 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/58 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/59 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/6 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/60 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/61 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/62 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/63 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/64 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/65 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/66 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/67 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/68 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/69 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/7 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/70 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/71 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/72 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/73 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/74 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/75 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/76 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/77 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/78 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/79 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/8 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/80 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/81 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/82 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/83 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/84 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/85 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/86 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/87 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/88 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/89 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/9 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/90 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/91 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/92 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/93 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/94 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/95 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/96 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/97 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/98 delete mode 100644 tests/unit/C/services/storage/sqlite/expected/99 rename tests/unit/C/services/storage/sqlite/{expected/102 => expected_ETC_UTC/1} (100%) rename tests/unit/C/services/storage/sqlite/{expected/103 => expected_ETC_UTC/2} (100%) rename tests/unit/C/services/storage/sqlite/{expected/104 => expected_ETC_UTC/3} (100%) create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 diff --git a/tests/unit/C/services/storage/sqlite/expected/1 b/tests/unit/C/services/storage/sqlite/expected/1 deleted file mode 100644 index 960917c42d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/1 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/10 b/tests/unit/C/services/storage/sqlite/expected/10 deleted file mode 100644 index 89b4f4207d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/10 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/100 b/tests/unit/C/services/storage/sqlite/expected/100 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/100 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/101 b/tests/unit/C/services/storage/sqlite/expected/101 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/C/services/storage/sqlite/expected/105 b/tests/unit/C/services/storage/sqlite/expected/105 deleted file mode 100644 index d5ef46e060..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/105 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/106 b/tests/unit/C/services/storage/sqlite/expected/106 deleted file mode 100644 index 36b92e7724..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/106 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"reading":{"value":9},"user_ts_alias":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/107 b/tests/unit/C/services/storage/sqlite/expected/107 deleted file mode 100644 index 761333c73d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/107 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/108 b/tests/unit/C/services/storage/sqlite/expected/108 deleted file mode 100644 index 96421c5796..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/108 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456","user_ts_max":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/11 b/tests/unit/C/services/storage/sqlite/expected/11 deleted file mode 100644 index 7d66483ebe..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/11 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/12 b/tests/unit/C/services/storage/sqlite/expected/12 deleted file mode 100644 index 1ebeaeaef3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/12 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/13 b/tests/unit/C/services/storage/sqlite/expected/13 deleted file mode 100644 index 27530161d2..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/13 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "insert", "message" : "table foglamp.test has no column named Nonexistant", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/14 b/tests/unit/C/services/storage/sqlite/expected/14 deleted file mode 100644 index 95390e2964..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/14 +++ /dev/null @@ -1,2 +0,0 @@ -{ "entryPoint" : "insert", "message" : "Failed to parse JSON payload -", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/15 b/tests/unit/C/services/storage/sqlite/expected/15 deleted file mode 100644 index 29146ee48a..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/15 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/16 b/tests/unit/C/services/storage/sqlite/expected/16 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/16 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/18 b/tests/unit/C/services/storage/sqlite/expected/18 deleted file mode 100644 index 7d66483ebe..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/18 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/19 b/tests/unit/C/services/storage/sqlite/expected/19 deleted file mode 100644 index 960917c42d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/19 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/2 b/tests/unit/C/services/storage/sqlite/expected/2 deleted file mode 100644 index 960917c42d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/2 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/20 b/tests/unit/C/services/storage/sqlite/expected/20 deleted file mode 100644 index 1ebeaeaef3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/20 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/21 b/tests/unit/C/services/storage/sqlite/expected/21 deleted file mode 100644 index 8b2d266eb9..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/21 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"column\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/22 b/tests/unit/C/services/storage/sqlite/expected/22 deleted file mode 100644 index 8584b11223..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/22 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"condition\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/23 b/tests/unit/C/services/storage/sqlite/expected/23 deleted file mode 100644 index 299df5fdf4..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/23 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"value\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/24 b/tests/unit/C/services/storage/sqlite/expected/24 deleted file mode 100644 index 305ed629e9..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/24 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"where\" property must be a JSON object", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/25 b/tests/unit/C/services/storage/sqlite/expected/25 deleted file mode 100644 index 1ebeaeaef3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/25 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/26 b/tests/unit/C/services/storage/sqlite/expected/26 deleted file mode 100644 index 831678c175..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/26 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "Select sort", "message" : "Missing property \"column\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/27 b/tests/unit/C/services/storage/sqlite/expected/27 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/27 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/28 b/tests/unit/C/services/storage/sqlite/expected/28 deleted file mode 100644 index fd1a64c2c5..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/28 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":2,"key":"TEST2","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/29 b/tests/unit/C/services/storage/sqlite/expected/29 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/29 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/3 b/tests/unit/C/services/storage/sqlite/expected/3 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/3 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/30 b/tests/unit/C/services/storage/sqlite/expected/30 deleted file mode 100644 index 40f3368d49..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/30 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/31 b/tests/unit/C/services/storage/sqlite/expected/31 deleted file mode 100644 index 81a6486e91..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/31 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "update", "message" : "Missing values or expressions object in payload", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/32 b/tests/unit/C/services/storage/sqlite/expected/32 deleted file mode 100644 index d985f72c2d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/32 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"count_id":1,"key":"UPDA"},{"count_id":1,"key":"TEST1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/33 b/tests/unit/C/services/storage/sqlite/expected/33 deleted file mode 100644 index e2fdaddfd5..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/33 +++ /dev/null @@ -1 +0,0 @@ -{ "error" : "Unsupported URL: /foglamp/nothing" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/34 b/tests/unit/C/services/storage/sqlite/expected/34 deleted file mode 100644 index 4584abf305..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/34 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "no such table: foglamp.doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/35 b/tests/unit/C/services/storage/sqlite/expected/35 deleted file mode 100644 index 54fc4416b2..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/35 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "no such column: doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/37 b/tests/unit/C/services/storage/sqlite/expected/37 deleted file mode 100644 index 303973146f..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/37 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "appended", "readings_added" : 2 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/38 b/tests/unit/C/services/storage/sqlite/expected/38 deleted file mode 100644 index b2d0997987..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/38 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"id":966457,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d08bef40","reading":{"rate":18.4},"user_ts":"2017-09-21 15:00:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"},{"id":966458,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d18bef40","reading":{"rate":45.1},"user_ts":"2017-09-21 15:03:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/39 b/tests/unit/C/services/storage/sqlite/expected/39 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/39 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/4 b/tests/unit/C/services/storage/sqlite/expected/4 deleted file mode 100644 index 960917c42d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/4 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/40 b/tests/unit/C/services/storage/sqlite/expected/40 deleted file mode 100644 index 61ef4e298c..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/40 +++ /dev/null @@ -1 +0,0 @@ -{ "error" : "Missing query parameter count" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/41 b/tests/unit/C/services/storage/sqlite/expected/41 deleted file mode 100644 index fb5f172fa7..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/41 +++ /dev/null @@ -1 +0,0 @@ -{ "error" : "Missing query parameter id" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/42 b/tests/unit/C/services/storage/sqlite/expected/42 deleted file mode 100644 index da2cfd9351..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/42 +++ /dev/null @@ -1 +0,0 @@ -{ "removed" : 2, "unsentPurged" : 2, "unsentRetained" : 0, "readings" : 0 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/43 b/tests/unit/C/services/storage/sqlite/expected/43 deleted file mode 100644 index 960917c42d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/43 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/44 b/tests/unit/C/services/storage/sqlite/expected/44 deleted file mode 100644 index 8017dbc45c..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/44 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min_id":1,"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/45 b/tests/unit/C/services/storage/sqlite/expected/45 deleted file mode 100644 index 3cf1b258b8..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/45 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"key":"TEST1","description":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/46 b/tests/unit/C/services/storage/sqlite/expected/46 deleted file mode 100644 index bf8e659fa0..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/46 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"key":"TEST1","MyDescription":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/47 b/tests/unit/C/services/storage/sqlite/expected/47 deleted file mode 100644 index d44fea2a6f..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/47 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"key":"TEST1","JSONvalue":"test1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/48 b/tests/unit/C/services/storage/sqlite/expected/48 deleted file mode 100644 index 7c1b23b0bf..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/48 +++ /dev/null @@ -1 +0,0 @@ -{"count":9,"rows":[{"key":"TEST1","description":"A test row","time":"12:14:26"},{"key":"TEST2","description":"A test row","time":"12:14:27"},{"key":"TEST3","description":"A test row","time":"11:14:28"},{"key":"TEST4","description":"A test row","time":"11:14:29"},{"key":"TEST5","description":"A test row","time":"11:15:00"},{"key":"TEST6","description":"A test row","time":"11:15:33"},{"key":"TEST7","description":"A test row","time":"11:16:20"},{"key":"TEST8","description":"A test row","time":"05:14:30"},{"key":"TEST9","description":"A test row","time":"21:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/49 b/tests/unit/C/services/storage/sqlite/expected/49 deleted file mode 100644 index ee07186753..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/49 +++ /dev/null @@ -1 +0,0 @@ -{"count":8,"rows":[{"key":"TEST2","description":"A test row","timestamp":"2017-10-10 12:14:27"},{"key":"TEST3","description":"A test row","timestamp":"2017-10-10 11:14:28"},{"key":"TEST4","description":"A test row","timestamp":"2017-10-10 11:14:29"},{"key":"TEST5","description":"A test row","timestamp":"2017-10-10 11:15:00"},{"key":"TEST6","description":"A test row","timestamp":"2017-10-10 11:15:33"},{"key":"TEST7","description":"A test row","timestamp":"2017-10-10 11:16:20"},{"key":"TEST8","description":"A test row","timestamp":"2017-10-10 05:14:30"},{"key":"TEST9","description":"A test row","timestamp":"2017-10-10 21:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/5 b/tests/unit/C/services/storage/sqlite/expected/5 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/5 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/50 b/tests/unit/C/services/storage/sqlite/expected/50 deleted file mode 100644 index 2d3ef169f5..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/50 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "return object must have either a column or json property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/51 b/tests/unit/C/services/storage/sqlite/expected/51 deleted file mode 100644 index 24d3191c54..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/51 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "The json property is missing a properties property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/52 b/tests/unit/C/services/storage/sqlite/expected/52 deleted file mode 100644 index 844af318e4..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/52 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"Entries":9}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/53 b/tests/unit/C/services/storage/sqlite/expected/53 deleted file mode 100644 index cfd0e7926b..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/53 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"sum_id":43}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/54 b/tests/unit/C/services/storage/sqlite/expected/54 deleted file mode 100644 index 21df9c0d48..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/54 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "appended", "readings_added" : 100 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/55 b/tests/unit/C/services/storage/sqlite/expected/55 deleted file mode 100644 index c5748d1a33..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/55 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/56 b/tests/unit/C/services/storage/sqlite/expected/56 deleted file mode 100644 index 2e72fb850e..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/56 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/57 b/tests/unit/C/services/storage/sqlite/expected/57 deleted file mode 100644 index 22751a50d2..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/57 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/58 b/tests/unit/C/services/storage/sqlite/expected/58 deleted file mode 100644 index b69be284c3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/58 +++ /dev/null @@ -1 +0,0 @@ -{"count":6,"rows":[{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/59 b/tests/unit/C/services/storage/sqlite/expected/59 deleted file mode 100644 index ef2efe563d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/59 +++ /dev/null @@ -1 +0,0 @@ -{"count":4,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/6 b/tests/unit/C/services/storage/sqlite/expected/6 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/6 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/60 b/tests/unit/C/services/storage/sqlite/expected/60 deleted file mode 100644 index 4488386053..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/60 +++ /dev/null @@ -1 +0,0 @@ -{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/61 b/tests/unit/C/services/storage/sqlite/expected/61 deleted file mode 100644 index 7300a791ac..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/61 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "update", "message" : "No rows where updated", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/62 b/tests/unit/C/services/storage/sqlite/expected/62 deleted file mode 100644 index 4488386053..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/62 +++ /dev/null @@ -1 +0,0 @@ -{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/63 b/tests/unit/C/services/storage/sqlite/expected/63 deleted file mode 100644 index f62302483f..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/63 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"Count":100,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/64 b/tests/unit/C/services/storage/sqlite/expected/64 deleted file mode 100644 index 421c29b18e..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/64 +++ /dev/null @@ -1 +0,0 @@ -{"count":100,"rows":[{"timestamp":"2017-10-11 15:10:51.927","Rate":90},{"timestamp":"2017-10-11 15:10:51.930","Rate":13},{"timestamp":"2017-10-11 15:10:51.933","Rate":84},{"timestamp":"2017-10-11 15:10:51.936","Rate":96},{"timestamp":"2017-10-11 15:10:51.939","Rate":2},{"timestamp":"2017-10-11 15:10:51.942","Rate":54},{"timestamp":"2017-10-11 15:10:51.946","Rate":28},{"timestamp":"2017-10-11 15:10:51.949","Rate":3},{"timestamp":"2017-10-11 15:10:51.952","Rate":77},{"timestamp":"2017-10-11 15:10:51.955","Rate":38},{"timestamp":"2017-10-11 15:10:51.959","Rate":26},{"timestamp":"2017-10-11 15:10:51.963","Rate":86},{"timestamp":"2017-10-11 15:10:51.966","Rate":39},{"timestamp":"2017-10-11 15:10:51.970","Rate":57},{"timestamp":"2017-10-11 15:10:51.973","Rate":73},{"timestamp":"2017-10-11 15:10:51.979","Rate":22},{"timestamp":"2017-10-11 15:10:51.982","Rate":34},{"timestamp":"2017-10-11 15:10:51.986","Rate":78},{"timestamp":"2017-10-11 15:10:51.990","Rate":20},{"timestamp":"2017-10-11 15:10:51.993","Rate":70},{"timestamp":"2017-10-11 15:10:51.996","Rate":17},{"timestamp":"2017-10-11 15:10:52.000","Rate":2},{"timestamp":"2017-10-11 15:10:52.005","Rate":18},{"timestamp":"2017-10-11 15:10:52.009","Rate":52},{"timestamp":"2017-10-11 15:10:52.012","Rate":62},{"timestamp":"2017-10-11 15:10:52.015","Rate":47},{"timestamp":"2017-10-11 15:10:52.019","Rate":73},{"timestamp":"2017-10-11 15:10:52.022","Rate":9},{"timestamp":"2017-10-11 15:10:52.026","Rate":66},{"timestamp":"2017-10-11 15:10:52.029","Rate":30},{"timestamp":"2017-10-11 15:10:52.031","Rate":70},{"timestamp":"2017-10-11 15:10:52.034","Rate":41},{"timestamp":"2017-10-11 15:10:52.037","Rate":2},{"timestamp":"2017-10-11 15:10:52.040","Rate":69},{"timestamp":"2017-10-11 15:10:52.043","Rate":98},{"timestamp":"2017-10-11 15:10:52.046","Rate":13},{"timestamp":"2017-10-11 15:10:52.050","Rate":91},{"timestamp":"2017-10-11 15:10:52.053","Rate":18},{"timestamp":"2017-10-11 15:10:52.056","Rate":78},{"timestamp":"2017-10-11 15:10:52.059","Rate":70},{"timestamp":"2017-10-11 15:10:52.062","Rate":48},{"timestamp":"2017-10-11 15:10:52.066","Rate":94},{"timestamp":"2017-10-11 15:10:52.070","Rate":79},{"timestamp":"2017-10-11 15:10:52.073","Rate":87},{"timestamp":"2017-10-11 15:10:52.075","Rate":60},{"timestamp":"2017-10-11 15:10:52.078","Rate":48},{"timestamp":"2017-10-11 15:10:52.081","Rate":88},{"timestamp":"2017-10-11 15:10:52.084","Rate":3},{"timestamp":"2017-10-11 15:10:52.086","Rate":93},{"timestamp":"2017-10-11 15:10:52.089","Rate":83},{"timestamp":"2017-10-11 15:10:52.092","Rate":76},{"timestamp":"2017-10-11 15:10:52.095","Rate":97},{"timestamp":"2017-10-11 15:10:52.098","Rate":31},{"timestamp":"2017-10-11 15:10:52.100","Rate":49},{"timestamp":"2017-10-11 15:10:52.103","Rate":36},{"timestamp":"2017-10-11 15:10:52.106","Rate":15},{"timestamp":"2017-10-11 15:10:52.109","Rate":67},{"timestamp":"2017-10-11 15:10:52.111","Rate":67},{"timestamp":"2017-10-11 15:10:52.114","Rate":94},{"timestamp":"2017-10-11 15:10:52.116","Rate":68},{"timestamp":"2017-10-11 15:10:52.119","Rate":22},{"timestamp":"2017-10-11 15:10:52.122","Rate":54},{"timestamp":"2017-10-11 15:10:52.124","Rate":94},{"timestamp":"2017-10-11 15:10:52.127","Rate":49},{"timestamp":"2017-10-11 15:10:52.130","Rate":59},{"timestamp":"2017-10-11 15:10:52.132","Rate":6},{"timestamp":"2017-10-11 15:10:52.135","Rate":82},{"timestamp":"2017-10-11 15:10:52.137","Rate":5},{"timestamp":"2017-10-11 15:10:52.140","Rate":1},{"timestamp":"2017-10-11 15:10:52.142","Rate":53},{"timestamp":"2017-10-11 15:10:52.145","Rate":69},{"timestamp":"2017-10-11 15:10:52.147","Rate":97},{"timestamp":"2017-10-11 15:10:52.150","Rate":58},{"timestamp":"2017-10-11 15:10:52.153","Rate":76},{"timestamp":"2017-10-11 15:10:52.157","Rate":81},{"timestamp":"2017-10-11 15:10:52.160","Rate":30},{"timestamp":"2017-10-11 15:10:52.163","Rate":4},{"timestamp":"2017-10-11 15:10:52.165","Rate":67},{"timestamp":"2017-10-11 15:10:52.169","Rate":5},{"timestamp":"2017-10-11 15:10:52.171","Rate":72},{"timestamp":"2017-10-11 15:10:52.174","Rate":20},{"timestamp":"2017-10-11 15:10:52.176","Rate":58},{"timestamp":"2017-10-11 15:10:52.179","Rate":75},{"timestamp":"2017-10-11 15:10:52.182","Rate":74},{"timestamp":"2017-10-11 15:10:52.184","Rate":60},{"timestamp":"2017-10-11 15:10:52.187","Rate":96},{"timestamp":"2017-10-11 15:10:52.189","Rate":30},{"timestamp":"2017-10-11 15:10:52.192","Rate":40},{"timestamp":"2017-10-11 15:10:52.195","Rate":33},{"timestamp":"2017-10-11 15:10:52.197","Rate":87},{"timestamp":"2017-10-11 15:10:52.200","Rate":67},{"timestamp":"2017-10-11 15:10:52.203","Rate":40},{"timestamp":"2017-10-11 15:10:52.206","Rate":44},{"timestamp":"2017-10-11 15:10:52.208","Rate":7},{"timestamp":"2017-10-11 15:10:52.211","Rate":52},{"timestamp":"2017-10-11 15:10:52.214","Rate":93},{"timestamp":"2017-10-11 15:10:52.219","Rate":43},{"timestamp":"2017-10-11 15:10:52.222","Rate":66},{"timestamp":"2017-10-11 15:10:52.225","Rate":8},{"timestamp":"2017-10-11 15:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/65 b/tests/unit/C/services/storage/sqlite/expected/65 deleted file mode 100644 index a9bebbfede..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/65 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 4 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/66 b/tests/unit/C/services/storage/sqlite/expected/66 deleted file mode 100644 index 6bcad7bc37..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/66 +++ /dev/null @@ -1 +0,0 @@ -{"count":4,"rows":[{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/67 b/tests/unit/C/services/storage/sqlite/expected/67 deleted file mode 100644 index c3fd30822b..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/67 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"description":"A test row"},{"description":"Updated with expression"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/68 b/tests/unit/C/services/storage/sqlite/expected/68 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/68 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/69 b/tests/unit/C/services/storage/sqlite/expected/69 deleted file mode 100644 index a5d4634456..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/69 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}},{"id":1,"key":"TEST1","description":"A test row","data":{"json":"new value"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/7 b/tests/unit/C/services/storage/sqlite/expected/7 deleted file mode 100644 index 379611dc02..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/7 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"count_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/70 b/tests/unit/C/services/storage/sqlite/expected/70 deleted file mode 100644 index 7d66483ebe..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/70 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/71 b/tests/unit/C/services/storage/sqlite/expected/71 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/71 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/72 b/tests/unit/C/services/storage/sqlite/expected/72 deleted file mode 100644 index 858823dc3c..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/72 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"id":4,"key":"Admin","description":"URL of the admin API","data":{"url":{"value":"new value"}}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/73 b/tests/unit/C/services/storage/sqlite/expected/73 deleted file mode 100644 index 7d66483ebe..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/73 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/74 b/tests/unit/C/services/storage/sqlite/expected/74 deleted file mode 100644 index 4bde9235b1..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/74 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"Count":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/75 b/tests/unit/C/services/storage/sqlite/expected/75 deleted file mode 100644 index 973df944e3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/75 +++ /dev/null @@ -1 +0,0 @@ -{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/76 b/tests/unit/C/services/storage/sqlite/expected/76 deleted file mode 100644 index 7e9c80429f..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/76 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"value\" of an \"newer\" condition must be an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/77 b/tests/unit/C/services/storage/sqlite/expected/77 deleted file mode 100644 index 88ff5140f2..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/77 +++ /dev/null @@ -1 +0,0 @@ -{"count":5,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/78 b/tests/unit/C/services/storage/sqlite/expected/78 deleted file mode 100644 index f22a289656..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/78 +++ /dev/null @@ -1 +0,0 @@ -{"count":2,"rows":[{"min":2,"max":96,"average":47.9523809523809,"user_ts":"2017-10-11 15:10:51"},{"min":1,"max":98,"average":53.7721518987342,"user_ts":"2017-10-11 15:10:52"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/79 b/tests/unit/C/services/storage/sqlite/expected/79 deleted file mode 100644 index 7d66483ebe..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/79 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/8 b/tests/unit/C/services/storage/sqlite/expected/8 deleted file mode 100644 index cd34346b4c..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/8 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"avg_id":1.0}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/80 b/tests/unit/C/services/storage/sqlite/expected/80 deleted file mode 100644 index 29146ee48a..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/80 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/81 b/tests/unit/C/services/storage/sqlite/expected/81 deleted file mode 100644 index c614b3d6e0..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/81 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "appendReadings", "message" : "Payload is missing a readings array", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/82 b/tests/unit/C/services/storage/sqlite/expected/82 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/82 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/83 b/tests/unit/C/services/storage/sqlite/expected/83 deleted file mode 100644 index 421c29b18e..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/83 +++ /dev/null @@ -1 +0,0 @@ -{"count":100,"rows":[{"timestamp":"2017-10-11 15:10:51.927","Rate":90},{"timestamp":"2017-10-11 15:10:51.930","Rate":13},{"timestamp":"2017-10-11 15:10:51.933","Rate":84},{"timestamp":"2017-10-11 15:10:51.936","Rate":96},{"timestamp":"2017-10-11 15:10:51.939","Rate":2},{"timestamp":"2017-10-11 15:10:51.942","Rate":54},{"timestamp":"2017-10-11 15:10:51.946","Rate":28},{"timestamp":"2017-10-11 15:10:51.949","Rate":3},{"timestamp":"2017-10-11 15:10:51.952","Rate":77},{"timestamp":"2017-10-11 15:10:51.955","Rate":38},{"timestamp":"2017-10-11 15:10:51.959","Rate":26},{"timestamp":"2017-10-11 15:10:51.963","Rate":86},{"timestamp":"2017-10-11 15:10:51.966","Rate":39},{"timestamp":"2017-10-11 15:10:51.970","Rate":57},{"timestamp":"2017-10-11 15:10:51.973","Rate":73},{"timestamp":"2017-10-11 15:10:51.979","Rate":22},{"timestamp":"2017-10-11 15:10:51.982","Rate":34},{"timestamp":"2017-10-11 15:10:51.986","Rate":78},{"timestamp":"2017-10-11 15:10:51.990","Rate":20},{"timestamp":"2017-10-11 15:10:51.993","Rate":70},{"timestamp":"2017-10-11 15:10:51.996","Rate":17},{"timestamp":"2017-10-11 15:10:52.000","Rate":2},{"timestamp":"2017-10-11 15:10:52.005","Rate":18},{"timestamp":"2017-10-11 15:10:52.009","Rate":52},{"timestamp":"2017-10-11 15:10:52.012","Rate":62},{"timestamp":"2017-10-11 15:10:52.015","Rate":47},{"timestamp":"2017-10-11 15:10:52.019","Rate":73},{"timestamp":"2017-10-11 15:10:52.022","Rate":9},{"timestamp":"2017-10-11 15:10:52.026","Rate":66},{"timestamp":"2017-10-11 15:10:52.029","Rate":30},{"timestamp":"2017-10-11 15:10:52.031","Rate":70},{"timestamp":"2017-10-11 15:10:52.034","Rate":41},{"timestamp":"2017-10-11 15:10:52.037","Rate":2},{"timestamp":"2017-10-11 15:10:52.040","Rate":69},{"timestamp":"2017-10-11 15:10:52.043","Rate":98},{"timestamp":"2017-10-11 15:10:52.046","Rate":13},{"timestamp":"2017-10-11 15:10:52.050","Rate":91},{"timestamp":"2017-10-11 15:10:52.053","Rate":18},{"timestamp":"2017-10-11 15:10:52.056","Rate":78},{"timestamp":"2017-10-11 15:10:52.059","Rate":70},{"timestamp":"2017-10-11 15:10:52.062","Rate":48},{"timestamp":"2017-10-11 15:10:52.066","Rate":94},{"timestamp":"2017-10-11 15:10:52.070","Rate":79},{"timestamp":"2017-10-11 15:10:52.073","Rate":87},{"timestamp":"2017-10-11 15:10:52.075","Rate":60},{"timestamp":"2017-10-11 15:10:52.078","Rate":48},{"timestamp":"2017-10-11 15:10:52.081","Rate":88},{"timestamp":"2017-10-11 15:10:52.084","Rate":3},{"timestamp":"2017-10-11 15:10:52.086","Rate":93},{"timestamp":"2017-10-11 15:10:52.089","Rate":83},{"timestamp":"2017-10-11 15:10:52.092","Rate":76},{"timestamp":"2017-10-11 15:10:52.095","Rate":97},{"timestamp":"2017-10-11 15:10:52.098","Rate":31},{"timestamp":"2017-10-11 15:10:52.100","Rate":49},{"timestamp":"2017-10-11 15:10:52.103","Rate":36},{"timestamp":"2017-10-11 15:10:52.106","Rate":15},{"timestamp":"2017-10-11 15:10:52.109","Rate":67},{"timestamp":"2017-10-11 15:10:52.111","Rate":67},{"timestamp":"2017-10-11 15:10:52.114","Rate":94},{"timestamp":"2017-10-11 15:10:52.116","Rate":68},{"timestamp":"2017-10-11 15:10:52.119","Rate":22},{"timestamp":"2017-10-11 15:10:52.122","Rate":54},{"timestamp":"2017-10-11 15:10:52.124","Rate":94},{"timestamp":"2017-10-11 15:10:52.127","Rate":49},{"timestamp":"2017-10-11 15:10:52.130","Rate":59},{"timestamp":"2017-10-11 15:10:52.132","Rate":6},{"timestamp":"2017-10-11 15:10:52.135","Rate":82},{"timestamp":"2017-10-11 15:10:52.137","Rate":5},{"timestamp":"2017-10-11 15:10:52.140","Rate":1},{"timestamp":"2017-10-11 15:10:52.142","Rate":53},{"timestamp":"2017-10-11 15:10:52.145","Rate":69},{"timestamp":"2017-10-11 15:10:52.147","Rate":97},{"timestamp":"2017-10-11 15:10:52.150","Rate":58},{"timestamp":"2017-10-11 15:10:52.153","Rate":76},{"timestamp":"2017-10-11 15:10:52.157","Rate":81},{"timestamp":"2017-10-11 15:10:52.160","Rate":30},{"timestamp":"2017-10-11 15:10:52.163","Rate":4},{"timestamp":"2017-10-11 15:10:52.165","Rate":67},{"timestamp":"2017-10-11 15:10:52.169","Rate":5},{"timestamp":"2017-10-11 15:10:52.171","Rate":72},{"timestamp":"2017-10-11 15:10:52.174","Rate":20},{"timestamp":"2017-10-11 15:10:52.176","Rate":58},{"timestamp":"2017-10-11 15:10:52.179","Rate":75},{"timestamp":"2017-10-11 15:10:52.182","Rate":74},{"timestamp":"2017-10-11 15:10:52.184","Rate":60},{"timestamp":"2017-10-11 15:10:52.187","Rate":96},{"timestamp":"2017-10-11 15:10:52.189","Rate":30},{"timestamp":"2017-10-11 15:10:52.192","Rate":40},{"timestamp":"2017-10-11 15:10:52.195","Rate":33},{"timestamp":"2017-10-11 15:10:52.197","Rate":87},{"timestamp":"2017-10-11 15:10:52.200","Rate":67},{"timestamp":"2017-10-11 15:10:52.203","Rate":40},{"timestamp":"2017-10-11 15:10:52.206","Rate":44},{"timestamp":"2017-10-11 15:10:52.208","Rate":7},{"timestamp":"2017-10-11 15:10:52.211","Rate":52},{"timestamp":"2017-10-11 15:10:52.214","Rate":93},{"timestamp":"2017-10-11 15:10:52.219","Rate":43},{"timestamp":"2017-10-11 15:10:52.222","Rate":66},{"timestamp":"2017-10-11 15:10:52.225","Rate":8},{"timestamp":"2017-10-11 15:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/84 b/tests/unit/C/services/storage/sqlite/expected/84 deleted file mode 100644 index 6e3d42ac03..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/84 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "limit", "message" : "Limit must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/85 b/tests/unit/C/services/storage/sqlite/expected/85 deleted file mode 100644 index 8d705bbb14..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/85 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "skip", "message" : "Skip must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/86 b/tests/unit/C/services/storage/sqlite/expected/86 deleted file mode 100644 index 4522ec1b85..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/86 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "unrecognized token: \":\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/87 b/tests/unit/C/services/storage/sqlite/expected/87 deleted file mode 100644 index 1d6fe91f5d..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/87 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "retrieve", "message" : "SQLite3 plugin does not support timezones in qeueries", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/88 b/tests/unit/C/services/storage/sqlite/expected/88 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/88 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/89 b/tests/unit/C/services/storage/sqlite/expected/89 deleted file mode 100644 index 93af3344da..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/89 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"description":"added'some'ch'''ars'"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/9 b/tests/unit/C/services/storage/sqlite/expected/9 deleted file mode 100644 index f951388d78..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/9 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/90 b/tests/unit/C/services/storage/sqlite/expected/90 deleted file mode 100644 index a8848cbc86..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/90 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/91 b/tests/unit/C/services/storage/sqlite/expected/91 deleted file mode 100644 index f70a6b7ed5..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/91 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 15:10"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/92 b/tests/unit/C/services/storage/sqlite/expected/92 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/92 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/93 b/tests/unit/C/services/storage/sqlite/expected/93 deleted file mode 100644 index a4bfc7b379..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/93 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min":"","max":"","average":""}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/94 b/tests/unit/C/services/storage/sqlite/expected/94 deleted file mode 100644 index 26bdeeb2ac..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/94 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 15"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/95 b/tests/unit/C/services/storage/sqlite/expected/95 deleted file mode 100644 index 2eeb09a60b..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/95 +++ /dev/null @@ -1 +0,0 @@ -{ "response" : "appended", "readings_added" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/96 b/tests/unit/C/services/storage/sqlite/expected/96 deleted file mode 100644 index 124e6704de..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/96 +++ /dev/null @@ -1 +0,0 @@ -{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/97 b/tests/unit/C/services/storage/sqlite/expected/97 deleted file mode 100644 index 22f00fc70c..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/97 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"count_id":10}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/98 b/tests/unit/C/services/storage/sqlite/expected/98 deleted file mode 100644 index 69ab957da3..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/98 +++ /dev/null @@ -1 +0,0 @@ -{ "entryPoint" : "where clause", "message" : "The \"value\" of a \"in\" condition must be an array and must not be empty.", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/99 b/tests/unit/C/services/storage/sqlite/expected/99 deleted file mode 100644 index cfddbd8a1a..0000000000 --- a/tests/unit/C/services/storage/sqlite/expected/99 +++ /dev/null @@ -1 +0,0 @@ -{"count":1,"rows":[{"min":1,"max":98,"average":52.9207920792079,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected/102 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 similarity index 100% rename from tests/unit/C/services/storage/sqlite/expected/102 rename to tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 diff --git a/tests/unit/C/services/storage/sqlite/expected/103 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 similarity index 100% rename from tests/unit/C/services/storage/sqlite/expected/103 rename to tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 diff --git a/tests/unit/C/services/storage/sqlite/expected/104 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 similarity index 100% rename from tests/unit/C/services/storage/sqlite/expected/104 rename to tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 new file mode 100644 index 0000000000..804a3da17d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 @@ -0,0 +1 @@ +{ "removed" : 100, "unsentPurged" : 100, "unsentRetained" : 1, "readings" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 new file mode 100644 index 0000000000..d84da94821 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 new file mode 100644 index 0000000000..b0e610c297 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 @@ -0,0 +1 @@ +{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 10:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 10:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 10:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 09:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 11:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 07:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 12:33:05.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/testRunner.sh b/tests/unit/C/services/storage/sqlite/testRunner.sh index 7e8be98cc3..1b19a6b6bc 100755 --- a/tests/unit/C/services/storage/sqlite/testRunner.sh +++ b/tests/unit/C/services/storage/sqlite/testRunner.sh @@ -1,19 +1,59 @@ -#!/bin/sh - -# Select the required plugin, either persistent storage or in memory -export FOGLAMP_DATA=./plugin_cfg/sqlite -#export FOGLAMP_DATA=./plugin_cfg/sqlitememory +#!/usr/bin/env bash +# Default values +export FOGLAMP_DATA=./plugin_cfg/sqlite # Select the persistent storage plugin +export storage_file="" export TZ='Etc/UTC' -if [ $# -eq 1 ] ; then - echo Starting storage layer $1 - $1 -elif [ "${FOGLAMP_ROOT}" != "" ] ; then +# +# evaluates : FOGLAMP_DATA, storage_file, TZ, and expected_dir +# +if [[ "$@" != "" ]]; +then + # Handles input parameters + SCRIPT_NAME=`basename $0` + options=`getopt -o c:f:t: --long configuration:,storage_file:,timezone: -n "$SCRIPT_NAME" -- "$@"` + eval set -- "$options" + + while true ; do + case "$1" in + -c|--configuration) + export FOGLAMP_DATA="$2" + shift 2 + ;; + + -f|--storage_file) + export storage_file="$2" + shift 2 + ;; + + -t|--timezone) + export TZ="$2" + shift 2 + ;; + --) + shift + break + ;; + esac + done +fi + +# Converts '/' to '_' and to upper case +step1="${TZ/\//_}" +expected_dir="expected_${step1^^}" + +if [[ "$storage_file" != "" ]] ; then + echo "Starting storage layer :$storage_file:" + echo "timezone :$TZ:" + echo "configuration :$FOGLAMP_DATA:" + echo "database file :$DEFAULT_SQLITE_DB_FILE:" + $storage_file +elif [[ "${FOGLAMP_ROOT}" != "" ]] ; then echo "Starting storage service in :$FOGLAMP_ROOT:" - echo "timezone :$TZ" + echo "timezone :$TZ:" echo "configuration :$FOGLAMP_DATA:" - echo "database :$DEFAULT_SQLITE_DB_FILE:" + echo "database file :$DEFAULT_SQLITE_DB_FILE:" $FOGLAMP_ROOT/services/foglamp.services.storage sleep 1 else @@ -21,6 +61,9 @@ else exit 1 fi +# +# Main +# export IFS="," testNum=1 n_failed=0 @@ -41,11 +84,11 @@ else curlstate=$? fi if [ "$optional" = "" ] ; then - if [ ! -f expected/$testNum ]; then + if [ ! -f ${expected_dir}/$testNum ]; then n_unchecked=`expr $n_unchecked + 1` - echo Missing expected results for test $testNum - result unchecked + echo Missing expected results in :${expected_dir}: for test $testNum - result unchecked else - cmp -s results/$testNum expected/$testNum + cmp -s results/$testNum ${expected_dir}/$testNum if [ $? -ne "0" ]; then echo Failed n_failed=`expr $n_failed + 1` @@ -57,7 +100,7 @@ if [ "$optional" = "" ] ; then fi ( unset IFS - echo " " Expected: "`cat expected/$testNum`" >> failed + echo " " Expected: "`cat ${expected_dir}/$testNum`" >> failed echo " " Got: "`cat results/$testNum`" >> failed ) echo >> failed diff --git a/tests/unit/C/services/storage/sqlite/testset b/tests/unit/C/services/storage/sqlite/testset index fa672be38f..a9002125ae 100644 --- a/tests/unit/C/services/storage/sqlite/testset +++ b/tests/unit/C/services/storage/sqlite/testset @@ -1,109 +1,4 @@ -Common Read,GET,http://localhost:8080/storage/table/test, -Common Read key,GET,http://localhost:8080/storage/table/test?id=1, -Common Read key empty,GET,http://localhost:8080/storage/table/test?id=2, -Common Read complex,PUT,http://localhost:8080/storage/table/test/query,where_code_1.json -Common Read complex empty,PUT,http://localhost:8080/storage/table/test/query,where_id_2.json -Common Read complex not equal,PUT,http://localhost:8080/storage/table/test/query,where_id_not_1.json -Common Read complex count,PUT,http://localhost:8080/storage/table/test/query,where_count.json -Common Read complex avg,PUT,http://localhost:8080/storage/table/test/query,where_avg.json -Common Read complex min,PUT,http://localhost:8080/storage/table/test/query,where_min.json -Common Read complex max,PUT,http://localhost:8080/storage/table/test/query,where_max.json -Common Insert,POST,http://localhost:8080/storage/table/test,insert.json -Common Read back,GET,http://localhost:8080/storage/table/test?id=2, -Common Insert bad column,POST,http://localhost:8080/storage/table/test,insert_bad.json -Common Insert bad syntax,POST,http://localhost:8080/storage/table/test,insert_bad2.json -Common Delete,DELETE,http://localhost:8080/storage/table/test,where_id_2.json -Common Read deleted,GET,http://localhost:8080/storage/table/test?id=2, -Common Delete non-existant,DELETE,http://localhost:8080/storage/table/test,where_id_2.json -Common Insert,POST,http://localhost:8080/storage/table/test,insert.json -Common Read limit,PUT,http://localhost:8080/storage/table/test/query,limit.json -Common Read skip,PUT,http://localhost:8080/storage/table/test/query,skip.json -Common Read bad 1,PUT,http://localhost:8080/storage/table/test/query,where_bad_1.json -Common Read bad 2,PUT,http://localhost:8080/storage/table/test/query,where_bad_2.json -Common Read bad 3,PUT,http://localhost:8080/storage/table/test/query,where_bad_3.json -Common Read bad 4,PUT,http://localhost:8080/storage/table/test/query,where_bad_4.json -Common Read default sort order,PUT,http://localhost:8080/storage/table/test/query,bad_sort_1.json -Common Read bad sort 2,PUT,http://localhost:8080/storage/table/test/query,bad_sort_2.json -Common Update,PUT,http://localhost:8080/storage/table/test,update.json -Common Read back,GET,http://localhost:8080/storage/table/test?id=2, -Common Update,PUT,http://localhost:8080/storage/table/test,updateKey.json -Common Read back,GET,http://localhost:8080/storage/table/test?key=UPDA, -Common Update no values,PUT,http://localhost:8080/storage/table/test,bad_update.json -Common Read group,PUT,http://localhost:8080/storage/table/test/query,group.json -Bad URL,GET,http://localhost:8080/foglamp/nothing, -Bad table,GET,http://localhost:8080/storage/table/doesntexist, -Bad column,GET,http://localhost:8080/storage/table/test?doesntexist=9, -Ping interface,GET,http://localhost:1081/foglamp/service/ping,,checkstate -Add Readings,POST,http://localhost:8080/storage/reading,asset.json -Fetch Readings,GET,http://localhost:8080/storage/reading?id=1&count=1000,,checkstate -Fetch Readings zero count,GET,http://localhost:8080/storage/reading?id=1&count=0, -Fetch Readings no count,GET,http://localhost:8080/storage/reading?id=1, -Fetch Readings no id,GET,http://localhost:8080/storage/reading?count=1000, -Purge Readings,PUT,http://localhost:8080/storage/reading/purge?age=1000&sent=10&flags=purge, -Common Read sort array,PUT,http://localhost:8080/storage/table/test/query,sort2.json -Common Read multiple aggregates,PUT,http://localhost:8080/storage/table/test/query,where_multi_aggregatee.json, -Common Read columns,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r1.json, -Common Read columns alias,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r2.json, -Common Read columns json,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r3.json, -Date format2,PUT,http://localhost:8080/storage/table/test2/query,where_test2_d2.json -Date format4,PUT,http://localhost:8080/storage/table/test2/query,where_test2_d4.json -Bad format1,PUT,http://localhost:8080/storage/table/test2/query,where_bad_format1.json -Bad format2,PUT,http://localhost:8080/storage/table/test2/query,where_bad_format2.json -Count star,PUT,http://localhost:8080/storage/table/test2/query,where_count_star.json -sum,PUT,http://localhost:8080/storage/table/test2/query,where_sum.json -Add more Readings,POST,http://localhost:8080/storage/reading,readings.json -Query Readings,PUT,http://localhost:8080/storage/reading/query,query_readings.json -Query Readings Timebucket,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket.json -Query Readings Timebucket 1,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket1.json -Multi And,PUT,http://localhost:8080/storage/table/test2/query,multi_and.json -Multi Or,PUT,http://localhost:8080/storage/table/test2/query,multi_or.json -Multi Mixed,PUT,http://localhost:8080/storage/table/test2/query,multi_mised.json -Update Bad Condition,PUT,http://localhost:8080/storage/table/test2,update_bad.json -Read back,GET,http://localhost:8080/storage/table/test2, -Count Assets,PUT,http://localhost:8080/storage/reading/query,count_assets.json -Reading Rate,PUT,http://localhost:8080/storage/reading/query,reading_property.json -Update expression,PUT,http://localhost:8080/storage/table/test2,update_expression.json -Read back update,PUT,http://localhost:8080/storage/table/test2/query,read_id_1xx.json -Distinct,PUT,http://localhost:8080/storage/table/test2/query,where_distinct.json -Update JSON,PUT,http://localhost:8080/storage/table/test,update_json.json -Read back update,PUT,http://localhost:8080/storage/table/test/query,sort.json -Add JSON,POST,http://localhost:8080/storage/table/test,insert2.json -Update Complex JSON,PUT,http://localhost:8080/storage/table/test,update_json2.json -Read back update,GET,http://localhost:8080/storage/table/test?id=4, -Add now,POST,http://localhost:8080/storage/table/test2,addnew.json -Newer,PUT,http://localhost:8080/storage/table/test2/query,newer.json -Older,PUT,http://localhost:8080/storage/table/test2/query,older.json -Newer Bad,PUT,http://localhost:8080/storage/table/test2/query,newerBad.json -Like,PUT,http://localhost:8080/storage/table/test2/query,where_like.json -Group Time,PUT,http://localhost:8080/storage/reading/query,group_time.json -Jira FOGL-690,POST,http://localhost:8080/storage/table/configuration,error-fogl690.json -Jira FOGL-690 cleanup,DELETE,http://localhost:8080/storage/table/configuration,delete.json -Add bad Readings,POST,http://localhost:8080/storage/reading,badreadings.json -Query Readings Timebucket Bad,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket_bad.json -Reading Rate Array,PUT,http://localhost:8080/storage/reading/query,reading_property_array.json -Common Read limit max_int,PUT,http://localhost:8080/storage/table/test/query,limit_max_int.json -Common Read skip max_int,PUT,http://localhost:8080/storage/table/test/query,skip_max_int.json -Timezone,PUT,http://localhost:8080/storage/table/test2/query,timezone.json -Bad Timezone,PUT,http://localhost:8080/storage/table/test2/query,timezone_bad.json -Set-FOGL-983,PUT,http://localhost:8080/storage/table/configuration,FOGL-983.json -Get-FOGL-983,PUT,http://localhost:8080/storage/table/configuration/query,get-FOGL-983.json -Update now,PUT,http://localhost:8080/storage/table/test2,updatenow.json -Get Reading series group by minutes,PUT,http://localhost:8080/storage/reading/query,series_group_by_minutes.json -Get Reading series (seconds),PUT,http://localhost:8080/storage/reading/query,series_seconds.json -Get Reading series summary (seconds),PUT,http://localhost:8080/storage/reading/query,series_summary_seconds.json -Get Reading series group by hours,PUT,http://localhost:8080/storage/reading/query,series_group_by_hours.json -Add Readings now,POST,http://localhost:8080/storage/reading,add_readings_now.json -Common table IN operator,PUT,http://localhost:8080/storage/table/test2/query,where_in.json -Common table NOT IN operator,PUT,http://localhost:8080/storage/table/test2/query,where_not_in.json -Common table IN operator bad values,PUT,http://localhost:8080/storage/table/test2/query,where_in_bad_values.json -Query Readings IN operator,PUT,http://localhost:8080/storage/reading/query,query_readings_in.json -Query Readings NOT IN operator,PUT,http://localhost:8080/storage/reading/query,query_readings_not_in.json -Query Readings IN operator bad values,PUT,http://localhost:8080/storage/reading/query,query_readings_in_bad_values.json microseconds - Purge Readings,PUT,http://localhost:8080/storage/reading/purge?age=1&sent=0&flags=purge, microseconds - Add Readings,POST,http://localhost:8080/storage/reading,msec_add_readings_user_ts.json microseconds - Query Readings,PUT,http://localhost:8080/storage/reading/query,msec_query_readings.json -microseconds - Query asset NO alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_noalias.json -microseconds - Query asset alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_alias.json -microseconds - Query asset aggregate min,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggmin.json -microseconds - Query asset aggregate min array,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggminarray.json -Shutdown,POST,http://localhost:1081/foglamp/service/shutdown,,checkstate + From fab4791d376cf46adf2095514f8ddd3427a225b5 Mon Sep 17 00:00:00 2001 From: stefano Date: Tue, 5 Mar 2019 15:27:58 +0100 Subject: [PATCH 65/86] FOGL-2545: improved implementation and applied latest changes done in the SQLite in memroy plugin to the SQLite plugin --- C/plugins/storage/sqlite/connection.cpp | 76 ++++++++++--- C/plugins/storage/sqlite/include/connection.h | 3 +- .../storage/sqlite/expected_ETC_UTC/1 | 2 +- .../storage/sqlite/expected_ETC_UTC/2 | 2 +- .../storage/sqlite/expected_ETC_UTC/3 | 2 +- .../storage/sqlite/expected_EUROPE_ROME/3 | 2 +- .../C/services/storage/sqlite/testRunner.sh | 16 +-- tests/unit/C/services/storage/sqlite/testset | 107 +++++++++++++++++- 8 files changed, 178 insertions(+), 32 deletions(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index 17ef01af6d..eec93384dc 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -860,7 +860,7 @@ SQLBuffer jsonConstraints; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'utc')"); } @@ -1540,14 +1540,14 @@ bool Connection::formatDate(char *formatted_date, size_t buffer_size, const char // Extract up to seconds memset(&tm, 0, sizeof(tm)); - valid_date = strptime(date, "%Y-%m-%d %H:%M:%S", &tm); + valid_date = strptime(date, F_DATEH24_SEC, &tm); if (! valid_date) { return (false); } - strftime (formatted_date, buffer_size, "%Y-%m-%d %H:%M:%S", &tm); + strftime (formatted_date, buffer_size, F_DATEH24_SEC, &tm); // Work out the microseconds from the fractional part of the seconds char fractional[10] = {0}; @@ -1890,9 +1890,9 @@ bool isAggregate = false; asset_code, read_key, reading, - strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + strftime(')" F_DATEH24_SEC R"(', user_ts, 'localtime') || substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, - strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + strftime(')" F_DATEH24_MS R"(', ts, 'localtime') AS ts FROM foglamp.readings)"; sql.append(sql_cmd); @@ -1913,7 +1913,7 @@ bool isAggregate = false; sql.append(document["modifier"].GetString()); sql.append(' '); } - if (!jsonAggregates(document, document["aggregate"], sql, jsonConstraints)) + if (!jsonAggregates(document, document["aggregate"], sql, jsonConstraints, true)) { return false; } @@ -1940,7 +1940,23 @@ bool isAggregate = false; sql.append(", "); if (!itr->IsObject()) // Simple column name { - sql.append(itr->GetString()); + if (strcmp(itr->GetString() ,"user_ts") == 0) + { + // Display without TZ expression and microseconds also + sql.append(" strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + sql.append(" as user_ts "); + } + else if (strcmp(itr->GetString() ,"ts") == 0) + { + // Display without TZ expression and microseconds also + sql.append(" strftime('" F_DATEH24_MS "', ts, 'localtime') "); + sql.append(" as ts "); + } + else + { + sql.append(itr->GetString()); + } } else { @@ -1986,7 +2002,7 @@ bool isAggregate = false; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'utc') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'utc') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -1996,7 +2012,7 @@ bool isAggregate = false; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'utc')"); if (! itr->HasMember("alias")) @@ -2012,7 +2028,7 @@ bool isAggregate = false; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -2022,7 +2038,7 @@ bool isAggregate = false; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'localtime')"); if (! itr->HasMember("alias")) @@ -2046,7 +2062,7 @@ bool isAggregate = false; { // Extract milliseconds and microseconds for the user_ts fields - sql.append("strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') "); + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); if (! itr->HasMember("alias")) { @@ -2056,7 +2072,7 @@ bool isAggregate = false; } else { - sql.append("strftime('%Y-%m-%d %H:%M:%f', "); + sql.append("strftime('" F_DATEH24_MS "', "); sql.append((*itr)["column"].GetString()); sql.append(", 'localtime')"); if (! itr->HasMember("alias")) @@ -2106,9 +2122,9 @@ bool isAggregate = false; asset_code, read_key, reading, - strftime('%Y-%m-%d %H:%M:%S', user_ts, 'localtime') || + strftime()" F_DATEH24_SEC R"(', user_ts, 'localtime') || substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, - strftime('%Y-%m-%d %H:%M:%f', ts, 'localtime') AS ts + strftime(')" F_DATEH24_MS R"(', ts, 'localtime') AS ts FROM foglamp.)"; sql.append(sql_cmd); @@ -2482,7 +2498,8 @@ int blocks = 0; bool Connection::jsonAggregates(const Value& payload, const Value& aggregates, SQLBuffer& sql, - SQLBuffer& jsonConstraint) + SQLBuffer& jsonConstraint, + bool isTableReading) { if (aggregates.IsObject()) { @@ -2509,7 +2526,18 @@ bool Connection::jsonAggregates(const Value& payload, } else { - sql.append(col); + // an operation different from the 'count' is requested + if (isTableReading && (col.compare("user_ts") == 0) ) + { + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + } + else + { + sql.append("\""); + sql.append(col); + sql.append("\""); + } } } else if (aggregates.HasMember("json")) @@ -2642,7 +2670,19 @@ bool Connection::jsonAggregates(const Value& payload, sql.append('('); if (itr->HasMember("column")) { - sql.append((*itr)["column"].GetString()); + string column_name= (*itr)["column"].GetString(); + if (isTableReading && (column_name.compare("user_ts") == 0) ) + { + sql.append("strftime('" F_DATEH24_SEC "', user_ts, 'localtime') "); + sql.append(" || substr(user_ts, instr(user_ts, '.'), 7) "); + } + else + { + sql.append("\""); + sql.append(column_name); + sql.append("\""); + } + } else if (itr->HasMember("json")) { diff --git a/C/plugins/storage/sqlite/include/connection.h b/C/plugins/storage/sqlite/include/connection.h index 1eb04d8110..2b26164f6c 100644 --- a/C/plugins/storage/sqlite/include/connection.h +++ b/C/plugins/storage/sqlite/include/connection.h @@ -50,7 +50,8 @@ class Connection { bool jsonAggregates(const rapidjson::Value&, const rapidjson::Value&, SQLBuffer&, - SQLBuffer&); + SQLBuffer&, + bool isTableReading = false); bool returnJson(const rapidjson::Value&, SQLBuffer&, SQLBuffer&); char *trim(char *str); const char *escape(const char *); diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 index 804a3da17d..960917c42d 100644 --- a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/1 @@ -1 +1 @@ -{ "removed" : 100, "unsentPurged" : 100, "unsentRetained" : 1, "readings" : 1 } \ No newline at end of file +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 index d84da94821..960917c42d 100644 --- a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/2 @@ -1 +1 @@ -{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 index b0e610c297..124e6704de 100644 --- a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/3 @@ -1 +1 @@ -{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 10:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 10:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 10:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 09:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 11:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 07:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 12:33:05.123456"}]} \ No newline at end of file +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 index b0e610c297..cc1e058c38 100644 --- a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 @@ -1 +1 @@ -{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 10:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 10:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 10:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 09:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 11:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 07:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 12:33:05.123456"}]} \ No newline at end of file +{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 11:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 11:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 11:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 11:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 11:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 11:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 11:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 10:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 12:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 08:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 13:33:05.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/testRunner.sh b/tests/unit/C/services/storage/sqlite/testRunner.sh index 1b19a6b6bc..ebc80274ee 100755 --- a/tests/unit/C/services/storage/sqlite/testRunner.sh +++ b/tests/unit/C/services/storage/sqlite/testRunner.sh @@ -2,17 +2,17 @@ # Default values export FOGLAMP_DATA=./plugin_cfg/sqlite # Select the persistent storage plugin -export storage_file="" +export storage_exec="" export TZ='Etc/UTC' # -# evaluates : FOGLAMP_DATA, storage_file, TZ, and expected_dir +# evaluates : FOGLAMP_DATA, storage_exec, TZ, and expected_dir # if [[ "$@" != "" ]]; then # Handles input parameters SCRIPT_NAME=`basename $0` - options=`getopt -o c:f:t: --long configuration:,storage_file:,timezone: -n "$SCRIPT_NAME" -- "$@"` + options=`getopt -o c:s:t: --long configuration:,storage_exec:,timezone: -n "$SCRIPT_NAME" -- "$@"` eval set -- "$options" while true ; do @@ -22,8 +22,8 @@ then shift 2 ;; - -f|--storage_file) - export storage_file="$2" + -s|--storage_exec) + export storage_exec="$2" shift 2 ;; @@ -43,12 +43,12 @@ fi step1="${TZ/\//_}" expected_dir="expected_${step1^^}" -if [[ "$storage_file" != "" ]] ; then - echo "Starting storage layer :$storage_file:" +if [[ "$storage_exec" != "" ]] ; then + echo "Starting storage layer :$storage_exec:" echo "timezone :$TZ:" echo "configuration :$FOGLAMP_DATA:" echo "database file :$DEFAULT_SQLITE_DB_FILE:" - $storage_file + $storage_exec elif [[ "${FOGLAMP_ROOT}" != "" ]] ; then echo "Starting storage service in :$FOGLAMP_ROOT:" echo "timezone :$TZ:" diff --git a/tests/unit/C/services/storage/sqlite/testset b/tests/unit/C/services/storage/sqlite/testset index a9002125ae..fa672be38f 100644 --- a/tests/unit/C/services/storage/sqlite/testset +++ b/tests/unit/C/services/storage/sqlite/testset @@ -1,4 +1,109 @@ +Common Read,GET,http://localhost:8080/storage/table/test, +Common Read key,GET,http://localhost:8080/storage/table/test?id=1, +Common Read key empty,GET,http://localhost:8080/storage/table/test?id=2, +Common Read complex,PUT,http://localhost:8080/storage/table/test/query,where_code_1.json +Common Read complex empty,PUT,http://localhost:8080/storage/table/test/query,where_id_2.json +Common Read complex not equal,PUT,http://localhost:8080/storage/table/test/query,where_id_not_1.json +Common Read complex count,PUT,http://localhost:8080/storage/table/test/query,where_count.json +Common Read complex avg,PUT,http://localhost:8080/storage/table/test/query,where_avg.json +Common Read complex min,PUT,http://localhost:8080/storage/table/test/query,where_min.json +Common Read complex max,PUT,http://localhost:8080/storage/table/test/query,where_max.json +Common Insert,POST,http://localhost:8080/storage/table/test,insert.json +Common Read back,GET,http://localhost:8080/storage/table/test?id=2, +Common Insert bad column,POST,http://localhost:8080/storage/table/test,insert_bad.json +Common Insert bad syntax,POST,http://localhost:8080/storage/table/test,insert_bad2.json +Common Delete,DELETE,http://localhost:8080/storage/table/test,where_id_2.json +Common Read deleted,GET,http://localhost:8080/storage/table/test?id=2, +Common Delete non-existant,DELETE,http://localhost:8080/storage/table/test,where_id_2.json +Common Insert,POST,http://localhost:8080/storage/table/test,insert.json +Common Read limit,PUT,http://localhost:8080/storage/table/test/query,limit.json +Common Read skip,PUT,http://localhost:8080/storage/table/test/query,skip.json +Common Read bad 1,PUT,http://localhost:8080/storage/table/test/query,where_bad_1.json +Common Read bad 2,PUT,http://localhost:8080/storage/table/test/query,where_bad_2.json +Common Read bad 3,PUT,http://localhost:8080/storage/table/test/query,where_bad_3.json +Common Read bad 4,PUT,http://localhost:8080/storage/table/test/query,where_bad_4.json +Common Read default sort order,PUT,http://localhost:8080/storage/table/test/query,bad_sort_1.json +Common Read bad sort 2,PUT,http://localhost:8080/storage/table/test/query,bad_sort_2.json +Common Update,PUT,http://localhost:8080/storage/table/test,update.json +Common Read back,GET,http://localhost:8080/storage/table/test?id=2, +Common Update,PUT,http://localhost:8080/storage/table/test,updateKey.json +Common Read back,GET,http://localhost:8080/storage/table/test?key=UPDA, +Common Update no values,PUT,http://localhost:8080/storage/table/test,bad_update.json +Common Read group,PUT,http://localhost:8080/storage/table/test/query,group.json +Bad URL,GET,http://localhost:8080/foglamp/nothing, +Bad table,GET,http://localhost:8080/storage/table/doesntexist, +Bad column,GET,http://localhost:8080/storage/table/test?doesntexist=9, +Ping interface,GET,http://localhost:1081/foglamp/service/ping,,checkstate +Add Readings,POST,http://localhost:8080/storage/reading,asset.json +Fetch Readings,GET,http://localhost:8080/storage/reading?id=1&count=1000,,checkstate +Fetch Readings zero count,GET,http://localhost:8080/storage/reading?id=1&count=0, +Fetch Readings no count,GET,http://localhost:8080/storage/reading?id=1, +Fetch Readings no id,GET,http://localhost:8080/storage/reading?count=1000, +Purge Readings,PUT,http://localhost:8080/storage/reading/purge?age=1000&sent=10&flags=purge, +Common Read sort array,PUT,http://localhost:8080/storage/table/test/query,sort2.json +Common Read multiple aggregates,PUT,http://localhost:8080/storage/table/test/query,where_multi_aggregatee.json, +Common Read columns,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r1.json, +Common Read columns alias,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r2.json, +Common Read columns json,PUT,http://localhost:8080/storage/table/test/query,where_id_1_r3.json, +Date format2,PUT,http://localhost:8080/storage/table/test2/query,where_test2_d2.json +Date format4,PUT,http://localhost:8080/storage/table/test2/query,where_test2_d4.json +Bad format1,PUT,http://localhost:8080/storage/table/test2/query,where_bad_format1.json +Bad format2,PUT,http://localhost:8080/storage/table/test2/query,where_bad_format2.json +Count star,PUT,http://localhost:8080/storage/table/test2/query,where_count_star.json +sum,PUT,http://localhost:8080/storage/table/test2/query,where_sum.json +Add more Readings,POST,http://localhost:8080/storage/reading,readings.json +Query Readings,PUT,http://localhost:8080/storage/reading/query,query_readings.json +Query Readings Timebucket,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket.json +Query Readings Timebucket 1,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket1.json +Multi And,PUT,http://localhost:8080/storage/table/test2/query,multi_and.json +Multi Or,PUT,http://localhost:8080/storage/table/test2/query,multi_or.json +Multi Mixed,PUT,http://localhost:8080/storage/table/test2/query,multi_mised.json +Update Bad Condition,PUT,http://localhost:8080/storage/table/test2,update_bad.json +Read back,GET,http://localhost:8080/storage/table/test2, +Count Assets,PUT,http://localhost:8080/storage/reading/query,count_assets.json +Reading Rate,PUT,http://localhost:8080/storage/reading/query,reading_property.json +Update expression,PUT,http://localhost:8080/storage/table/test2,update_expression.json +Read back update,PUT,http://localhost:8080/storage/table/test2/query,read_id_1xx.json +Distinct,PUT,http://localhost:8080/storage/table/test2/query,where_distinct.json +Update JSON,PUT,http://localhost:8080/storage/table/test,update_json.json +Read back update,PUT,http://localhost:8080/storage/table/test/query,sort.json +Add JSON,POST,http://localhost:8080/storage/table/test,insert2.json +Update Complex JSON,PUT,http://localhost:8080/storage/table/test,update_json2.json +Read back update,GET,http://localhost:8080/storage/table/test?id=4, +Add now,POST,http://localhost:8080/storage/table/test2,addnew.json +Newer,PUT,http://localhost:8080/storage/table/test2/query,newer.json +Older,PUT,http://localhost:8080/storage/table/test2/query,older.json +Newer Bad,PUT,http://localhost:8080/storage/table/test2/query,newerBad.json +Like,PUT,http://localhost:8080/storage/table/test2/query,where_like.json +Group Time,PUT,http://localhost:8080/storage/reading/query,group_time.json +Jira FOGL-690,POST,http://localhost:8080/storage/table/configuration,error-fogl690.json +Jira FOGL-690 cleanup,DELETE,http://localhost:8080/storage/table/configuration,delete.json +Add bad Readings,POST,http://localhost:8080/storage/reading,badreadings.json +Query Readings Timebucket Bad,PUT,http://localhost:8080/storage/reading/query,query_readings_timebucket_bad.json +Reading Rate Array,PUT,http://localhost:8080/storage/reading/query,reading_property_array.json +Common Read limit max_int,PUT,http://localhost:8080/storage/table/test/query,limit_max_int.json +Common Read skip max_int,PUT,http://localhost:8080/storage/table/test/query,skip_max_int.json +Timezone,PUT,http://localhost:8080/storage/table/test2/query,timezone.json +Bad Timezone,PUT,http://localhost:8080/storage/table/test2/query,timezone_bad.json +Set-FOGL-983,PUT,http://localhost:8080/storage/table/configuration,FOGL-983.json +Get-FOGL-983,PUT,http://localhost:8080/storage/table/configuration/query,get-FOGL-983.json +Update now,PUT,http://localhost:8080/storage/table/test2,updatenow.json +Get Reading series group by minutes,PUT,http://localhost:8080/storage/reading/query,series_group_by_minutes.json +Get Reading series (seconds),PUT,http://localhost:8080/storage/reading/query,series_seconds.json +Get Reading series summary (seconds),PUT,http://localhost:8080/storage/reading/query,series_summary_seconds.json +Get Reading series group by hours,PUT,http://localhost:8080/storage/reading/query,series_group_by_hours.json +Add Readings now,POST,http://localhost:8080/storage/reading,add_readings_now.json +Common table IN operator,PUT,http://localhost:8080/storage/table/test2/query,where_in.json +Common table NOT IN operator,PUT,http://localhost:8080/storage/table/test2/query,where_not_in.json +Common table IN operator bad values,PUT,http://localhost:8080/storage/table/test2/query,where_in_bad_values.json +Query Readings IN operator,PUT,http://localhost:8080/storage/reading/query,query_readings_in.json +Query Readings NOT IN operator,PUT,http://localhost:8080/storage/reading/query,query_readings_not_in.json +Query Readings IN operator bad values,PUT,http://localhost:8080/storage/reading/query,query_readings_in_bad_values.json microseconds - Purge Readings,PUT,http://localhost:8080/storage/reading/purge?age=1&sent=0&flags=purge, microseconds - Add Readings,POST,http://localhost:8080/storage/reading,msec_add_readings_user_ts.json microseconds - Query Readings,PUT,http://localhost:8080/storage/reading/query,msec_query_readings.json - +microseconds - Query asset NO alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_noalias.json +microseconds - Query asset alias,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_alias.json +microseconds - Query asset aggregate min,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggmin.json +microseconds - Query asset aggregate min array,PUT,http://localhost:8080/storage/reading/query,msec_query_asset_aggminarray.json +Shutdown,POST,http://localhost:1081/foglamp/service/shutdown,,checkstate From fd1bc11f95141d9918f4929ad631973104516952 Mon Sep 17 00:00:00 2001 From: stefano Date: Tue, 5 Mar 2019 15:28:31 +0100 Subject: [PATCH 66/86] FOGL-2545: UTC expected files --- tests/unit/C/services/storage/sqlite/expected_ETC_UTC/10 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/100 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/101 | 0 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/102 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/103 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/104 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/105 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/106 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/107 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/108 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/11 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/12 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/13 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/14 | 2 ++ tests/unit/C/services/storage/sqlite/expected_ETC_UTC/15 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/16 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/18 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/19 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/20 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/21 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/22 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/23 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/24 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/25 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/26 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/27 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/28 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/29 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/30 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/31 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/32 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/33 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/34 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/35 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/37 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/38 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/39 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/4 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/40 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/41 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/42 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/43 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/44 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/45 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/46 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/47 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/48 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/49 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/5 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/50 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/51 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/52 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/53 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/54 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/55 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/56 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/57 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/58 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/59 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/6 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/60 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/61 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/62 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/63 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/64 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/65 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/66 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/67 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/68 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/69 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/7 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/70 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/71 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/72 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/73 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/74 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/75 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/76 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/77 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/78 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/79 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/8 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/80 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/81 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/82 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/83 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/84 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/85 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/86 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/87 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/88 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/89 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/9 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/90 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/91 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/92 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/93 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/94 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/95 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/96 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/97 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/98 | 1 + tests/unit/C/services/storage/sqlite/expected_ETC_UTC/99 | 1 + 103 files changed, 103 insertions(+) create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/10 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/100 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/101 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/102 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/103 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/104 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/105 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/106 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/107 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/108 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/11 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/12 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/13 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/14 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/15 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/16 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/18 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/19 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/20 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/21 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/22 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/23 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/24 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/25 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/26 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/27 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/28 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/29 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/30 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/31 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/32 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/33 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/34 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/35 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/37 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/38 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/39 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/4 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/40 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/41 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/42 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/43 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/44 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/45 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/46 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/47 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/48 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/49 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/5 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/50 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/51 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/52 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/53 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/54 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/55 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/56 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/57 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/58 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/59 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/6 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/60 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/61 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/62 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/63 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/64 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/65 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/66 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/67 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/68 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/69 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/7 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/70 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/71 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/72 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/73 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/74 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/75 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/76 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/77 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/78 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/79 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/8 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/80 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/81 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/82 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/83 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/84 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/85 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/86 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/87 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/88 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/89 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/9 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/90 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/91 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/92 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/93 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/94 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/95 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/96 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/97 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/98 create mode 100644 tests/unit/C/services/storage/sqlite/expected_ETC_UTC/99 diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/10 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/10 new file mode 100644 index 0000000000..89b4f4207d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/10 @@ -0,0 +1 @@ +{"count":1,"rows":[{"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/100 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/100 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/100 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/101 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/101 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/102 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/102 new file mode 100644 index 0000000000..804a3da17d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/102 @@ -0,0 +1 @@ +{ "removed" : 100, "unsentPurged" : 100, "unsentRetained" : 1, "readings" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/103 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/103 new file mode 100644 index 0000000000..d84da94821 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/103 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/104 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/104 new file mode 100644 index 0000000000..b0e610c297 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/104 @@ -0,0 +1 @@ +{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 10:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 10:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 10:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 10:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 10:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 10:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 09:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 11:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 07:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 12:33:05.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/105 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/105 new file mode 100644 index 0000000000..d5ef46e060 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/105 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/106 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/106 new file mode 100644 index 0000000000..36b92e7724 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/106 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts_alias":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/107 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/107 new file mode 100644 index 0000000000..761333c73d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/107 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/108 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/108 new file mode 100644 index 0000000000..96421c5796 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/108 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 10:03:03.123456","user_ts_max":"2019-03-03 10:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/11 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/11 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/11 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/12 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/12 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/12 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/13 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/13 new file mode 100644 index 0000000000..27530161d2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/13 @@ -0,0 +1 @@ +{ "entryPoint" : "insert", "message" : "table foglamp.test has no column named Nonexistant", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/14 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/14 new file mode 100644 index 0000000000..95390e2964 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/14 @@ -0,0 +1,2 @@ +{ "entryPoint" : "insert", "message" : "Failed to parse JSON payload +", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/15 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/15 new file mode 100644 index 0000000000..29146ee48a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/15 @@ -0,0 +1 @@ +{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/16 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/16 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/16 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/18 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/18 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/18 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/19 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/19 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/19 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/20 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/20 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/20 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/21 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/21 new file mode 100644 index 0000000000..8b2d266eb9 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/21 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"column\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/22 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/22 new file mode 100644 index 0000000000..8584b11223 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/22 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"condition\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/23 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/23 new file mode 100644 index 0000000000..299df5fdf4 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/23 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"value\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/24 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/24 new file mode 100644 index 0000000000..305ed629e9 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/24 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" property must be a JSON object", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/25 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/25 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/25 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/26 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/26 new file mode 100644 index 0000000000..831678c175 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/26 @@ -0,0 +1 @@ +{ "entryPoint" : "Select sort", "message" : "Missing property \"column\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/27 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/27 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/27 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/28 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/28 new file mode 100644 index 0000000000..fd1a64c2c5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/28 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/29 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/29 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/29 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/30 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/30 new file mode 100644 index 0000000000..40f3368d49 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/30 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/31 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/31 new file mode 100644 index 0000000000..81a6486e91 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/31 @@ -0,0 +1 @@ +{ "entryPoint" : "update", "message" : "Missing values or expressions object in payload", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/32 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/32 new file mode 100644 index 0000000000..d985f72c2d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/32 @@ -0,0 +1 @@ +{"count":2,"rows":[{"count_id":1,"key":"UPDA"},{"count_id":1,"key":"TEST1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/33 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/33 new file mode 100644 index 0000000000..e2fdaddfd5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/33 @@ -0,0 +1 @@ +{ "error" : "Unsupported URL: /foglamp/nothing" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/34 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/34 new file mode 100644 index 0000000000..4584abf305 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/34 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "no such table: foglamp.doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/35 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/35 new file mode 100644 index 0000000000..54fc4416b2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/35 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "no such column: doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/37 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/37 new file mode 100644 index 0000000000..303973146f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/37 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 2 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/38 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/38 new file mode 100644 index 0000000000..b2d0997987 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/38 @@ -0,0 +1 @@ +{"count":2,"rows":[{"id":966457,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d08bef40","reading":{"rate":18.4},"user_ts":"2017-09-21 15:00:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"},{"id":966458,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d18bef40","reading":{"rate":45.1},"user_ts":"2017-09-21 15:03:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/39 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/39 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/39 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/4 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/4 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/4 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/40 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/40 new file mode 100644 index 0000000000..61ef4e298c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/40 @@ -0,0 +1 @@ +{ "error" : "Missing query parameter count" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/41 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/41 new file mode 100644 index 0000000000..fb5f172fa7 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/41 @@ -0,0 +1 @@ +{ "error" : "Missing query parameter id" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/42 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/42 new file mode 100644 index 0000000000..da2cfd9351 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/42 @@ -0,0 +1 @@ +{ "removed" : 2, "unsentPurged" : 2, "unsentRetained" : 0, "readings" : 0 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/43 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/43 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/43 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/44 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/44 new file mode 100644 index 0000000000..8017dbc45c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/44 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min_id":1,"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/45 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/45 new file mode 100644 index 0000000000..3cf1b258b8 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/45 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","description":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/46 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/46 new file mode 100644 index 0000000000..bf8e659fa0 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/46 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","MyDescription":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/47 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/47 new file mode 100644 index 0000000000..d44fea2a6f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/47 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","JSONvalue":"test1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/48 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/48 new file mode 100644 index 0000000000..7c1b23b0bf --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/48 @@ -0,0 +1 @@ +{"count":9,"rows":[{"key":"TEST1","description":"A test row","time":"12:14:26"},{"key":"TEST2","description":"A test row","time":"12:14:27"},{"key":"TEST3","description":"A test row","time":"11:14:28"},{"key":"TEST4","description":"A test row","time":"11:14:29"},{"key":"TEST5","description":"A test row","time":"11:15:00"},{"key":"TEST6","description":"A test row","time":"11:15:33"},{"key":"TEST7","description":"A test row","time":"11:16:20"},{"key":"TEST8","description":"A test row","time":"05:14:30"},{"key":"TEST9","description":"A test row","time":"21:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/49 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/49 new file mode 100644 index 0000000000..ee07186753 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/49 @@ -0,0 +1 @@ +{"count":8,"rows":[{"key":"TEST2","description":"A test row","timestamp":"2017-10-10 12:14:27"},{"key":"TEST3","description":"A test row","timestamp":"2017-10-10 11:14:28"},{"key":"TEST4","description":"A test row","timestamp":"2017-10-10 11:14:29"},{"key":"TEST5","description":"A test row","timestamp":"2017-10-10 11:15:00"},{"key":"TEST6","description":"A test row","timestamp":"2017-10-10 11:15:33"},{"key":"TEST7","description":"A test row","timestamp":"2017-10-10 11:16:20"},{"key":"TEST8","description":"A test row","timestamp":"2017-10-10 05:14:30"},{"key":"TEST9","description":"A test row","timestamp":"2017-10-10 21:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/5 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/5 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/5 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/50 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/50 new file mode 100644 index 0000000000..2d3ef169f5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/50 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "return object must have either a column or json property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/51 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/51 new file mode 100644 index 0000000000..24d3191c54 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/51 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "The json property is missing a properties property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/52 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/52 new file mode 100644 index 0000000000..844af318e4 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/52 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Entries":9}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/53 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/53 new file mode 100644 index 0000000000..cfd0e7926b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/53 @@ -0,0 +1 @@ +{"count":1,"rows":[{"sum_id":43}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/54 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/54 new file mode 100644 index 0000000000..21df9c0d48 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/54 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 100 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/55 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/55 new file mode 100644 index 0000000000..c5748d1a33 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/55 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/56 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/56 new file mode 100644 index 0000000000..2e72fb850e --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/56 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/57 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/57 new file mode 100644 index 0000000000..22751a50d2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/57 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/58 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/58 new file mode 100644 index 0000000000..b69be284c3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/58 @@ -0,0 +1 @@ +{"count":6,"rows":[{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/59 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/59 new file mode 100644 index 0000000000..ef2efe563d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/59 @@ -0,0 +1 @@ +{"count":4,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/6 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/6 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/6 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/60 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/60 new file mode 100644 index 0000000000..4488386053 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/60 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/61 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/61 new file mode 100644 index 0000000000..7300a791ac --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/61 @@ -0,0 +1 @@ +{ "entryPoint" : "update", "message" : "No rows where updated", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/62 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/62 new file mode 100644 index 0000000000..4488386053 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/62 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/63 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/63 new file mode 100644 index 0000000000..f62302483f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/63 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Count":100,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/64 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/64 new file mode 100644 index 0000000000..421c29b18e --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/64 @@ -0,0 +1 @@ +{"count":100,"rows":[{"timestamp":"2017-10-11 15:10:51.927","Rate":90},{"timestamp":"2017-10-11 15:10:51.930","Rate":13},{"timestamp":"2017-10-11 15:10:51.933","Rate":84},{"timestamp":"2017-10-11 15:10:51.936","Rate":96},{"timestamp":"2017-10-11 15:10:51.939","Rate":2},{"timestamp":"2017-10-11 15:10:51.942","Rate":54},{"timestamp":"2017-10-11 15:10:51.946","Rate":28},{"timestamp":"2017-10-11 15:10:51.949","Rate":3},{"timestamp":"2017-10-11 15:10:51.952","Rate":77},{"timestamp":"2017-10-11 15:10:51.955","Rate":38},{"timestamp":"2017-10-11 15:10:51.959","Rate":26},{"timestamp":"2017-10-11 15:10:51.963","Rate":86},{"timestamp":"2017-10-11 15:10:51.966","Rate":39},{"timestamp":"2017-10-11 15:10:51.970","Rate":57},{"timestamp":"2017-10-11 15:10:51.973","Rate":73},{"timestamp":"2017-10-11 15:10:51.979","Rate":22},{"timestamp":"2017-10-11 15:10:51.982","Rate":34},{"timestamp":"2017-10-11 15:10:51.986","Rate":78},{"timestamp":"2017-10-11 15:10:51.990","Rate":20},{"timestamp":"2017-10-11 15:10:51.993","Rate":70},{"timestamp":"2017-10-11 15:10:51.996","Rate":17},{"timestamp":"2017-10-11 15:10:52.000","Rate":2},{"timestamp":"2017-10-11 15:10:52.005","Rate":18},{"timestamp":"2017-10-11 15:10:52.009","Rate":52},{"timestamp":"2017-10-11 15:10:52.012","Rate":62},{"timestamp":"2017-10-11 15:10:52.015","Rate":47},{"timestamp":"2017-10-11 15:10:52.019","Rate":73},{"timestamp":"2017-10-11 15:10:52.022","Rate":9},{"timestamp":"2017-10-11 15:10:52.026","Rate":66},{"timestamp":"2017-10-11 15:10:52.029","Rate":30},{"timestamp":"2017-10-11 15:10:52.031","Rate":70},{"timestamp":"2017-10-11 15:10:52.034","Rate":41},{"timestamp":"2017-10-11 15:10:52.037","Rate":2},{"timestamp":"2017-10-11 15:10:52.040","Rate":69},{"timestamp":"2017-10-11 15:10:52.043","Rate":98},{"timestamp":"2017-10-11 15:10:52.046","Rate":13},{"timestamp":"2017-10-11 15:10:52.050","Rate":91},{"timestamp":"2017-10-11 15:10:52.053","Rate":18},{"timestamp":"2017-10-11 15:10:52.056","Rate":78},{"timestamp":"2017-10-11 15:10:52.059","Rate":70},{"timestamp":"2017-10-11 15:10:52.062","Rate":48},{"timestamp":"2017-10-11 15:10:52.066","Rate":94},{"timestamp":"2017-10-11 15:10:52.070","Rate":79},{"timestamp":"2017-10-11 15:10:52.073","Rate":87},{"timestamp":"2017-10-11 15:10:52.075","Rate":60},{"timestamp":"2017-10-11 15:10:52.078","Rate":48},{"timestamp":"2017-10-11 15:10:52.081","Rate":88},{"timestamp":"2017-10-11 15:10:52.084","Rate":3},{"timestamp":"2017-10-11 15:10:52.086","Rate":93},{"timestamp":"2017-10-11 15:10:52.089","Rate":83},{"timestamp":"2017-10-11 15:10:52.092","Rate":76},{"timestamp":"2017-10-11 15:10:52.095","Rate":97},{"timestamp":"2017-10-11 15:10:52.098","Rate":31},{"timestamp":"2017-10-11 15:10:52.100","Rate":49},{"timestamp":"2017-10-11 15:10:52.103","Rate":36},{"timestamp":"2017-10-11 15:10:52.106","Rate":15},{"timestamp":"2017-10-11 15:10:52.109","Rate":67},{"timestamp":"2017-10-11 15:10:52.111","Rate":67},{"timestamp":"2017-10-11 15:10:52.114","Rate":94},{"timestamp":"2017-10-11 15:10:52.116","Rate":68},{"timestamp":"2017-10-11 15:10:52.119","Rate":22},{"timestamp":"2017-10-11 15:10:52.122","Rate":54},{"timestamp":"2017-10-11 15:10:52.124","Rate":94},{"timestamp":"2017-10-11 15:10:52.127","Rate":49},{"timestamp":"2017-10-11 15:10:52.130","Rate":59},{"timestamp":"2017-10-11 15:10:52.132","Rate":6},{"timestamp":"2017-10-11 15:10:52.135","Rate":82},{"timestamp":"2017-10-11 15:10:52.137","Rate":5},{"timestamp":"2017-10-11 15:10:52.140","Rate":1},{"timestamp":"2017-10-11 15:10:52.142","Rate":53},{"timestamp":"2017-10-11 15:10:52.145","Rate":69},{"timestamp":"2017-10-11 15:10:52.147","Rate":97},{"timestamp":"2017-10-11 15:10:52.150","Rate":58},{"timestamp":"2017-10-11 15:10:52.153","Rate":76},{"timestamp":"2017-10-11 15:10:52.157","Rate":81},{"timestamp":"2017-10-11 15:10:52.160","Rate":30},{"timestamp":"2017-10-11 15:10:52.163","Rate":4},{"timestamp":"2017-10-11 15:10:52.165","Rate":67},{"timestamp":"2017-10-11 15:10:52.169","Rate":5},{"timestamp":"2017-10-11 15:10:52.171","Rate":72},{"timestamp":"2017-10-11 15:10:52.174","Rate":20},{"timestamp":"2017-10-11 15:10:52.176","Rate":58},{"timestamp":"2017-10-11 15:10:52.179","Rate":75},{"timestamp":"2017-10-11 15:10:52.182","Rate":74},{"timestamp":"2017-10-11 15:10:52.184","Rate":60},{"timestamp":"2017-10-11 15:10:52.187","Rate":96},{"timestamp":"2017-10-11 15:10:52.189","Rate":30},{"timestamp":"2017-10-11 15:10:52.192","Rate":40},{"timestamp":"2017-10-11 15:10:52.195","Rate":33},{"timestamp":"2017-10-11 15:10:52.197","Rate":87},{"timestamp":"2017-10-11 15:10:52.200","Rate":67},{"timestamp":"2017-10-11 15:10:52.203","Rate":40},{"timestamp":"2017-10-11 15:10:52.206","Rate":44},{"timestamp":"2017-10-11 15:10:52.208","Rate":7},{"timestamp":"2017-10-11 15:10:52.211","Rate":52},{"timestamp":"2017-10-11 15:10:52.214","Rate":93},{"timestamp":"2017-10-11 15:10:52.219","Rate":43},{"timestamp":"2017-10-11 15:10:52.222","Rate":66},{"timestamp":"2017-10-11 15:10:52.225","Rate":8},{"timestamp":"2017-10-11 15:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/65 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/65 new file mode 100644 index 0000000000..a9bebbfede --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/65 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 4 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/66 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/66 new file mode 100644 index 0000000000..6bcad7bc37 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/66 @@ -0,0 +1 @@ +{"count":4,"rows":[{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/67 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/67 new file mode 100644 index 0000000000..c3fd30822b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/67 @@ -0,0 +1 @@ +{"count":2,"rows":[{"description":"A test row"},{"description":"Updated with expression"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/68 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/68 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/68 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/69 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/69 new file mode 100644 index 0000000000..a5d4634456 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/69 @@ -0,0 +1 @@ +{"count":2,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}},{"id":1,"key":"TEST1","description":"A test row","data":{"json":"new value"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/7 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/7 new file mode 100644 index 0000000000..379611dc02 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/7 @@ -0,0 +1 @@ +{"count":1,"rows":[{"count_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/70 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/70 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/70 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/71 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/71 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/71 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/72 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/72 new file mode 100644 index 0000000000..858823dc3c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/72 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":4,"key":"Admin","description":"URL of the admin API","data":{"url":{"value":"new value"}}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/73 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/73 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/73 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/74 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/74 new file mode 100644 index 0000000000..4bde9235b1 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/74 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Count":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/75 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/75 new file mode 100644 index 0000000000..973df944e3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/75 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/76 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/76 new file mode 100644 index 0000000000..7e9c80429f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/76 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"value\" of an \"newer\" condition must be an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/77 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/77 new file mode 100644 index 0000000000..88ff5140f2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/77 @@ -0,0 +1 @@ +{"count":5,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/78 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/78 new file mode 100644 index 0000000000..f22a289656 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/78 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":2,"max":96,"average":47.9523809523809,"user_ts":"2017-10-11 15:10:51"},{"min":1,"max":98,"average":53.7721518987342,"user_ts":"2017-10-11 15:10:52"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/79 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/79 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/79 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/8 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/8 new file mode 100644 index 0000000000..cd34346b4c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/8 @@ -0,0 +1 @@ +{"count":1,"rows":[{"avg_id":1.0}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/80 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/80 new file mode 100644 index 0000000000..29146ee48a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/80 @@ -0,0 +1 @@ +{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/81 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/81 new file mode 100644 index 0000000000..c614b3d6e0 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/81 @@ -0,0 +1 @@ +{ "entryPoint" : "appendReadings", "message" : "Payload is missing a readings array", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/82 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/82 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/82 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/83 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/83 new file mode 100644 index 0000000000..421c29b18e --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/83 @@ -0,0 +1 @@ +{"count":100,"rows":[{"timestamp":"2017-10-11 15:10:51.927","Rate":90},{"timestamp":"2017-10-11 15:10:51.930","Rate":13},{"timestamp":"2017-10-11 15:10:51.933","Rate":84},{"timestamp":"2017-10-11 15:10:51.936","Rate":96},{"timestamp":"2017-10-11 15:10:51.939","Rate":2},{"timestamp":"2017-10-11 15:10:51.942","Rate":54},{"timestamp":"2017-10-11 15:10:51.946","Rate":28},{"timestamp":"2017-10-11 15:10:51.949","Rate":3},{"timestamp":"2017-10-11 15:10:51.952","Rate":77},{"timestamp":"2017-10-11 15:10:51.955","Rate":38},{"timestamp":"2017-10-11 15:10:51.959","Rate":26},{"timestamp":"2017-10-11 15:10:51.963","Rate":86},{"timestamp":"2017-10-11 15:10:51.966","Rate":39},{"timestamp":"2017-10-11 15:10:51.970","Rate":57},{"timestamp":"2017-10-11 15:10:51.973","Rate":73},{"timestamp":"2017-10-11 15:10:51.979","Rate":22},{"timestamp":"2017-10-11 15:10:51.982","Rate":34},{"timestamp":"2017-10-11 15:10:51.986","Rate":78},{"timestamp":"2017-10-11 15:10:51.990","Rate":20},{"timestamp":"2017-10-11 15:10:51.993","Rate":70},{"timestamp":"2017-10-11 15:10:51.996","Rate":17},{"timestamp":"2017-10-11 15:10:52.000","Rate":2},{"timestamp":"2017-10-11 15:10:52.005","Rate":18},{"timestamp":"2017-10-11 15:10:52.009","Rate":52},{"timestamp":"2017-10-11 15:10:52.012","Rate":62},{"timestamp":"2017-10-11 15:10:52.015","Rate":47},{"timestamp":"2017-10-11 15:10:52.019","Rate":73},{"timestamp":"2017-10-11 15:10:52.022","Rate":9},{"timestamp":"2017-10-11 15:10:52.026","Rate":66},{"timestamp":"2017-10-11 15:10:52.029","Rate":30},{"timestamp":"2017-10-11 15:10:52.031","Rate":70},{"timestamp":"2017-10-11 15:10:52.034","Rate":41},{"timestamp":"2017-10-11 15:10:52.037","Rate":2},{"timestamp":"2017-10-11 15:10:52.040","Rate":69},{"timestamp":"2017-10-11 15:10:52.043","Rate":98},{"timestamp":"2017-10-11 15:10:52.046","Rate":13},{"timestamp":"2017-10-11 15:10:52.050","Rate":91},{"timestamp":"2017-10-11 15:10:52.053","Rate":18},{"timestamp":"2017-10-11 15:10:52.056","Rate":78},{"timestamp":"2017-10-11 15:10:52.059","Rate":70},{"timestamp":"2017-10-11 15:10:52.062","Rate":48},{"timestamp":"2017-10-11 15:10:52.066","Rate":94},{"timestamp":"2017-10-11 15:10:52.070","Rate":79},{"timestamp":"2017-10-11 15:10:52.073","Rate":87},{"timestamp":"2017-10-11 15:10:52.075","Rate":60},{"timestamp":"2017-10-11 15:10:52.078","Rate":48},{"timestamp":"2017-10-11 15:10:52.081","Rate":88},{"timestamp":"2017-10-11 15:10:52.084","Rate":3},{"timestamp":"2017-10-11 15:10:52.086","Rate":93},{"timestamp":"2017-10-11 15:10:52.089","Rate":83},{"timestamp":"2017-10-11 15:10:52.092","Rate":76},{"timestamp":"2017-10-11 15:10:52.095","Rate":97},{"timestamp":"2017-10-11 15:10:52.098","Rate":31},{"timestamp":"2017-10-11 15:10:52.100","Rate":49},{"timestamp":"2017-10-11 15:10:52.103","Rate":36},{"timestamp":"2017-10-11 15:10:52.106","Rate":15},{"timestamp":"2017-10-11 15:10:52.109","Rate":67},{"timestamp":"2017-10-11 15:10:52.111","Rate":67},{"timestamp":"2017-10-11 15:10:52.114","Rate":94},{"timestamp":"2017-10-11 15:10:52.116","Rate":68},{"timestamp":"2017-10-11 15:10:52.119","Rate":22},{"timestamp":"2017-10-11 15:10:52.122","Rate":54},{"timestamp":"2017-10-11 15:10:52.124","Rate":94},{"timestamp":"2017-10-11 15:10:52.127","Rate":49},{"timestamp":"2017-10-11 15:10:52.130","Rate":59},{"timestamp":"2017-10-11 15:10:52.132","Rate":6},{"timestamp":"2017-10-11 15:10:52.135","Rate":82},{"timestamp":"2017-10-11 15:10:52.137","Rate":5},{"timestamp":"2017-10-11 15:10:52.140","Rate":1},{"timestamp":"2017-10-11 15:10:52.142","Rate":53},{"timestamp":"2017-10-11 15:10:52.145","Rate":69},{"timestamp":"2017-10-11 15:10:52.147","Rate":97},{"timestamp":"2017-10-11 15:10:52.150","Rate":58},{"timestamp":"2017-10-11 15:10:52.153","Rate":76},{"timestamp":"2017-10-11 15:10:52.157","Rate":81},{"timestamp":"2017-10-11 15:10:52.160","Rate":30},{"timestamp":"2017-10-11 15:10:52.163","Rate":4},{"timestamp":"2017-10-11 15:10:52.165","Rate":67},{"timestamp":"2017-10-11 15:10:52.169","Rate":5},{"timestamp":"2017-10-11 15:10:52.171","Rate":72},{"timestamp":"2017-10-11 15:10:52.174","Rate":20},{"timestamp":"2017-10-11 15:10:52.176","Rate":58},{"timestamp":"2017-10-11 15:10:52.179","Rate":75},{"timestamp":"2017-10-11 15:10:52.182","Rate":74},{"timestamp":"2017-10-11 15:10:52.184","Rate":60},{"timestamp":"2017-10-11 15:10:52.187","Rate":96},{"timestamp":"2017-10-11 15:10:52.189","Rate":30},{"timestamp":"2017-10-11 15:10:52.192","Rate":40},{"timestamp":"2017-10-11 15:10:52.195","Rate":33},{"timestamp":"2017-10-11 15:10:52.197","Rate":87},{"timestamp":"2017-10-11 15:10:52.200","Rate":67},{"timestamp":"2017-10-11 15:10:52.203","Rate":40},{"timestamp":"2017-10-11 15:10:52.206","Rate":44},{"timestamp":"2017-10-11 15:10:52.208","Rate":7},{"timestamp":"2017-10-11 15:10:52.211","Rate":52},{"timestamp":"2017-10-11 15:10:52.214","Rate":93},{"timestamp":"2017-10-11 15:10:52.219","Rate":43},{"timestamp":"2017-10-11 15:10:52.222","Rate":66},{"timestamp":"2017-10-11 15:10:52.225","Rate":8},{"timestamp":"2017-10-11 15:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/84 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/84 new file mode 100644 index 0000000000..6e3d42ac03 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/84 @@ -0,0 +1 @@ +{ "entryPoint" : "limit", "message" : "Limit must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/85 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/85 new file mode 100644 index 0000000000..8d705bbb14 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/85 @@ -0,0 +1 @@ +{ "entryPoint" : "skip", "message" : "Skip must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/86 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/86 new file mode 100644 index 0000000000..4522ec1b85 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/86 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "unrecognized token: \":\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/87 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/87 new file mode 100644 index 0000000000..1d6fe91f5d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/87 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "SQLite3 plugin does not support timezones in qeueries", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/88 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/88 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/88 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/89 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/89 new file mode 100644 index 0000000000..93af3344da --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/89 @@ -0,0 +1 @@ +{"count":1,"rows":[{"description":"added'some'ch'''ars'"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/9 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/9 new file mode 100644 index 0000000000..f951388d78 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/9 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/90 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/90 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/90 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/91 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/91 new file mode 100644 index 0000000000..f70a6b7ed5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/91 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 15:10"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/92 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/92 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/92 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/93 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/93 new file mode 100644 index 0000000000..a4bfc7b379 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/93 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":"","max":"","average":""}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/94 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/94 new file mode 100644 index 0000000000..26bdeeb2ac --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/94 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 15"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/95 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/95 new file mode 100644 index 0000000000..2eeb09a60b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/95 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/96 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/96 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/96 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/97 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/97 new file mode 100644 index 0000000000..22f00fc70c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/97 @@ -0,0 +1 @@ +{"count":1,"rows":[{"count_id":10}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/98 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/98 new file mode 100644 index 0000000000..69ab957da3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/98 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"value\" of a \"in\" condition must be an array and must not be empty.", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/99 b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/99 new file mode 100644 index 0000000000..cfddbd8a1a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_ETC_UTC/99 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.9207920792079,"asset_code":"MyAsset"}]} \ No newline at end of file From 7e9026e49ad39cf9230b1152c84aaceb41a3059c Mon Sep 17 00:00:00 2001 From: stefano Date: Tue, 5 Mar 2019 15:46:15 +0100 Subject: [PATCH 67/86] FOGL-2545: fixed bug --- C/plugins/storage/sqlite/connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/C/plugins/storage/sqlite/connection.cpp b/C/plugins/storage/sqlite/connection.cpp index eec93384dc..b02cf0503f 100644 --- a/C/plugins/storage/sqlite/connection.cpp +++ b/C/plugins/storage/sqlite/connection.cpp @@ -2122,7 +2122,7 @@ bool isAggregate = false; asset_code, read_key, reading, - strftime()" F_DATEH24_SEC R"(', user_ts, 'localtime') || + strftime(')" F_DATEH24_SEC R"(', user_ts, 'localtime') || substr(user_ts, instr(user_ts, '.'), 7) AS user_ts, strftime(')" F_DATEH24_MS R"(', ts, 'localtime') AS ts FROM foglamp.)"; From 938d1de6215eec3c193a0882cb004b3a6a53c6e5 Mon Sep 17 00:00:00 2001 From: stefano Date: Wed, 6 Mar 2019 09:46:49 +0100 Subject: [PATCH 68/86] FOGL-2545: fixed expected files for expected_EUROPE_ROME --- tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 | 2 +- tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/10 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/100 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/101 | 0 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/102 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/103 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/104 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/105 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/106 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/107 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/108 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/11 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/12 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/13 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/14 | 2 ++ tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/15 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/16 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/18 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/19 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 | 2 +- tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/20 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/21 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/22 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/23 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/24 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/25 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/26 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/27 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/28 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/29 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 | 2 +- tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/30 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/31 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/32 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/33 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/34 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/35 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/37 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/38 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/39 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/4 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/40 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/41 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/42 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/43 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/44 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/45 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/46 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/47 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/48 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/49 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/5 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/50 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/51 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/52 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/53 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/54 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/55 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/56 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/57 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/58 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/59 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/6 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/60 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/61 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/62 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/63 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/64 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/65 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/66 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/67 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/68 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/69 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/7 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/70 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/71 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/72 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/73 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/74 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/75 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/76 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/77 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/78 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/79 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/8 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/80 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/81 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/82 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/83 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/84 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/85 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/86 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/87 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/88 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/89 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/9 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/90 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/91 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/92 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/93 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/94 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/95 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/96 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/97 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/98 | 1 + tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/99 | 1 + 106 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/10 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/100 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/101 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/102 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/103 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/104 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/105 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/106 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/107 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/108 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/11 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/12 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/13 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/14 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/15 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/16 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/18 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/19 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/20 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/21 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/22 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/23 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/24 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/25 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/26 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/27 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/28 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/29 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/30 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/31 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/32 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/33 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/34 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/35 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/37 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/38 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/39 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/4 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/40 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/41 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/42 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/43 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/44 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/45 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/46 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/47 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/48 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/49 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/5 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/50 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/51 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/52 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/53 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/54 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/55 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/56 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/57 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/58 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/59 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/6 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/60 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/61 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/62 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/63 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/64 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/65 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/66 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/67 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/68 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/69 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/7 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/70 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/71 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/72 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/73 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/74 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/75 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/76 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/77 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/78 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/79 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/8 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/80 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/81 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/82 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/83 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/84 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/85 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/86 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/87 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/88 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/89 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/9 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/90 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/91 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/92 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/93 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/94 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/95 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/96 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/97 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/98 create mode 100644 tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/99 diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 index 804a3da17d..960917c42d 100644 --- a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/1 @@ -1 +1 @@ -{ "removed" : 100, "unsentPurged" : 100, "unsentRetained" : 1, "readings" : 1 } \ No newline at end of file +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/10 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/10 new file mode 100644 index 0000000000..89b4f4207d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/10 @@ -0,0 +1 @@ +{"count":1,"rows":[{"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/100 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/100 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/100 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/101 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/101 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/102 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/102 new file mode 100644 index 0000000000..804a3da17d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/102 @@ -0,0 +1 @@ +{ "removed" : 100, "unsentPurged" : 100, "unsentRetained" : 1, "readings" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/103 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/103 new file mode 100644 index 0000000000..d84da94821 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/103 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/104 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/104 new file mode 100644 index 0000000000..cc1e058c38 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/104 @@ -0,0 +1 @@ +{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 11:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 11:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 11:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 11:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 11:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 11:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 11:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 10:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 12:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 08:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 13:33:05.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/105 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/105 new file mode 100644 index 0000000000..6e1d09a064 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/105 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts":"2019-03-03 11:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/106 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/106 new file mode 100644 index 0000000000..19ea46d15f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/106 @@ -0,0 +1 @@ +{"count":1,"rows":[{"reading":{"value":9},"user_ts_alias":"2019-03-03 11:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/107 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/107 new file mode 100644 index 0000000000..d570db1c2a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/107 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 11:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/108 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/108 new file mode 100644 index 0000000000..5a1706da6e --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/108 @@ -0,0 +1 @@ +{"count":1,"rows":[{"user_ts_min":"2019-03-03 11:03:03.123456","user_ts_max":"2019-03-03 11:03:03.123456"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/11 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/11 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/11 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/12 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/12 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/12 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/13 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/13 new file mode 100644 index 0000000000..27530161d2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/13 @@ -0,0 +1 @@ +{ "entryPoint" : "insert", "message" : "table foglamp.test has no column named Nonexistant", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/14 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/14 new file mode 100644 index 0000000000..95390e2964 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/14 @@ -0,0 +1,2 @@ +{ "entryPoint" : "insert", "message" : "Failed to parse JSON payload +", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/15 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/15 new file mode 100644 index 0000000000..29146ee48a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/15 @@ -0,0 +1 @@ +{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/16 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/16 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/16 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/18 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/18 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/18 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/19 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/19 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/19 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 index d84da94821..960917c42d 100644 --- a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/2 @@ -1 +1 @@ -{ "response" : "appended", "readings_added" : 11 } \ No newline at end of file +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/20 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/20 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/20 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/21 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/21 new file mode 100644 index 0000000000..8b2d266eb9 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/21 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"column\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/22 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/22 new file mode 100644 index 0000000000..8584b11223 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/22 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"condition\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/23 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/23 new file mode 100644 index 0000000000..299df5fdf4 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/23 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" object is missing a \"value\" property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/24 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/24 new file mode 100644 index 0000000000..305ed629e9 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/24 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"where\" property must be a JSON object", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/25 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/25 new file mode 100644 index 0000000000..1ebeaeaef3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/25 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"An inserted row","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/26 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/26 new file mode 100644 index 0000000000..831678c175 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/26 @@ -0,0 +1 @@ +{ "entryPoint" : "Select sort", "message" : "Missing property \"column\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/27 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/27 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/27 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/28 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/28 new file mode 100644 index 0000000000..fd1a64c2c5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/28 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"TEST2","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/29 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/29 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/29 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 index cc1e058c38..124e6704de 100644 --- a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/3 @@ -1 +1 @@ -{"count":11,"rows":[{"asset_code":"msec_003_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000003","reading":{"value":3},"user_ts":"2019-01-01 11:01:01.000000"},{"asset_code":"msec_004_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000004","reading":{"value":4},"user_ts":"2019-01-02 11:02:01.000000"},{"asset_code":"msec_005_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000005","reading":{"value":5},"user_ts":"2019-01-03 11:02:02.841000"},{"asset_code":"msec_006_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000006","reading":{"value":6},"user_ts":"2019-01-04 11:03:05.123456"},{"asset_code":"msec_007_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000007","reading":{"value":7},"user_ts":"2019-01-04 11:03:05.100000"},{"asset_code":"msec_008_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000008","reading":{"value":8},"user_ts":"2019-01-04 11:03:05.123000"},{"asset_code":"msec_009_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000009","reading":{"value":9},"user_ts":"2019-03-03 11:03:03.123456"},{"asset_code":"msec_010_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000010","reading":{"value":10},"user_ts":"2019-03-04 10:03:04.123456"},{"asset_code":"msec_011_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000011","reading":{"value":11},"user_ts":"2019-03-05 12:03:05.123456"},{"asset_code":"msec_012_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000012","reading":{"value":12},"user_ts":"2019-03-04 08:33:04.123456"},{"asset_code":"msec_013_OK","read_key":"f1cfff7a-3769-4f47-9ded-000000000013","reading":{"value":13},"user_ts":"2019-03-05 13:33:05.123456"}]} \ No newline at end of file +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/30 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/30 new file mode 100644 index 0000000000..40f3368d49 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/30 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/31 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/31 new file mode 100644 index 0000000000..81a6486e91 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/31 @@ -0,0 +1 @@ +{ "entryPoint" : "update", "message" : "Missing values or expressions object in payload", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/32 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/32 new file mode 100644 index 0000000000..d985f72c2d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/32 @@ -0,0 +1 @@ +{"count":2,"rows":[{"count_id":1,"key":"UPDA"},{"count_id":1,"key":"TEST1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/33 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/33 new file mode 100644 index 0000000000..e2fdaddfd5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/33 @@ -0,0 +1 @@ +{ "error" : "Unsupported URL: /foglamp/nothing" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/34 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/34 new file mode 100644 index 0000000000..4584abf305 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/34 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "no such table: foglamp.doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/35 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/35 new file mode 100644 index 0000000000..54fc4416b2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/35 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "no such column: doesntexist", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/37 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/37 new file mode 100644 index 0000000000..303973146f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/37 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 2 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/38 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/38 new file mode 100644 index 0000000000..b2d0997987 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/38 @@ -0,0 +1 @@ +{"count":2,"rows":[{"id":966457,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d08bef40","reading":{"rate":18.4},"user_ts":"2017-09-21 15:00:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"},{"id":966458,"asset_code":"MyAsset","read_key":"5b3be500-ff95-41ae-b5a4-cc99d18bef40","reading":{"rate":45.1},"user_ts":"2017-09-21 15:03:09.025655+01","ts":"2017-10-04 11:38:39.368881+01"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/39 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/39 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/39 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/4 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/4 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/4 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/40 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/40 new file mode 100644 index 0000000000..61ef4e298c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/40 @@ -0,0 +1 @@ +{ "error" : "Missing query parameter count" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/41 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/41 new file mode 100644 index 0000000000..fb5f172fa7 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/41 @@ -0,0 +1 @@ +{ "error" : "Missing query parameter id" } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/42 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/42 new file mode 100644 index 0000000000..da2cfd9351 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/42 @@ -0,0 +1 @@ +{ "removed" : 2, "unsentPurged" : 2, "unsentRetained" : 0, "readings" : 0 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/43 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/43 new file mode 100644 index 0000000000..960917c42d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/43 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"json":"test1"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/44 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/44 new file mode 100644 index 0000000000..8017dbc45c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/44 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min_id":1,"max_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/45 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/45 new file mode 100644 index 0000000000..3cf1b258b8 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/45 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","description":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/46 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/46 new file mode 100644 index 0000000000..bf8e659fa0 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/46 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","MyDescription":"A test row"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/47 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/47 new file mode 100644 index 0000000000..d44fea2a6f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/47 @@ -0,0 +1 @@ +{"count":1,"rows":[{"key":"TEST1","JSONvalue":"test1"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/48 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/48 new file mode 100644 index 0000000000..90b3a11539 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/48 @@ -0,0 +1 @@ +{"count":9,"rows":[{"key":"TEST1","description":"A test row","time":"14:14:26"},{"key":"TEST2","description":"A test row","time":"14:14:27"},{"key":"TEST3","description":"A test row","time":"13:14:28"},{"key":"TEST4","description":"A test row","time":"13:14:29"},{"key":"TEST5","description":"A test row","time":"13:15:00"},{"key":"TEST6","description":"A test row","time":"13:15:33"},{"key":"TEST7","description":"A test row","time":"13:16:20"},{"key":"TEST8","description":"A test row","time":"07:14:30"},{"key":"TEST9","description":"A test row","time":"23:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/49 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/49 new file mode 100644 index 0000000000..fe9af5ecf6 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/49 @@ -0,0 +1 @@ +{"count":8,"rows":[{"key":"TEST2","description":"A test row","timestamp":"2017-10-10 14:14:27"},{"key":"TEST3","description":"A test row","timestamp":"2017-10-10 13:14:28"},{"key":"TEST4","description":"A test row","timestamp":"2017-10-10 13:14:29"},{"key":"TEST5","description":"A test row","timestamp":"2017-10-10 13:15:00"},{"key":"TEST6","description":"A test row","timestamp":"2017-10-10 13:15:33"},{"key":"TEST7","description":"A test row","timestamp":"2017-10-10 13:16:20"},{"key":"TEST8","description":"A test row","timestamp":"2017-10-10 07:14:30"},{"key":"TEST9","description":"A test row","timestamp":"2017-10-10 23:14:30"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/5 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/5 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/5 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/50 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/50 new file mode 100644 index 0000000000..2d3ef169f5 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/50 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "return object must have either a column or json property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/51 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/51 new file mode 100644 index 0000000000..24d3191c54 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/51 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "The json property is missing a properties property", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/52 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/52 new file mode 100644 index 0000000000..844af318e4 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/52 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Entries":9}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/53 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/53 new file mode 100644 index 0000000000..cfd0e7926b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/53 @@ -0,0 +1 @@ +{"count":1,"rows":[{"sum_id":43}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/54 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/54 new file mode 100644 index 0000000000..21df9c0d48 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/54 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 100 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/55 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/55 new file mode 100644 index 0000000000..c5748d1a33 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/55 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/56 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/56 new file mode 100644 index 0000000000..2e72fb850e --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/56 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","timestamp":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/57 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/57 new file mode 100644 index 0000000000..22751a50d2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/57 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":1,"max":98,"average":53.7721518987342,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"},{"min":2,"max":96,"average":47.9523809523809,"asset_code":"MyAsset","bucket":"2017-10-11 15:10:51"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/58 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/58 new file mode 100644 index 0000000000..b69be284c3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/58 @@ -0,0 +1 @@ +{"count":6,"rows":[{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/59 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/59 new file mode 100644 index 0000000000..ef2efe563d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/59 @@ -0,0 +1 @@ +{"count":4,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/6 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/6 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/6 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/60 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/60 new file mode 100644 index 0000000000..4488386053 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/60 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/61 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/61 new file mode 100644 index 0000000000..7300a791ac --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/61 @@ -0,0 +1 @@ +{ "entryPoint" : "update", "message" : "No rows where updated", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/62 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/62 new file mode 100644 index 0000000000..4488386053 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/62 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":6,"key":"TEST6","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":6,"key":"TEST7","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":8,"key":"TEST8","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":9,"key":"TEST9","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/63 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/63 new file mode 100644 index 0000000000..f62302483f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/63 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Count":100,"asset_code":"MyAsset"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/64 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/64 new file mode 100644 index 0000000000..4401963f82 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/64 @@ -0,0 +1 @@ +{"count":100,"rows":[{"timestamp":"2017-10-11 17:10:51.927","Rate":90},{"timestamp":"2017-10-11 17:10:51.930","Rate":13},{"timestamp":"2017-10-11 17:10:51.933","Rate":84},{"timestamp":"2017-10-11 17:10:51.936","Rate":96},{"timestamp":"2017-10-11 17:10:51.939","Rate":2},{"timestamp":"2017-10-11 17:10:51.942","Rate":54},{"timestamp":"2017-10-11 17:10:51.946","Rate":28},{"timestamp":"2017-10-11 17:10:51.949","Rate":3},{"timestamp":"2017-10-11 17:10:51.952","Rate":77},{"timestamp":"2017-10-11 17:10:51.955","Rate":38},{"timestamp":"2017-10-11 17:10:51.959","Rate":26},{"timestamp":"2017-10-11 17:10:51.963","Rate":86},{"timestamp":"2017-10-11 17:10:51.966","Rate":39},{"timestamp":"2017-10-11 17:10:51.970","Rate":57},{"timestamp":"2017-10-11 17:10:51.973","Rate":73},{"timestamp":"2017-10-11 17:10:51.979","Rate":22},{"timestamp":"2017-10-11 17:10:51.982","Rate":34},{"timestamp":"2017-10-11 17:10:51.986","Rate":78},{"timestamp":"2017-10-11 17:10:51.990","Rate":20},{"timestamp":"2017-10-11 17:10:51.993","Rate":70},{"timestamp":"2017-10-11 17:10:51.996","Rate":17},{"timestamp":"2017-10-11 17:10:52.000","Rate":2},{"timestamp":"2017-10-11 17:10:52.005","Rate":18},{"timestamp":"2017-10-11 17:10:52.009","Rate":52},{"timestamp":"2017-10-11 17:10:52.012","Rate":62},{"timestamp":"2017-10-11 17:10:52.015","Rate":47},{"timestamp":"2017-10-11 17:10:52.019","Rate":73},{"timestamp":"2017-10-11 17:10:52.022","Rate":9},{"timestamp":"2017-10-11 17:10:52.026","Rate":66},{"timestamp":"2017-10-11 17:10:52.029","Rate":30},{"timestamp":"2017-10-11 17:10:52.031","Rate":70},{"timestamp":"2017-10-11 17:10:52.034","Rate":41},{"timestamp":"2017-10-11 17:10:52.037","Rate":2},{"timestamp":"2017-10-11 17:10:52.040","Rate":69},{"timestamp":"2017-10-11 17:10:52.043","Rate":98},{"timestamp":"2017-10-11 17:10:52.046","Rate":13},{"timestamp":"2017-10-11 17:10:52.050","Rate":91},{"timestamp":"2017-10-11 17:10:52.053","Rate":18},{"timestamp":"2017-10-11 17:10:52.056","Rate":78},{"timestamp":"2017-10-11 17:10:52.059","Rate":70},{"timestamp":"2017-10-11 17:10:52.062","Rate":48},{"timestamp":"2017-10-11 17:10:52.066","Rate":94},{"timestamp":"2017-10-11 17:10:52.070","Rate":79},{"timestamp":"2017-10-11 17:10:52.073","Rate":87},{"timestamp":"2017-10-11 17:10:52.075","Rate":60},{"timestamp":"2017-10-11 17:10:52.078","Rate":48},{"timestamp":"2017-10-11 17:10:52.081","Rate":88},{"timestamp":"2017-10-11 17:10:52.084","Rate":3},{"timestamp":"2017-10-11 17:10:52.086","Rate":93},{"timestamp":"2017-10-11 17:10:52.089","Rate":83},{"timestamp":"2017-10-11 17:10:52.092","Rate":76},{"timestamp":"2017-10-11 17:10:52.095","Rate":97},{"timestamp":"2017-10-11 17:10:52.098","Rate":31},{"timestamp":"2017-10-11 17:10:52.100","Rate":49},{"timestamp":"2017-10-11 17:10:52.103","Rate":36},{"timestamp":"2017-10-11 17:10:52.106","Rate":15},{"timestamp":"2017-10-11 17:10:52.109","Rate":67},{"timestamp":"2017-10-11 17:10:52.111","Rate":67},{"timestamp":"2017-10-11 17:10:52.114","Rate":94},{"timestamp":"2017-10-11 17:10:52.116","Rate":68},{"timestamp":"2017-10-11 17:10:52.119","Rate":22},{"timestamp":"2017-10-11 17:10:52.122","Rate":54},{"timestamp":"2017-10-11 17:10:52.124","Rate":94},{"timestamp":"2017-10-11 17:10:52.127","Rate":49},{"timestamp":"2017-10-11 17:10:52.130","Rate":59},{"timestamp":"2017-10-11 17:10:52.132","Rate":6},{"timestamp":"2017-10-11 17:10:52.135","Rate":82},{"timestamp":"2017-10-11 17:10:52.137","Rate":5},{"timestamp":"2017-10-11 17:10:52.140","Rate":1},{"timestamp":"2017-10-11 17:10:52.142","Rate":53},{"timestamp":"2017-10-11 17:10:52.145","Rate":69},{"timestamp":"2017-10-11 17:10:52.147","Rate":97},{"timestamp":"2017-10-11 17:10:52.150","Rate":58},{"timestamp":"2017-10-11 17:10:52.153","Rate":76},{"timestamp":"2017-10-11 17:10:52.157","Rate":81},{"timestamp":"2017-10-11 17:10:52.160","Rate":30},{"timestamp":"2017-10-11 17:10:52.163","Rate":4},{"timestamp":"2017-10-11 17:10:52.165","Rate":67},{"timestamp":"2017-10-11 17:10:52.169","Rate":5},{"timestamp":"2017-10-11 17:10:52.171","Rate":72},{"timestamp":"2017-10-11 17:10:52.174","Rate":20},{"timestamp":"2017-10-11 17:10:52.176","Rate":58},{"timestamp":"2017-10-11 17:10:52.179","Rate":75},{"timestamp":"2017-10-11 17:10:52.182","Rate":74},{"timestamp":"2017-10-11 17:10:52.184","Rate":60},{"timestamp":"2017-10-11 17:10:52.187","Rate":96},{"timestamp":"2017-10-11 17:10:52.189","Rate":30},{"timestamp":"2017-10-11 17:10:52.192","Rate":40},{"timestamp":"2017-10-11 17:10:52.195","Rate":33},{"timestamp":"2017-10-11 17:10:52.197","Rate":87},{"timestamp":"2017-10-11 17:10:52.200","Rate":67},{"timestamp":"2017-10-11 17:10:52.203","Rate":40},{"timestamp":"2017-10-11 17:10:52.206","Rate":44},{"timestamp":"2017-10-11 17:10:52.208","Rate":7},{"timestamp":"2017-10-11 17:10:52.211","Rate":52},{"timestamp":"2017-10-11 17:10:52.214","Rate":93},{"timestamp":"2017-10-11 17:10:52.219","Rate":43},{"timestamp":"2017-10-11 17:10:52.222","Rate":66},{"timestamp":"2017-10-11 17:10:52.225","Rate":8},{"timestamp":"2017-10-11 17:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/65 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/65 new file mode 100644 index 0000000000..a9bebbfede --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/65 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 4 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/66 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/66 new file mode 100644 index 0000000000..6bcad7bc37 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/66 @@ -0,0 +1 @@ +{"count":4,"rows":[{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/67 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/67 new file mode 100644 index 0000000000..c3fd30822b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/67 @@ -0,0 +1 @@ +{"count":2,"rows":[{"description":"A test row"},{"description":"Updated with expression"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/68 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/68 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/68 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/69 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/69 new file mode 100644 index 0000000000..a5d4634456 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/69 @@ -0,0 +1 @@ +{"count":2,"rows":[{"id":2,"key":"UPDA","description":"updated description","data":{"json":"inserted object"}},{"id":1,"key":"TEST1","description":"A test row","data":{"json":"new value"}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/7 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/7 new file mode 100644 index 0000000000..379611dc02 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/7 @@ -0,0 +1 @@ +{"count":1,"rows":[{"count_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/70 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/70 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/70 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/71 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/71 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/71 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/72 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/72 new file mode 100644 index 0000000000..858823dc3c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/72 @@ -0,0 +1 @@ +{"count":1,"rows":[{"id":4,"key":"Admin","description":"URL of the admin API","data":{"url":{"value":"new value"}}}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/73 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/73 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/73 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/74 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/74 new file mode 100644 index 0000000000..4bde9235b1 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/74 @@ -0,0 +1 @@ +{"count":1,"rows":[{"Count":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/75 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/75 new file mode 100644 index 0000000000..973df944e3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/75 @@ -0,0 +1 @@ +{"count":9,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"},{"id":106,"key":"TEST6","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:33.622"},{"id":106,"key":"TEST7","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:16:20.622"},{"id":108,"key":"TEST8","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 05:14:30.622"},{"id":109,"key":"TEST9","description":"Updated with expression","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 21:14:30.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/76 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/76 new file mode 100644 index 0000000000..7e9c80429f --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/76 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"value\" of an \"newer\" condition must be an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/77 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/77 new file mode 100644 index 0000000000..88ff5140f2 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/77 @@ -0,0 +1 @@ +{"count":5,"rows":[{"id":1,"key":"TEST1","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:26.622"},{"id":2,"key":"TEST2","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 12:14:27.422"},{"id":3,"key":"TEST3","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:28.622"},{"id":4,"key":"TEST4","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:14:29.622"},{"id":5,"key":"TEST5","description":"A test row","data":{"prop1":"test1","obj1":{"p1":"v1","p2":"v2"}},"ts":"2017-10-10 11:15:00.622"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/78 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/78 new file mode 100644 index 0000000000..543994f16c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/78 @@ -0,0 +1 @@ +{"count":2,"rows":[{"min":2,"max":96,"average":47.9523809523809,"user_ts":"2017-10-11 17:10:51"},{"min":1,"max":98,"average":53.7721518987342,"user_ts":"2017-10-11 17:10:52"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/79 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/79 new file mode 100644 index 0000000000..7d66483ebe --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/79 @@ -0,0 +1 @@ +{ "response" : "inserted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/8 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/8 new file mode 100644 index 0000000000..cd34346b4c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/8 @@ -0,0 +1 @@ +{"count":1,"rows":[{"avg_id":1.0}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/80 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/80 new file mode 100644 index 0000000000..29146ee48a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/80 @@ -0,0 +1 @@ +{ "response" : "deleted", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/81 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/81 new file mode 100644 index 0000000000..c614b3d6e0 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/81 @@ -0,0 +1 @@ +{ "entryPoint" : "appendReadings", "message" : "Payload is missing a readings array", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/82 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/82 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/82 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/83 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/83 new file mode 100644 index 0000000000..4401963f82 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/83 @@ -0,0 +1 @@ +{"count":100,"rows":[{"timestamp":"2017-10-11 17:10:51.927","Rate":90},{"timestamp":"2017-10-11 17:10:51.930","Rate":13},{"timestamp":"2017-10-11 17:10:51.933","Rate":84},{"timestamp":"2017-10-11 17:10:51.936","Rate":96},{"timestamp":"2017-10-11 17:10:51.939","Rate":2},{"timestamp":"2017-10-11 17:10:51.942","Rate":54},{"timestamp":"2017-10-11 17:10:51.946","Rate":28},{"timestamp":"2017-10-11 17:10:51.949","Rate":3},{"timestamp":"2017-10-11 17:10:51.952","Rate":77},{"timestamp":"2017-10-11 17:10:51.955","Rate":38},{"timestamp":"2017-10-11 17:10:51.959","Rate":26},{"timestamp":"2017-10-11 17:10:51.963","Rate":86},{"timestamp":"2017-10-11 17:10:51.966","Rate":39},{"timestamp":"2017-10-11 17:10:51.970","Rate":57},{"timestamp":"2017-10-11 17:10:51.973","Rate":73},{"timestamp":"2017-10-11 17:10:51.979","Rate":22},{"timestamp":"2017-10-11 17:10:51.982","Rate":34},{"timestamp":"2017-10-11 17:10:51.986","Rate":78},{"timestamp":"2017-10-11 17:10:51.990","Rate":20},{"timestamp":"2017-10-11 17:10:51.993","Rate":70},{"timestamp":"2017-10-11 17:10:51.996","Rate":17},{"timestamp":"2017-10-11 17:10:52.000","Rate":2},{"timestamp":"2017-10-11 17:10:52.005","Rate":18},{"timestamp":"2017-10-11 17:10:52.009","Rate":52},{"timestamp":"2017-10-11 17:10:52.012","Rate":62},{"timestamp":"2017-10-11 17:10:52.015","Rate":47},{"timestamp":"2017-10-11 17:10:52.019","Rate":73},{"timestamp":"2017-10-11 17:10:52.022","Rate":9},{"timestamp":"2017-10-11 17:10:52.026","Rate":66},{"timestamp":"2017-10-11 17:10:52.029","Rate":30},{"timestamp":"2017-10-11 17:10:52.031","Rate":70},{"timestamp":"2017-10-11 17:10:52.034","Rate":41},{"timestamp":"2017-10-11 17:10:52.037","Rate":2},{"timestamp":"2017-10-11 17:10:52.040","Rate":69},{"timestamp":"2017-10-11 17:10:52.043","Rate":98},{"timestamp":"2017-10-11 17:10:52.046","Rate":13},{"timestamp":"2017-10-11 17:10:52.050","Rate":91},{"timestamp":"2017-10-11 17:10:52.053","Rate":18},{"timestamp":"2017-10-11 17:10:52.056","Rate":78},{"timestamp":"2017-10-11 17:10:52.059","Rate":70},{"timestamp":"2017-10-11 17:10:52.062","Rate":48},{"timestamp":"2017-10-11 17:10:52.066","Rate":94},{"timestamp":"2017-10-11 17:10:52.070","Rate":79},{"timestamp":"2017-10-11 17:10:52.073","Rate":87},{"timestamp":"2017-10-11 17:10:52.075","Rate":60},{"timestamp":"2017-10-11 17:10:52.078","Rate":48},{"timestamp":"2017-10-11 17:10:52.081","Rate":88},{"timestamp":"2017-10-11 17:10:52.084","Rate":3},{"timestamp":"2017-10-11 17:10:52.086","Rate":93},{"timestamp":"2017-10-11 17:10:52.089","Rate":83},{"timestamp":"2017-10-11 17:10:52.092","Rate":76},{"timestamp":"2017-10-11 17:10:52.095","Rate":97},{"timestamp":"2017-10-11 17:10:52.098","Rate":31},{"timestamp":"2017-10-11 17:10:52.100","Rate":49},{"timestamp":"2017-10-11 17:10:52.103","Rate":36},{"timestamp":"2017-10-11 17:10:52.106","Rate":15},{"timestamp":"2017-10-11 17:10:52.109","Rate":67},{"timestamp":"2017-10-11 17:10:52.111","Rate":67},{"timestamp":"2017-10-11 17:10:52.114","Rate":94},{"timestamp":"2017-10-11 17:10:52.116","Rate":68},{"timestamp":"2017-10-11 17:10:52.119","Rate":22},{"timestamp":"2017-10-11 17:10:52.122","Rate":54},{"timestamp":"2017-10-11 17:10:52.124","Rate":94},{"timestamp":"2017-10-11 17:10:52.127","Rate":49},{"timestamp":"2017-10-11 17:10:52.130","Rate":59},{"timestamp":"2017-10-11 17:10:52.132","Rate":6},{"timestamp":"2017-10-11 17:10:52.135","Rate":82},{"timestamp":"2017-10-11 17:10:52.137","Rate":5},{"timestamp":"2017-10-11 17:10:52.140","Rate":1},{"timestamp":"2017-10-11 17:10:52.142","Rate":53},{"timestamp":"2017-10-11 17:10:52.145","Rate":69},{"timestamp":"2017-10-11 17:10:52.147","Rate":97},{"timestamp":"2017-10-11 17:10:52.150","Rate":58},{"timestamp":"2017-10-11 17:10:52.153","Rate":76},{"timestamp":"2017-10-11 17:10:52.157","Rate":81},{"timestamp":"2017-10-11 17:10:52.160","Rate":30},{"timestamp":"2017-10-11 17:10:52.163","Rate":4},{"timestamp":"2017-10-11 17:10:52.165","Rate":67},{"timestamp":"2017-10-11 17:10:52.169","Rate":5},{"timestamp":"2017-10-11 17:10:52.171","Rate":72},{"timestamp":"2017-10-11 17:10:52.174","Rate":20},{"timestamp":"2017-10-11 17:10:52.176","Rate":58},{"timestamp":"2017-10-11 17:10:52.179","Rate":75},{"timestamp":"2017-10-11 17:10:52.182","Rate":74},{"timestamp":"2017-10-11 17:10:52.184","Rate":60},{"timestamp":"2017-10-11 17:10:52.187","Rate":96},{"timestamp":"2017-10-11 17:10:52.189","Rate":30},{"timestamp":"2017-10-11 17:10:52.192","Rate":40},{"timestamp":"2017-10-11 17:10:52.195","Rate":33},{"timestamp":"2017-10-11 17:10:52.197","Rate":87},{"timestamp":"2017-10-11 17:10:52.200","Rate":67},{"timestamp":"2017-10-11 17:10:52.203","Rate":40},{"timestamp":"2017-10-11 17:10:52.206","Rate":44},{"timestamp":"2017-10-11 17:10:52.208","Rate":7},{"timestamp":"2017-10-11 17:10:52.211","Rate":52},{"timestamp":"2017-10-11 17:10:52.214","Rate":93},{"timestamp":"2017-10-11 17:10:52.219","Rate":43},{"timestamp":"2017-10-11 17:10:52.222","Rate":66},{"timestamp":"2017-10-11 17:10:52.225","Rate":8},{"timestamp":"2017-10-11 17:10:52.228","Rate":79}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/84 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/84 new file mode 100644 index 0000000000..6e3d42ac03 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/84 @@ -0,0 +1 @@ +{ "entryPoint" : "limit", "message" : "Limit must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/85 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/85 new file mode 100644 index 0000000000..8d705bbb14 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/85 @@ -0,0 +1 @@ +{ "entryPoint" : "skip", "message" : "Skip must be specfied as an integer", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/86 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/86 new file mode 100644 index 0000000000..4522ec1b85 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/86 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "unrecognized token: \":\"", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/87 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/87 new file mode 100644 index 0000000000..1d6fe91f5d --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/87 @@ -0,0 +1 @@ +{ "entryPoint" : "retrieve", "message" : "SQLite3 plugin does not support timezones in qeueries", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/88 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/88 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/88 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/89 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/89 new file mode 100644 index 0000000000..93af3344da --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/89 @@ -0,0 +1 @@ +{"count":1,"rows":[{"description":"added'some'ch'''ars'"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/9 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/9 new file mode 100644 index 0000000000..f951388d78 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/9 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min_id":1}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/90 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/90 new file mode 100644 index 0000000000..a8848cbc86 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/90 @@ -0,0 +1 @@ +{ "response" : "updated", "rows_affected" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/91 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/91 new file mode 100644 index 0000000000..54cbb9d707 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/91 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 17:10"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/92 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/92 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/92 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/93 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/93 new file mode 100644 index 0000000000..a4bfc7b379 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/93 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":"","max":"","average":""}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/94 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/94 new file mode 100644 index 0000000000..10da128d7a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/94 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.55,"timestamp":"2017-10-11 17"}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/95 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/95 new file mode 100644 index 0000000000..2eeb09a60b --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/95 @@ -0,0 +1 @@ +{ "response" : "appended", "readings_added" : 1 } \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/96 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/96 new file mode 100644 index 0000000000..124e6704de --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/96 @@ -0,0 +1 @@ +{"count":0,"rows":[]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/97 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/97 new file mode 100644 index 0000000000..22f00fc70c --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/97 @@ -0,0 +1 @@ +{"count":1,"rows":[{"count_id":10}]} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/98 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/98 new file mode 100644 index 0000000000..69ab957da3 --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/98 @@ -0,0 +1 @@ +{ "entryPoint" : "where clause", "message" : "The \"value\" of a \"in\" condition must be an array and must not be empty.", "retryable" : false} \ No newline at end of file diff --git a/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/99 b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/99 new file mode 100644 index 0000000000..cfddbd8a1a --- /dev/null +++ b/tests/unit/C/services/storage/sqlite/expected_EUROPE_ROME/99 @@ -0,0 +1 @@ +{"count":1,"rows":[{"min":1,"max":98,"average":52.9207920792079,"asset_code":"MyAsset"}]} \ No newline at end of file From 2926dc89ed728ca07ef0bcd9df1a0928639c7760 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Wed, 6 Mar 2019 17:01:55 +0530 Subject: [PATCH 69/86] E2E Test for varying asset structure added --- .../python/e2e/test_e2e_vary_asset_http_pi.py | 112 ++++++++++++++++++ .../python/scripts/install_python_plugin | 6 +- 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/system/python/e2e/test_e2e_vary_asset_http_pi.py diff --git a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py new file mode 100644 index 0000000000..f244c84390 --- /dev/null +++ b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +""" Test system/python/e2e/test_e2e_vary_asset_http_pi.py + +""" + +import http.client +import json +import time +from datetime import datetime, timezone +import uuid +import pytest + + +__author__ = "Vaibhav Singhal" +__copyright__ = "Copyright (c) 2019 Dianomic Systems" +__license__ = "Apache 2.0" +__version__ = "${VERSION}" + + +class TestE2EAssetHttpPI: + + @pytest.fixture + def start_south_north(self, reset_and_start_foglamp, add_south, start_north_pi_server_c, remove_directories, + south_branch, foglamp_url, pi_host, pi_port, pi_token): + """ This fixture clone a south repo and starts both south and north instance + reset_and_start_foglamp: Fixture that resets and starts foglamp, no explicit invocation, called at start + add_south: Fixture that adds a south service with given configuration + start_north_pi_server_c: Fixture that starts PI north task + remove_directories: Fixture that remove directories created during the tests""" + + south_plugin = "http" + add_south("http_south", south_branch, foglamp_url, config={"assetNamePrefix": {"value": ""}}, + service_name="http_south") + start_north_pi_server_c(foglamp_url, pi_host, pi_port, pi_token) + + yield self.start_south_north + + # Cleanup code that runs after the caller test is over + remove_directories("/tmp/foglamp-south-{}".format(south_plugin)) + + def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_host, pi_admin, pi_passwd, pi_db, + wait_time, retries): + """ Test that data is inserted in FogLAMP and sent to PI + start_south_north: Fixture that starts FogLAMP with south and north instance + read_data_from_pi: Fixture to read data from PI + Assertions: + on endpoint GET /foglamp/asset + on endpoint GET /foglamp/asset/ + data received from PI is same as data sent""" + + conn = http.client.HTTPConnection(foglamp_url) + time.sleep(wait_time) + + # Send data to foglamp-south-http + conn_http_south = http.client.HTTPConnection("localhost:6683") + + asset_name = "e2e_varying" + sensor_data = [{"a": 1}, {"a": 2, "b": 3}, {"b": 4}] + for d in sensor_data: + tm = str(datetime.now(timezone.utc).astimezone()) + data = [{"asset": "{}".format(asset_name), "timestamp": "{}".format(tm), "key": str(uuid.uuid4()), + "readings": d}] + conn_http_south.request("POST", '/sensor-reading', json.dumps(data)) + r = conn_http_south.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert {'result': 'success'} == jdoc + + time.sleep(wait_time) + conn.request("GET", '/foglamp/asset') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + retval = json.loads(r) + assert len(retval) == 1 + assert asset_name == retval[0]["assetCode"] + assert 3 == retval[0]["count"] + + conn.request("GET", '/foglamp/asset/{}'.format(asset_name)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + retval = json.loads(r) + assert sensor_data[2] == retval[0]["reading"] + assert sensor_data[1] == retval[1]["reading"] + assert sensor_data[0] == retval[2]["reading"] + + retry_count = 0 + data_from_pi = None + while (data_from_pi is None or data_from_pi == []) and retry_count < retries: + data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, asset_name, {"a", "b"}) + retry_count += 1 + time.sleep(wait_time*2) + + if data_from_pi is None or retry_count == retries: + assert False, "Failed to read data from PI" + + assert data_from_pi["b"][-1] == sensor_data[2]["b"] + assert data_from_pi["b"][-2] == sensor_data[1]["b"] + # On PI b does not have a datapoint for 1st set {"a": 1} and hence value is 0.0 + assert data_from_pi["b"][-3] == 0.0 + + # On PI a does not have a datapoint for 3rd set {"b": 4} and hence value is 0.0 + assert data_from_pi["a"][-1] == 0.0 + assert data_from_pi["a"][-2] == sensor_data[1]["a"] + assert data_from_pi["a"][-3] == sensor_data[0]["a"] diff --git a/tests/system/python/scripts/install_python_plugin b/tests/system/python/scripts/install_python_plugin index 1a49b3637d..9b659486e7 100755 --- a/tests/system/python/scripts/install_python_plugin +++ b/tests/system/python/scripts/install_python_plugin @@ -41,7 +41,11 @@ clone_repo () { } copy_file_and_requirement () { - cp -r /tmp/${REPO_NAME}/python/foglamp/plugins/${PLUGIN_TYPE}/${PLUGIN_NAME} $FOGLAMP_ROOT/python/foglamp/plugins/${PLUGIN_TYPE}/ + if [ "$PLUGIN_NAME" = "http" ]; then + cp -r /tmp/${REPO_NAME}/python/foglamp/plugins/${PLUGIN_TYPE}/${PLUGIN_NAME}_south $FOGLAMP_ROOT/python/foglamp/plugins/${PLUGIN_TYPE}/ + else + cp -r /tmp/${REPO_NAME}/python/foglamp/plugins/${PLUGIN_TYPE}/${PLUGIN_NAME} $FOGLAMP_ROOT/python/foglamp/plugins/${PLUGIN_TYPE}/ + fi req_file=$(find /tmp/${REPO_NAME} -name requirement*.txt) [ ! -z "${req_file}" ] && pip3 install --user -Ir ${req_file} ${USE_PIP_CACHE} || echo "No such external dependency needed for ${PLUGIN_NAME} plugin." } From 9317fff77285cf9511520c678f7b838c85ec9846 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 6 Mar 2019 13:03:53 +0000 Subject: [PATCH 70/86] FOGL-2548 Allow readings per minute and per hour as well as per second --- C/services/south/include/defaults.h | 4 +-- C/services/south/include/south_service.h | 4 +-- C/services/south/south.cpp | 42 ++++++++++++++++++++---- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/C/services/south/include/defaults.h b/C/services/south/include/defaults.h index b2b2129ff3..b1e7c21ab0 100644 --- a/C/services/south/include/defaults.h +++ b/C/services/south/include/defaults.h @@ -17,12 +17,12 @@ static struct { const char *type; const char *value; } defaults[] = { - { "readingsPerSec", "Readings Per Second", - "Number of readings to generate per sec", "integer", "1" }, { "maxSendLatency", "Maximum Reading Latency (mS)", "Maximum time to spend filling buffer before sending", "integer", "5000" }, { "bufferThreshold", "Maximum buffered Readings", "Number of readings to buffer before sending", "integer", "100" }, + { "readingsPerSec", "Reading Rate", + "Number of readings to generate per interval", "integer", "1" }, { NULL, NULL, NULL, NULL, NULL } }; #endif diff --git a/C/services/south/include/south_service.h b/C/services/south/include/south_service.h index e7caf14617..f8b105b11c 100644 --- a/C/services/south/include/south_service.h +++ b/C/services/south/include/south_service.h @@ -37,7 +37,7 @@ class SouthService : public ServiceHandler { private: void addConfigDefaults(DefaultConfigCategory& defaults); bool loadPlugin(); - int createTimerFd(int usecs); + int createTimerFd(struct timeval rate); void createConfigCategories(DefaultConfigCategory configCategory, std::string parent_name,std::string current_name); private: SouthPlugin *southPlugin; @@ -48,7 +48,7 @@ class SouthService : public ServiceHandler { ConfigCategory m_config; ConfigCategory m_configAdvanced; ManagementClient *m_mgtClient; - unsigned long m_readingsPerSec; + unsigned long m_readingsPerSec; // May not be per second, new rate defines time units unsigned int m_threshold; unsigned long m_timeout; Ingest *m_ingest; diff --git a/C/services/south/south.cpp b/C/services/south/south.cpp index d6b0037e2d..a828c8a7dc 100644 --- a/C/services/south/south.cpp +++ b/C/services/south/south.cpp @@ -285,7 +285,19 @@ void SouthService::start(string& coreAddress, unsigned short corePort) // Get and ingest data if (! southPlugin->isAsync()) { - m_timerfd = createTimerFd(1000000/(int)m_readingsPerSec); // interval to be passed is in usecs + string units = m_configAdvanced.getValue("units"); + unsigned long dividend = 1000000; + if (units.compare("second") == 0) + dividend = 1000000; + else if (units.compare("minute") == 0) + dividend = 60000000; + else if (units.compare("hour") == 0) + dividend = 3600000000; + unsigned long usecs = dividend / m_readingsPerSec; + struct timeval rate; + rate.tv_sec = usecs / 1000000; + rate.tv_usec = usecs % 1000000; + m_timerfd = createTimerFd(rate); // interval to be passed is in usecs if (m_timerfd < 0) { logger->fatal("Could not create timer FD"); @@ -537,11 +549,23 @@ void SouthService::configChange(const string& categoryName, const string& catego m_configAdvanced = ConfigCategory(m_name+"Advanced", category); try { unsigned long newval = (unsigned long)strtol(m_configAdvanced.getValue("readingsPerSec").c_str(), NULL, 10); + string units = m_configAdvanced.getValue("units"); + unsigned long dividend = 1000000; + if (units.compare("second") == 0) + dividend = 1000000; + else if (units.compare("minute") == 0) + dividend = 60000000; + else if (units.compare("hour") == 0) + dividend = 3600000000; if (newval != m_readingsPerSec) { m_readingsPerSec = newval; close(m_timerfd); - m_timerfd = createTimerFd(1000000/(int)m_readingsPerSec); // interval to be passed is in usecs + unsigned long usecs = dividend / m_readingsPerSec; + struct timeval rate; + rate.tv_sec = usecs / 1000000; + rate.tv_usec = usecs % 1000000; + m_timerfd = createTimerFd(rate); // interval to be passed is in usecs } } catch (ConfigItemNotFound e) { logger->error("Failed to update poll interval following configuration change"); @@ -576,6 +600,12 @@ void SouthService::addConfigDefaults(DefaultConfigCategory& defaultConfig) defaultConfig.setItemDisplayName(defaults[i].name, defaults[i].displayName); } + /* Add the reading rate units */ + vector rateUnits = { "second", "minute", "hour" }; + defaultConfig.addItem("units", "Reading Rate Per", + "second", "second", rateUnits); + defaultConfig.setItemDisplayName("units", "Reading Rate Per"); + /* Add the set of logging levels to the service */ vector logLevels = { "error", "warning", "info", "debug" }; defaultConfig.addItem("logLevel", "Minimum logging level reported", @@ -589,7 +619,7 @@ void SouthService::addConfigDefaults(DefaultConfigCategory& defaultConfig) * * @param usecs Time in micro-secs after which data would be available on the timer FD */ -int SouthService::createTimerFd(int usecs) +int SouthService::createTimerFd(struct timeval rate) { int fd = -1; struct itimerspec new_value; @@ -599,15 +629,15 @@ int SouthService::createTimerFd(int usecs) Logger::getLogger()->error("clock_gettime"); new_value.it_value.tv_sec = now.tv_sec; - new_value.it_value.tv_nsec = now.tv_nsec + usecs*1000; + new_value.it_value.tv_nsec = now.tv_nsec + rate.tv_usec*1000; if (new_value.it_value.tv_nsec >= 1000000000) { new_value.it_value.tv_sec += new_value.it_value.tv_nsec/1000000000; new_value.it_value.tv_nsec %= 1000000000; } - new_value.it_interval.tv_sec = 0; - new_value.it_interval.tv_nsec = usecs*1000; + new_value.it_interval.tv_sec = rate.tv_sec; + new_value.it_interval.tv_nsec = rate.tv_usec*1000; if (new_value.it_interval.tv_nsec >= 1000000000) { new_value.it_interval.tv_sec += new_value.it_interval.tv_nsec/1000000000; From 5e079483cbb937b31d3ab1d3c1c2bacd24765e59 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Wed, 6 Mar 2019 19:13:02 +0530 Subject: [PATCH 71/86] rule optional attribute added; handled on create/update category along with unit tests --- .../foglamp/common/configuration_manager.py | 10 ++- .../common/test_configuration_manager.py | 62 ++++++++++++++++--- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/python/foglamp/common/configuration_manager.py b/python/foglamp/common/configuration_manager.py index 8aecdfa942..6d2d47b92f 100644 --- a/python/foglamp/common/configuration_manager.py +++ b/python/foglamp/common/configuration_manager.py @@ -13,6 +13,7 @@ import ipaddress import datetime import os +from math import * from foglamp.common.storage_client.payload_builder import PayloadBuilder from foglamp.common.storage_client.storage_client import StorageClientAsync @@ -218,7 +219,7 @@ async def _validate_category_val(self, category_name, category_val, set_value_va .format(category_name, item_name, type(item_val))) optional_item_entries = {'readonly': 0, 'order': 0, 'length': 0, 'maximum': 0, 'minimum': 0, - 'deprecated': 0, 'displayName': 0} + 'deprecated': 0, 'displayName': 0, 'rule': 0} expected_item_entries = {'description': 0, 'default': 0, 'type': 0} if require_entry_value: @@ -269,7 +270,7 @@ def get_entry_val(k): if (self._validate_type_value('integer', entry_val) or self._validate_type_value('float', entry_val)) is False: raise ValueError('For {} category, entry value must be an integer or float for item name ' '{}; got {}'.format(category_name, entry_name, type(entry_val))) - elif entry_name == 'displayName': + elif entry_name == 'rule' or entry_name == 'displayName': if not isinstance(entry_val, str): raise ValueError('For {} category, entry value must be string for item name {}; got {}' .format(category_name, entry_name, type(entry_val))) @@ -786,6 +787,11 @@ async def create_category(self, category_name, category_value, category_descript try: # validate new category_val, set "value" from default category_val_prepared = await self._validate_category_val(category_name, category_value, True) + for item_name in category_val_prepared: + if 'rule' in category_val_prepared[item_name]: + category_val_prepared[item_name]['rule'] = category_val_prepared[item_name]['rule'].replace("value", category_val_prepared[item_name]['value']) + if eval(category_val_prepared[item_name]['rule']) is False: + raise ValueError('For {} category, Proposed value for item_name {} is not allowed as per rule defined'.format(category_name, item_name)) # check if category_name is already in storage category_val_storage = await self._read_category_val(category_name) if category_val_storage is None: diff --git a/tests/unit/python/foglamp/common/test_configuration_manager.py b/tests/unit/python/foglamp/common/test_configuration_manager.py index 11e063f441..83da7c1c34 100644 --- a/tests/unit/python/foglamp/common/test_configuration_manager.py +++ b/tests/unit/python/foglamp/common/test_configuration_manager.py @@ -835,6 +835,52 @@ async def test__merge_category_vals_no_mutual_items_include_original(self, reset assert test_config_storage is not c_return_value assert test_config_new is not test_config_storage + @pytest.mark.parametrize("payload, message", [ + ((2, 'catvalue', 'catdesc'), "category_name must be a string"), + (('catname', 'catvalue', 3), "category_description must be a string") + ]) + async def test_bad_create_category(self, reset_singleton, payload, message): + storage_client_mock = MagicMock(spec=StorageClientAsync) + c_mgr = ConfigurationManager(storage_client_mock) + with pytest.raises(Exception) as excinfo: + await c_mgr.create_category(category_name=payload[0], category_value=payload[1], category_description=payload[2]) + assert excinfo.type is TypeError + assert message == str(excinfo.value) + + @pytest.mark.parametrize("rule", [ + 'value * 3 == 6', + 'value > 4', + 'value % 2 == 0', + 'value * (value + 1) == 9', + '(value + 1) / (value - 1) >= 3', + 'sqrt(value) < 1', + 'pow(value, value) != 27', + 'value ^ value == 2', + 'factorial(value) != 6', + 'fabs(value) != 3.0', + 'ceil(value) != 3', + 'floor(value) != 3', + 'sin(value) <= 0', + 'degrees(value) < 171' + ]) + @pytest.mark.asyncio + async def test_bad_rule_create_category(self, reset_singleton, rule): + + async def async_mock(return_value): + return return_value + + d = {'info': {'rule': rule, 'default': '3', 'type': 'integer', 'description': 'Test', 'value': '3'}} + storage_client_mock = MagicMock(spec=StorageClientAsync) + c_mgr = ConfigurationManager(storage_client_mock) + with patch.object(_logger, 'exception') as log_exc: + with patch.object(ConfigurationManager, '_validate_category_val', side_effect=[async_mock(d), Exception()]) as valpatch: + with pytest.raises(Exception) as excinfo: + await c_mgr.create_category('catname', 'catvalue', 'catdesc') + assert excinfo.type is ValueError + assert 'For catname category, Proposed value for item_name info is not allowed as per rule defined' == str(excinfo.value) + valpatch.assert_called_once_with('catname', 'catvalue', True) + assert 1 == log_exc.call_count + @pytest.mark.asyncio async def test_create_category_good_newval_bad_storageval_good_update(self, reset_singleton): @@ -966,13 +1012,13 @@ async def async_mock(return_value): storage_client_mock = MagicMock(spec=StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) - with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock(None)) as valpatch: + with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock({})) as valpatch: with patch.object(ConfigurationManager, '_read_category_val', return_value=async_mock(None)) as readpatch: with patch.object(ConfigurationManager, '_create_new_category', return_value=async_mock(None)) as createpatch: with patch.object(ConfigurationManager, '_run_callbacks', return_value=async_mock(None)) as callbackpatch: await c_mgr.create_category('catname', 'catvalue', "catdesc") callbackpatch.assert_called_once_with('catname') - createpatch.assert_called_once_with('catname', None, 'catdesc', None) + createpatch.assert_called_once_with('catname', {}, 'catdesc', None) readpatch.assert_called_once_with('catname') valpatch.assert_called_once_with('catname', 'catvalue', True) @@ -985,18 +1031,18 @@ async def async_mock(return_value): storage_client_mock = MagicMock(spec=StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(_logger, 'exception') as log_exc: - with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock(None)) as valpatch: + with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock({})) as valpatch: with patch.object(ConfigurationManager, '_read_category_val', return_value=async_mock(None)) as readpatch: with patch.object(ConfigurationManager, '_create_new_category', side_effect=StorageServerError(None, None, None)) as createpatch: with patch.object(ConfigurationManager, '_run_callbacks') as callbackpatch: with pytest.raises(StorageServerError): await c_mgr.create_category('catname', 'catvalue', "catdesc") callbackpatch.assert_not_called() - createpatch.assert_called_once_with('catname', None, 'catdesc', None) + createpatch.assert_called_once_with('catname', {}, 'catdesc', None) readpatch.assert_called_once_with('catname') valpatch.assert_called_once_with('catname', 'catvalue', True) assert 1 == log_exc.call_count - log_exc.assert_called_once_with('Unable to create new category based on category_name %s and category_description %s and category_json_schema %s', 'catname', 'catdesc', None) + log_exc.assert_called_once_with('Unable to create new category based on category_name %s and category_description %s and category_json_schema %s', 'catname', 'catdesc', {}) @pytest.mark.asyncio async def test_create_category_good_newval_keyerror_bad_create(self, reset_singleton): @@ -1007,18 +1053,18 @@ async def async_mock(return_value): storage_client_mock = MagicMock(spec=StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(_logger, 'exception') as log_exc: - with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock(None)) as valpatch: + with patch.object(ConfigurationManager, '_validate_category_val', return_value=async_mock({})) as valpatch: with patch.object(ConfigurationManager, '_read_category_val', return_value=async_mock(None)) as readpatch: with patch.object(ConfigurationManager, '_create_new_category', side_effect=KeyError()) as createpatch: with patch.object(ConfigurationManager, '_run_callbacks') as callbackpatch: with pytest.raises(KeyError): await c_mgr.create_category('catname', 'catvalue', "catdesc") callbackpatch.assert_not_called() - createpatch.assert_called_once_with('catname', None, 'catdesc', None) + createpatch.assert_called_once_with('catname', {}, 'catdesc', None) readpatch.assert_called_once_with('catname') valpatch.assert_called_once_with('catname', 'catvalue', True) assert 1 == log_exc.call_count - log_exc.assert_called_once_with('Unable to create new category based on category_name %s and category_description %s and category_json_schema %s', 'catname', 'catdesc', None) + log_exc.assert_called_once_with('Unable to create new category based on category_name %s and category_description %s and category_json_schema %s', 'catname', 'catdesc', {}) @pytest.mark.asyncio async def test_create_category_bad_newval(self, reset_singleton): From cc04c22988e69799dbb211b69db8acf1b3cb1240 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Wed, 6 Mar 2019 19:47:40 +0530 Subject: [PATCH 72/86] Feedback fixes --- .../python/e2e/test_e2e_vary_asset_http_pi.py | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py index f244c84390..d22cbb719f 100644 --- a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py +++ b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py @@ -54,14 +54,16 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ data received from PI is same as data sent""" conn = http.client.HTTPConnection(foglamp_url) - time.sleep(wait_time) # Send data to foglamp-south-http conn_http_south = http.client.HTTPConnection("localhost:6683") asset_name = "e2e_varying" + # 2 list having mixed data simulating different sensors + # (sensors coming up and down, sensors throwing int and float data) sensor_data = [{"a": 1}, {"a": 2, "b": 3}, {"b": 4}] - for d in sensor_data: + sensor_data_2 = [{"b": 1.1}, {"a2": 2, "b2": 3}, {"a": 4.0}] + for d in sensor_data + sensor_data_2: tm = str(datetime.now(timezone.utc).astimezone()) data = [{"asset": "{}".format(asset_name), "timestamp": "{}".format(tm), "key": str(uuid.uuid4()), "readings": d}] @@ -72,6 +74,7 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ jdoc = json.loads(r) assert {'result': 'success'} == jdoc + # Allow some buffer so that data is ingested before retrieval time.sleep(wait_time) conn.request("GET", '/foglamp/asset') r = conn.getresponse() @@ -80,33 +83,60 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ retval = json.loads(r) assert len(retval) == 1 assert asset_name == retval[0]["assetCode"] - assert 3 == retval[0]["count"] + assert 6 == retval[0]["count"] conn.request("GET", '/foglamp/asset/{}'.format(asset_name)) r = conn.getresponse() assert 200 == r.status r = r.read().decode() retval = json.loads(r) - assert sensor_data[2] == retval[0]["reading"] - assert sensor_data[1] == retval[1]["reading"] - assert sensor_data[0] == retval[2]["reading"] + assert sensor_data_2[2] == retval[0]["reading"] + assert sensor_data_2[1] == retval[1]["reading"] + assert sensor_data_2[0] == retval[2]["reading"] + assert sensor_data[2] == retval[3]["reading"] + assert sensor_data[1] == retval[4]["reading"] + assert sensor_data[0] == retval[5]["reading"] + + time.sleep(wait_time) retry_count = 0 data_from_pi = None while (data_from_pi is None or data_from_pi == []) and retry_count < retries: - data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, asset_name, {"a", "b"}) + data_from_pi = read_data_from_pi(pi_host, pi_admin, pi_passwd, pi_db, asset_name, {"a", "b", "a2", "b2"}) retry_count += 1 time.sleep(wait_time*2) if data_from_pi is None or retry_count == retries: assert False, "Failed to read data from PI" - assert data_from_pi["b"][-1] == sensor_data[2]["b"] - assert data_from_pi["b"][-2] == sensor_data[1]["b"] - # On PI b does not have a datapoint for 1st set {"a": 1} and hence value is 0.0 - assert data_from_pi["b"][-3] == 0.0 + assert data_from_pi["b"][-1] == 0.0 + assert data_from_pi["b"][-2] == 0.0 + assert data_from_pi["b"][-3] == sensor_data_2[0]["b"] + assert data_from_pi["b"][-4] == sensor_data[2]["b"] + assert data_from_pi["b"][-5] == sensor_data[1]["b"] + assert data_from_pi["b"][-6] == 0.0 + + assert data_from_pi["a"][-1] == sensor_data_2[2]["a"] + assert data_from_pi["a"][-2] == 0.0 + assert data_from_pi["a"][-3] == 0.0 + assert data_from_pi["a"][-4] == 0.0 + assert data_from_pi["a"][-5] == sensor_data[1]["a"] + assert data_from_pi["a"][-6] == sensor_data[0]["a"] + + assert data_from_pi["b2"][-1] == 0.0 + assert data_from_pi["b2"][-2] == sensor_data_2[1]["b2"] + assert data_from_pi["b2"][-3] == 0.0 + assert data_from_pi["b2"][-4] == 0.0 + assert data_from_pi["b2"][-5] == 0.0 + assert data_from_pi["b2"][-6] == 0.0 + + assert data_from_pi["a2"][-1] == 0.0 + assert data_from_pi["a2"][-2] == sensor_data_2[1]["a2"] + assert data_from_pi["a2"][-3] == 0.0 + assert data_from_pi["a2"][-4] == 0.0 + assert data_from_pi["a2"][-5] == 0.0 + assert data_from_pi["a2"][-6] == 0.0 + + + - # On PI a does not have a datapoint for 3rd set {"b": 4} and hence value is 0.0 - assert data_from_pi["a"][-1] == 0.0 - assert data_from_pi["a"][-2] == sensor_data[1]["a"] - assert data_from_pi["a"][-3] == sensor_data[0]["a"] From 11e95d2cd4e84f2d130be9d0a81d9840fb42e977 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Wed, 6 Mar 2019 19:50:39 +0530 Subject: [PATCH 73/86] Minor fix --- tests/system/python/e2e/test_e2e_vary_asset_http_pi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py index d22cbb719f..c49d3b3496 100644 --- a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py +++ b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py @@ -76,6 +76,7 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ # Allow some buffer so that data is ingested before retrieval time.sleep(wait_time) + conn.request("GET", '/foglamp/asset') r = conn.getresponse() assert 200 == r.status From d809e20c29593148d567aabc8192d89466febea3 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 6 Mar 2019 14:52:06 +0000 Subject: [PATCH 74/86] FOGL-2548 Fix warnings --- C/services/south/south.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/C/services/south/south.cpp b/C/services/south/south.cpp index a828c8a7dc..ff8d99e5e3 100644 --- a/C/services/south/south.cpp +++ b/C/services/south/south.cpp @@ -295,8 +295,8 @@ void SouthService::start(string& coreAddress, unsigned short corePort) dividend = 3600000000; unsigned long usecs = dividend / m_readingsPerSec; struct timeval rate; - rate.tv_sec = usecs / 1000000; - rate.tv_usec = usecs % 1000000; + rate.tv_sec = (int)(usecs / 1000000); + rate.tv_usec = (int)(usecs % 1000000); m_timerfd = createTimerFd(rate); // interval to be passed is in usecs if (m_timerfd < 0) { @@ -563,8 +563,8 @@ void SouthService::configChange(const string& categoryName, const string& catego close(m_timerfd); unsigned long usecs = dividend / m_readingsPerSec; struct timeval rate; - rate.tv_sec = usecs / 1000000; - rate.tv_usec = usecs % 1000000; + rate.tv_sec = (int)(usecs / 1000000); + rate.tv_usec = (int)(usecs % 1000000); m_timerfd = createTimerFd(rate); // interval to be passed is in usecs } } catch (ConfigItemNotFound e) { From 1c2e81d44e2e85ec9ad4e4c4206f9f68bb9657eb Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 6 Mar 2019 16:59:09 +0000 Subject: [PATCH 75/86] FOGL-2548 Added seconds to initials call --- C/services/south/south.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/C/services/south/south.cpp b/C/services/south/south.cpp index ff8d99e5e3..087f9529b6 100644 --- a/C/services/south/south.cpp +++ b/C/services/south/south.cpp @@ -628,7 +628,7 @@ int SouthService::createTimerFd(struct timeval rate) if (clock_gettime(CLOCK_REALTIME, &now) == -1) Logger::getLogger()->error("clock_gettime"); - new_value.it_value.tv_sec = now.tv_sec; + new_value.it_value.tv_sec = now.tv_sec + rate.tv_sec; new_value.it_value.tv_nsec = now.tv_nsec + rate.tv_usec*1000; if (new_value.it_value.tv_nsec >= 1000000000) { From 284aa0664cee3d73a0902e7a4a46d6556a8d00a9 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Thu, 7 Mar 2019 12:12:40 +0530 Subject: [PATCH 76/86] rule validation as per new value got handled in bulk update endpoint; & other minor fixes to previous commit --- .../foglamp/common/configuration_manager.py | 9 +++++--- .../common/test_configuration_manager.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/python/foglamp/common/configuration_manager.py b/python/foglamp/common/configuration_manager.py index 6d2d47b92f..374d70ac76 100644 --- a/python/foglamp/common/configuration_manager.py +++ b/python/foglamp/common/configuration_manager.py @@ -468,7 +468,10 @@ async def update_configuration_item_bulk(self, category_name, config_item_list): for item_name, new_val in config_item_list.items(): if item_name not in cat_info: raise KeyError('{} config item not found'.format(item_name)) - + if 'rule' in cat_info[item_name]: + rule = cat_info[item_name]['rule'].replace("value", new_val) + if eval(rule) is False: + raise ValueError('Proposed value for item_name {} is not allowed as per rule defined'.format(item_name)) if cat_info[item_name]['type'] == 'JSON': if isinstance(new_val, dict): pass @@ -789,8 +792,8 @@ async def create_category(self, category_name, category_value, category_descript category_val_prepared = await self._validate_category_val(category_name, category_value, True) for item_name in category_val_prepared: if 'rule' in category_val_prepared[item_name]: - category_val_prepared[item_name]['rule'] = category_val_prepared[item_name]['rule'].replace("value", category_val_prepared[item_name]['value']) - if eval(category_val_prepared[item_name]['rule']) is False: + rule = category_val_prepared[item_name]['rule'].replace("value", category_val_prepared[item_name]['value']) + if eval(rule) is False: raise ValueError('For {} category, Proposed value for item_name {} is not allowed as per rule defined'.format(category_name, item_name)) # check if category_name is already in storage category_val_storage = await self._read_category_val(category_name) diff --git a/tests/unit/python/foglamp/common/test_configuration_manager.py b/tests/unit/python/foglamp/common/test_configuration_manager.py index 83da7c1c34..69aa112b38 100644 --- a/tests/unit/python/foglamp/common/test_configuration_manager.py +++ b/tests/unit/python/foglamp/common/test_configuration_manager.py @@ -2541,3 +2541,26 @@ async def async_mock(return_value): patch_audit.assert_not_called() patch_update.assert_not_called() patch_get_all_items.assert_called_once_with(category_name) + + @pytest.mark.parametrize("config_item_list", [ + {'info': "2"}, + {'info': "2", "info1": "9"}, + {'info1': "2", "info": "9"} + ]) + async def test_update_configuration_item_bulk_with_rule_optional_attribute(self, config_item_list, category_name='testcat'): + async def async_mock(return_value): + return return_value + + cat_info = {'info': {'rule': 'value*3==9', 'default': '3', 'description': 'Test', 'value': '3', 'type': 'integer'}, + 'info1': {'default': '3', 'description': 'Test', 'value': '3', 'type': 'integer'}} + storage_client_mock = MagicMock(spec=StorageClientAsync) + c_mgr = ConfigurationManager(storage_client_mock) + with patch.object(c_mgr, 'get_category_all_items', return_value=async_mock(cat_info)) as patch_get_all_items: + with patch.object(_logger, 'exception') as patch_log_exc: + with pytest.raises(Exception) as exc_info: + await c_mgr.update_configuration_item_bulk(category_name, config_item_list) + assert exc_info.type is ValueError + assert 'Proposed value for item_name {} is not allowed as per rule defined'.format( + list(cat_info.keys())[0]) == str(exc_info.value) + assert 1 == patch_log_exc.call_count + patch_get_all_items.assert_called_once_with(category_name) From 3e9944c9d5526fd1c0ecaf5e3d04721ecb77df72 Mon Sep 17 00:00:00 2001 From: Vaibhav Singhal Date: Thu, 7 Mar 2019 12:21:14 +0530 Subject: [PATCH 77/86] Feedback fixes --- tests/system/python/e2e/test_e2e_vary_asset_http_pi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py index c49d3b3496..2909a89480 100644 --- a/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py +++ b/tests/system/python/e2e/test_e2e_vary_asset_http_pi.py @@ -76,7 +76,7 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ # Allow some buffer so that data is ingested before retrieval time.sleep(wait_time) - + conn.request("GET", '/foglamp/asset') r = conn.getresponse() assert 200 == r.status @@ -99,7 +99,9 @@ def test_end_to_end(self, start_south_north, read_data_from_pi, foglamp_url, pi_ assert sensor_data[1] == retval[4]["reading"] assert sensor_data[0] == retval[5]["reading"] + # Allow some buffer so that data is ingested in PI before fetching using PI Web API time.sleep(wait_time) + retry_count = 0 data_from_pi = None while (data_from_pi is None or data_from_pi == []) and retry_count < retries: From b3fb12ca4ac0f2a7a16812e7fe5a59f97ddbfafe Mon Sep 17 00:00:00 2001 From: amarendra-dianomic Date: Thu, 7 Mar 2019 12:40:33 +0530 Subject: [PATCH 78/86] FOGL-2150 - Create a new endpoint to DELETE category --- .../common/microservice_management/routes.py | 1 + .../services/core/api/configuration.py | 31 ++++++++++++++++--- python/foglamp/services/core/routes.py | 1 + python/foglamp/services/core/server.py | 5 +++ .../services/core/api/test_configuration.py | 16 ++++++++++ 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/python/foglamp/services/common/microservice_management/routes.py b/python/foglamp/services/common/microservice_management/routes.py index 97c938fccd..f104f9f203 100644 --- a/python/foglamp/services/common/microservice_management/routes.py +++ b/python/foglamp/services/common/microservice_management/routes.py @@ -27,6 +27,7 @@ def setup(app, obj, is_core=False): app.router.add_route('GET', '/foglamp/service/category', obj.get_configuration_categories) app.router.add_route('POST', '/foglamp/service/category', obj.create_configuration_category) app.router.add_route('GET', '/foglamp/service/category/{category_name}', obj.get_configuration_category) + app.router.add_route('DELETE', '/foglamp/service/category/{category_name}', obj.delete_configuration_category) app.router.add_route('GET', '/foglamp/service/category/{category_name}/children', obj.get_child_category) app.router.add_route('POST', '/foglamp/service/category/{category_name}/children', obj.create_child_category) app.router.add_route('GET', '/foglamp/service/category/{category_name}/{config_item}', obj.get_configuration_item) diff --git a/python/foglamp/services/core/api/configuration.py b/python/foglamp/services/core/api/configuration.py index 8355c2fefb..4cd0bff08d 100644 --- a/python/foglamp/services/core/api/configuration.py +++ b/python/foglamp/services/core/api/configuration.py @@ -25,7 +25,7 @@ _help = """ -------------------------------------------------------------------------------- | GET POST | /foglamp/category | - | GET PUT | /foglamp/category/{category_name} | + | GET PUT DELETE | /foglamp/category/{category_name} | | GET POST PUT | /foglamp/category/{category_name}/{config_item} | | DELETE | /foglamp/category/{category_name}/{config_item}/value | | POST | /foglamp/category/{category_name}/{config_item}/upload | @@ -143,19 +143,40 @@ async def create_category(request): if data.get('children'): r = await cf_mgr.create_child_category(category_name, data.get('children')) result.update(r) - except (KeyError, ValueError, TypeError) as ex: raise web.HTTPBadRequest(reason=str(ex)) - except LookupError as ex: raise web.HTTPNotFound(reason=str(ex)) - except Exception as ex: raise web.HTTPException(reason=str(ex)) - return web.json_response(result) +async def delete_category(request): + """ + Args: + request: category_name required + Returns: + Success message on successful deletion + Raises: + TypeError/ValueError/Exception on error + :Example: + curl -X DELETE http://localhost:8081/foglamp/category/{category_name} + """ + category_name = request.match_info.get('category_name', None) + category_name = urllib.parse.unquote(category_name) if category_name is not None else None + + try: + cf_mgr = ConfigurationManager(connect.get_storage_async()) + await cf_mgr.delete_category_and_children_recursively(category_name) + except (ValueError, TypeError) as ex: + raise web.HTTPBadRequest(reason=ex) + except Exception as ex: + raise web.HTTPInternalServerError(reason=ex) + else: + return web.json_response({'result': 'Category {} deleted successfully.'.format(category_name)}) + + async def get_category_item(request): """ Args: diff --git a/python/foglamp/services/core/routes.py b/python/foglamp/services/core/routes.py index a51e98ae67..f6f390a012 100644 --- a/python/foglamp/services/core/routes.py +++ b/python/foglamp/services/core/routes.py @@ -61,6 +61,7 @@ def setup(app): app.router.add_route('POST', '/foglamp/category', api_configuration.create_category) app.router.add_route('GET', '/foglamp/category/{category_name}', api_configuration.get_category) app.router.add_route('PUT', '/foglamp/category/{category_name}', api_configuration.update_configuration_item_bulk) + app.router.add_route('DELETE', '/foglamp/category/{category_name}', api_configuration.delete_category) app.router.add_route('POST', '/foglamp/category/{category_name}/children', api_configuration.create_child_category) app.router.add_route('GET', '/foglamp/category/{category_name}/children', api_configuration.get_child_category) app.router.add_route('DELETE', '/foglamp/category/{category_name}/children/{child_category}', api_configuration.delete_child_category) diff --git a/python/foglamp/services/core/server.py b/python/foglamp/services/core/server.py index 3c78523e6e..f1e2b5db7d 100755 --- a/python/foglamp/services/core/server.py +++ b/python/foglamp/services/core/server.py @@ -1260,6 +1260,11 @@ async def create_configuration_category(cls, request): res = await conf_api.create_category(request) return res + @classmethod + async def delete_configuration_category(cls, request): + res = await conf_api.delete_category(request) + return res + @classmethod async def create_child_category(cls, request): res = await conf_api.create_child_category(request) diff --git a/tests/unit/python/foglamp/services/core/api/test_configuration.py b/tests/unit/python/foglamp/services/core/api/test_configuration.py index c162b36aec..dd29922ed0 100644 --- a/tests/unit/python/foglamp/services/core/api/test_configuration.py +++ b/tests/unit/python/foglamp/services/core/api/test_configuration.py @@ -688,3 +688,19 @@ async def async_mock(return_value): assert result == json_response patch_get_all_items.assert_called_once_with(category_name) patch_update_bulk.assert_called_once_with(category_name, payload) + + async def test_delete_configuration(self, client, category_name='rest_api'): + result = {'result': 'Category {} deleted successfully.'.format(category_name)} + storage_client_mock = MagicMock(StorageClientAsync) + c_mgr = ConfigurationManager(storage_client_mock) + with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): + with patch.object(c_mgr, 'delete_category_and_children_recursively', return_value=asyncio.sleep(.1)) as patch_delete_cat: + resp = await client.delete('/foglamp/category/{}'.format(category_name)) + assert 200 == resp.status + r = await resp.text() + json_response = json.loads(r) + assert result == json_response + assert 1 == patch_delete_cat.call_count + args, kwargs = patch_delete_cat.call_args + assert category_name == args[0] + From 81bd3573eeda57279f8a8576c508640c602184c5 Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Thu, 7 Mar 2019 13:02:25 +0530 Subject: [PATCH 79/86] rule validation as per new value entry got handled in set_config_item endpoint --- .../foglamp/common/configuration_manager.py | 7 +++++ .../common/test_configuration_manager.py | 30 +++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/python/foglamp/common/configuration_manager.py b/python/foglamp/common/configuration_manager.py index 374d70ac76..6de3199abd 100644 --- a/python/foglamp/common/configuration_manager.py +++ b/python/foglamp/common/configuration_manager.py @@ -468,6 +468,7 @@ async def update_configuration_item_bulk(self, category_name, config_item_list): for item_name, new_val in config_item_list.items(): if item_name not in cat_info: raise KeyError('{} config item not found'.format(item_name)) + # Evaluate new_val as per rule if defined if 'rule' in cat_info[item_name]: rule = cat_info[item_name]['rule'].replace("value", new_val) if eval(rule) is False: @@ -696,6 +697,11 @@ async def set_category_item_value_entry(self, category_name, item_name, new_valu raise TypeError('Unrecognized value name for item_name {}'.format(item_name)) new_value_entry = self._clean(storage_value_entry['type'], new_value_entry) + # Evaluate new_value_entry as per rule if defined + if 'rule' in storage_value_entry: + rule = storage_value_entry['rule'].replace("value", new_value_entry) + if eval(rule) is False: + raise ValueError('Proposed value for item_name {} is not allowed as per rule defined'.format(item_name)) await self._update_value_val(category_name, item_name, new_value_entry) # always get value from storage cat_item = await self._read_item_val(category_name, item_name) @@ -790,6 +796,7 @@ async def create_category(self, category_name, category_value, category_descript try: # validate new category_val, set "value" from default category_val_prepared = await self._validate_category_val(category_name, category_value, True) + # Evaluate value as per rule if defined for item_name in category_val_prepared: if 'rule' in category_val_prepared[item_name]: rule = category_val_prepared[item_name]['rule'].replace("value", category_val_prepared[item_name]['value']) diff --git a/tests/unit/python/foglamp/common/test_configuration_manager.py b/tests/unit/python/foglamp/common/test_configuration_manager.py index 69aa112b38..afd8ce3d24 100644 --- a/tests/unit/python/foglamp/common/test_configuration_manager.py +++ b/tests/unit/python/foglamp/common/test_configuration_manager.py @@ -1237,6 +1237,25 @@ async def async_mock(): readpatch.assert_called_once_with(category_name, item_name) assert 1 == log_exc.call_count + async def test_set_category_item_value_entry_with_rule_optional_attribute(self): + + async def async_mock(): + return {'rule': 'value*3==9', 'default': '3', 'description': 'Test', 'value': '3', 'type': 'integer'} + + storage_client_mock = MagicMock(spec=StorageClientAsync) + c_mgr = ConfigurationManager(storage_client_mock) + category_name = 'catname' + item_name = 'info' + new_value_entry = '13' + with patch.object(_logger, 'exception') as log_exc: + with patch.object(ConfigurationManager, '_read_item_val', return_value=async_mock()) as readpatch: + with pytest.raises(Exception) as excinfo: + await c_mgr.set_category_item_value_entry(category_name, item_name, new_value_entry) + assert excinfo.type is ValueError + assert 'Proposed value for item_name {} is not allowed as per rule defined'.format(item_name) == str(excinfo.value) + readpatch.assert_called_once_with(category_name, item_name) + assert 1 == log_exc.call_count + @pytest.mark.asyncio async def test_get_all_category_names_good(self, reset_singleton): @@ -2547,12 +2566,14 @@ async def async_mock(return_value): {'info': "2", "info1": "9"}, {'info1': "2", "info": "9"} ]) - async def test_update_configuration_item_bulk_with_rule_optional_attribute(self, config_item_list, category_name='testcat'): + async def test_update_configuration_item_bulk_with_rule_optional_attribute(self, config_item_list, + category_name='testcat'): async def async_mock(return_value): return return_value - cat_info = {'info': {'rule': 'value*3==9', 'default': '3', 'description': 'Test', 'value': '3', 'type': 'integer'}, - 'info1': {'default': '3', 'description': 'Test', 'value': '3', 'type': 'integer'}} + cat_info = {'info': {'rule': 'value*3==9', 'default': '3', 'description': 'Test', 'value': '3', + 'type': 'integer'}, 'info1': {'default': '3', 'description': 'Test', 'value': '3', + 'type': 'integer'}} storage_client_mock = MagicMock(spec=StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(c_mgr, 'get_category_all_items', return_value=async_mock(cat_info)) as patch_get_all_items: @@ -2560,7 +2581,6 @@ async def async_mock(return_value): with pytest.raises(Exception) as exc_info: await c_mgr.update_configuration_item_bulk(category_name, config_item_list) assert exc_info.type is ValueError - assert 'Proposed value for item_name {} is not allowed as per rule defined'.format( - list(cat_info.keys())[0]) == str(exc_info.value) + assert 'Proposed value for item_name info is not allowed as per rule defined' == str(exc_info.value) assert 1 == patch_log_exc.call_count patch_get_all_items.assert_called_once_with(category_name) From ec32260c887ca89730cb9c64d85226a19118ccca Mon Sep 17 00:00:00 2001 From: ashish-jabble Date: Thu, 7 Mar 2019 14:33:54 +0530 Subject: [PATCH 80/86] GET endpoint added to return list of notification types --- python/foglamp/services/core/api/notification.py | 9 +++++++++ python/foglamp/services/core/routes.py | 1 + .../foglamp/services/core/api/test_notification.py | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/python/foglamp/services/core/api/notification.py b/python/foglamp/services/core/api/notification.py index c348d49468..ee7f76c888 100644 --- a/python/foglamp/services/core/api/notification.py +++ b/python/foglamp/services/core/api/notification.py @@ -59,6 +59,15 @@ async def get_plugin(request): return web.json_response({'rules': rule_plugins, 'delivery': delivery_plugins}) +async def get_type(request): + """ GET the list of available notification types + + :Example: + curl -X GET http://localhost:8081/foglamp/notification/type + """ + return web.json_response({'notification_type': NOTIFICATION_TYPE}) + + async def get_notification(request): """ GET an existing notification diff --git a/python/foglamp/services/core/routes.py b/python/foglamp/services/core/routes.py index a51e98ae67..23eaff86e5 100644 --- a/python/foglamp/services/core/routes.py +++ b/python/foglamp/services/core/routes.py @@ -168,6 +168,7 @@ def setup(app): # Notification app.router.add_route('GET', '/foglamp/notification', notification.get_notifications) app.router.add_route('GET', '/foglamp/notification/plugin', notification.get_plugin) + app.router.add_route('GET', '/foglamp/notification/type', notification.get_type) app.router.add_route('GET', '/foglamp/notification/{notification_name}', notification.get_notification) app.router.add_route('POST', '/foglamp/notification', notification.post_notification) app.router.add_route('PUT', '/foglamp/notification/{notification_name}', notification.put_notification) diff --git a/tests/unit/python/foglamp/services/core/api/test_notification.py b/tests/unit/python/foglamp/services/core/api/test_notification.py index 0cbc33ccbb..5dd6b37771 100644 --- a/tests/unit/python/foglamp/services/core/api/test_notification.py +++ b/tests/unit/python/foglamp/services/core/api/test_notification.py @@ -316,6 +316,14 @@ async def test_get_plugin(self, mocker, client): json_response = json.loads(result) assert rules_and_delivery == json_response + async def test_get_type(self, client): + notification_type = {'notification_type': NOTIFICATION_TYPE} + resp = await client.get('/foglamp/notification/type') + assert 200 == resp.status + result = await resp.text() + json_response = json.loads(result) + assert notification_type == json_response + async def test_get_notification(self, mocker, client): r = list(filter(lambda rules: rules['name'] == notification_config['rule']['value'], rule_config)) c = list(filter(lambda channels: channels['name'] == notification_config['channel']['value'], delivery_config)) From 2fe376e61fae24ff261f7daead07187da8f76f3f Mon Sep 17 00:00:00 2001 From: Praveen Garg Date: Thu, 7 Mar 2019 15:36:29 +0530 Subject: [PATCH 81/86] fixed test_e2e_expr_pi.py and test added for config manager API --- tests/system/python/api/.gitkeep | 0 tests/system/python/e2e/test_e2e_expr_pi.py | 14 ++++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 tests/system/python/api/.gitkeep diff --git a/tests/system/python/api/.gitkeep b/tests/system/python/api/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/system/python/e2e/test_e2e_expr_pi.py b/tests/system/python/e2e/test_e2e_expr_pi.py index 61a2d50073..4469d3cd6f 100644 --- a/tests/system/python/e2e/test_e2e_expr_pi.py +++ b/tests/system/python/e2e/test_e2e_expr_pi.py @@ -47,18 +47,14 @@ def start_south_north(self, reset_and_start_foglamp, add_south, enable_schedule, "stepX": {"value": "0"}} add_south(SOUTH_PLUGIN, south_branch, foglamp_url, service_name=SVC_NAME, config=cfg, - plugin_lang=SOUTH_PLUGIN_LANGUAGE, start_service=False) + plugin_lang=SOUTH_PLUGIN_LANGUAGE, start_service=True) filter_cfg = {"enable": "true"} filter_plugin = "metadata" add_filter(filter_plugin, filter_branch, filter_name, filter_cfg, foglamp_url, SVC_NAME) - enable_schedule(foglamp_url, SVC_NAME) + # enable_schedule(foglamp_url, SVC_NAME) - # FIXME: FOGL-2417 - # We need to make north PI sending process to handle the case, to send and retrieve applied filter data - # in running service, so that we don't need to add south service in disabled mode And enable after applying - # filter pipeline start_north_pi_server_c(foglamp_url, pi_host, pi_port, pi_token) yield self.start_south_north @@ -103,7 +99,8 @@ def _verify_ingest(self, conn): assert 0 < len(jdoc) read = jdoc[0]["reading"] - assert 1.61977519054386 == read["Expression"] + # FOGL-2438 values like tan(45) = 1.61977519054386 gets truncated to 1.6197751905 with ingest + assert 1.6197751905 == read["Expression"] # verify filter is applied and we have {name: value} pair added by metadata filter assert "value" == read["name"] @@ -125,4 +122,5 @@ def _verify_egress(self, read_data_from_pi, pi_host, pi_admin, pi_passwd, pi_db, assert isinstance(data_from_pi["name"], list) assert isinstance(data_from_pi["Expression"], list) assert "value" in data_from_pi["name"] - assert 1.61977519054386 in data_from_pi["Expression"] + # FOGL-2438 values like tan(45) = 1.61977519054386 gets truncated to 1.6197751905 with ingest + assert 1.6197751905 in data_from_pi["Expression"] From 5728918ddb71c39ad662e395d9cd7f65445b5366 Mon Sep 17 00:00:00 2001 From: Praveen Garg Date: Thu, 7 Mar 2019 15:36:58 +0530 Subject: [PATCH 82/86] fixed test_e2e_expr_pi.py and test added for config manager API --- tests/system/python/api/test_configuration.py | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/system/python/api/test_configuration.py diff --git a/tests/system/python/api/test_configuration.py b/tests/system/python/api/test_configuration.py new file mode 100644 index 0000000000..6c66831c56 --- /dev/null +++ b/tests/system/python/api/test_configuration.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + +# FOGLAMP_BEGIN +# See: http://foglamp.readthedocs.io/ +# FOGLAMP_END + +""" Test Configuration REST API """ + + +import http.client +import json +from urllib.parse import quote +import time + +__author__ = "Praveen Garg" +__copyright__ = "Copyright (c) 2019 Dianomic Systems" +__license__ = "Apache 2.0" +__version__ = "${VERSION}" + + +class TestConfiguration: + + def test_default(self, foglamp_url, reset_and_start_foglamp, wait_time): + conn = http.client.HTTPConnection(foglamp_url) + + conn.request("GET", '/foglamp/category') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert len(jdoc) + + # Utilities parent key creation + time.sleep(wait_time) + + conn.request("GET", '/foglamp/category?root=true') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + cats = jdoc["categories"] + assert 3 == len(cats) + assert {'key': 'General', 'displayName': 'General', 'description': 'General'} == cats[0] + assert {'key': 'Advanced', 'displayName': 'Advanced', 'description': 'Advanced'} == cats[1] + assert {'key': 'Utilities', 'displayName': 'Utilities', 'description': 'Utilities'} == cats[2] + + conn.request("GET", '/foglamp/category?root=true&children=true') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 3 == len(jdoc["categories"]) + + expected_with_utilities = [ + {'children': [{'children': [], 'displayName': 'Admin API', 'key': 'rest_api', + 'description': 'FogLAMP Admin and User REST API'}, + {'children': [], 'displayName': 'FogLAMP Service', 'key': 'service', + 'description': 'FogLAMP Service'} + ], + 'displayName': 'General', 'key': 'General', 'description': 'General' + }, + {'children': [{'children': [], 'displayName': 'Scheduler', 'key': 'SCHEDULER', + 'description': 'Scheduler configuration'}, + {'children': [], 'displayName': 'Service Monitor', 'key': 'SMNTR', + 'description': 'Service Monitor'}], + 'displayName': 'Advanced', 'key': 'Advanced', 'description': 'Advanced' + }, + {'children': [], + 'displayName': 'Utilities', 'key': 'Utilities', 'description': 'Utilities' + } + ] + + assert expected_with_utilities == jdoc["categories"] + + def test_get_category(self, foglamp_url): + conn = http.client.HTTPConnection(foglamp_url) + conn.request("GET", '/foglamp/category/rest_api') + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert len(jdoc) + for k, v in jdoc.items(): + assert 'type' in v + assert 'value' in v + assert 'default' in v + assert 'description' in v + + assert 'displayName' in v + + def test_create_category(self, foglamp_url): + payload = {'key': 'pub #1', 'description': 'a publisher', 'display_name': 'Pub #1'} + conf = {'check': {'type': 'boolean', 'description': 'A Boolean check', 'default': 'False'}} + payload.update({'value': conf}) + conn = http.client.HTTPConnection(foglamp_url) + conn.request('POST', '/foglamp/category', body=json.dumps(payload)) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert "pub #1" == jdoc['key'] + assert "a publisher" == jdoc['description'] + assert "Pub #1" == jdoc['displayName'] + expected_value = {'check': { + 'type': 'boolean', 'default': 'false', 'value': 'false', 'description': 'A Boolean check'} + } + assert expected_value == jdoc['value'] + + def test_get_category_item(self, foglamp_url): + conn = http.client.HTTPConnection(foglamp_url) + encoded_url = '/foglamp/category/{}/check'.format(quote('pub #1')) + conn.request("GET", encoded_url) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 'boolean' == jdoc['type'] + assert 'A Boolean check' == jdoc['description'] + assert 'false' == jdoc['value'] + + def test_set_configuration_item(self, foglamp_url): + conn = http.client.HTTPConnection(foglamp_url) + encoded_url = '/foglamp/category/{}/check'.format(quote('pub #1')) + conn.request("PUT", encoded_url, body=json.dumps({"value": "true"})) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 'boolean' == jdoc['type'] + + conn.request("GET", encoded_url) + r = conn.getresponse() + assert 200 == r.status + r = r.read().decode() + jdoc = json.loads(r) + assert 'boolean' == jdoc['type'] + assert 'true' == jdoc['value'] + assert 'false' == jdoc['default'] + + def test_update_configuration_item_bulk(self, foglamp_url): + pass + + def test_add_configuration_item(self, foglamp_url): + pass + + def test_delete_configuration_item_value(self, foglamp_url): + pass + + def test_get_child_category(self, foglamp_url): + pass + + def test_create_child_category(self, foglamp_url): + pass + + def test_delete_child_category(self, foglamp_url): + pass + + def test_delete_parent_category(self, foglamp_url): + pass + + def test_upload_script(self, foglamp_url): + pass From bd8dd8c70da82114e063ead57ba1d8bb1766b2d3 Mon Sep 17 00:00:00 2001 From: Praveen Garg Date: Thu, 7 Mar 2019 16:06:46 +0530 Subject: [PATCH 83/86] skipped OCS test, as OCS service is not available --- tests/system/python/e2e/test_e2e_coap_OCS.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system/python/e2e/test_e2e_coap_OCS.py b/tests/system/python/e2e/test_e2e_coap_OCS.py index f137ddb6dd..dab69e5daf 100644 --- a/tests/system/python/e2e/test_e2e_coap_OCS.py +++ b/tests/system/python/e2e/test_e2e_coap_OCS.py @@ -153,6 +153,7 @@ def _read_data_from_ocs(ocs_client_id, ocs_client_secret, ocs_tenant, ocs_namesp return _read_data_from_ocs +@pytest.mark.skip(reason="OCS is currently disabled!") class TestE2EOCS: def test_end_to_end(self, start_south_north, read_data_from_ocs, foglamp_url, wait_time, retries, ocs_client_id, ocs_client_secret, ocs_tenant, ocs_namespace, asset_name="endToEndCoAP"): From 767f4a02174adfdf8c62d248bd9e005994763abe Mon Sep 17 00:00:00 2001 From: pintomax Date: Fri, 8 Mar 2019 12:19:10 +0100 Subject: [PATCH 84/86] FOGL-2306: Separate sending process configuration into advanced and basic configuration blocks (#1447) FOGL-2306: Separate sending process configuration into advanced and basic configuration blocks blockSize, duration, sleepInterval, memoryBufferSize configuration items, used by sending process only, are now part of $categoryNameAdvanced --- C/tasks/north/sending_process/sending.cpp | 79 +++++++++++++++-------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/C/tasks/north/sending_process/sending.cpp b/C/tasks/north/sending_process/sending.cpp index 0c068c156a..e5d2d96749 100644 --- a/C/tasks/north/sending_process/sending.cpp +++ b/C/tasks/north/sending_process/sending.cpp @@ -70,28 +70,36 @@ static const string sendingDefaultConfig = "\"enable\": {" "\"description\": \"A switch that can be used to enable or disable execution of " "the sending process.\", \"type\": \"boolean\", \"default\": \"true\" , \"readonly\": \"true\" }," - "\"duration\": {" - "\"description\": \"How long the sending process should run (in seconds) before stopping.\", " - "\"type\": \"integer\", \"default\": \"60\" , \"order\": \"7\", \"displayName\" : \"Duration\" }, " - "\"blockSize\": {" - "\"description\": \"The size of a block of readings to send in each transmission.\", " - "\"type\": \"integer\", \"default\": \"500\", \"order\": \"8\", \"displayName\" : \"Readings Block Size\" }, " - "\"sleepInterval\": {" - "\"description\": \"A period of time, expressed in seconds, " - "to wait between attempts to send readings when there are no " - "readings to be sent.\", \"type\": \"integer\", \"default\": \"1\", \"order\": \"11\", \"displayName\" : \"Sleep Interval\" }, " "\"streamId\": {" "\"description\": \"Identifies the specific stream to handle and the related information," " among them the ID of the last object streamed.\", " "\"type\": \"integer\", \"default\": \"0\", " - "\"readonly\": \"true\" }, " - "\"memoryBufferSize\": {" - "\"description\": \"Number of elements of blockSize size to be buffered in memory\"," - "\"type\": \"integer\", " - "\"default\": \"10\", " - "\"order\": \"12\", \"displayName\" : \"Memory Buffer Size\" ," - "\"readonly\": \"false\" " - "} " + "\"readonly\": \"true\" } " + "}"; + +// Sending process advanced configuration +static const string sendingAdvancedConfig = + "{" \ + "\"duration\": {" \ + "\"description\": \"How long the sending process " \ + "should run (in seconds) before stopping.\", " \ + "\"type\": \"integer\", \"default\": \"60\" , " \ + "\"order\": \"7\", \"displayName\" : \"Duration\" }, " \ + "\"blockSize\": {" \ + "\"description\": \"The size of a block of readings to send " \ + "in each transmission.\", " \ + "\"type\": \"integer\", \"default\": \"500\", \"order\": \"8\", " \ + "\"displayName\" : \"Readings Block Size\" }, " \ + "\"sleepInterval\": {" \ + "\"description\": \"A period of time, expressed in seconds, " \ + "to wait between attempts to send readings when there are no " \ + "readings to be sent.\", \"type\": \"integer\", \"default\": \"1\", " \ + "\"order\": \"11\", \"displayName\" : \"Sleep Interval\" }, " \ + "\"memoryBufferSize\": {" \ + "\"description\": \"Number of elements of blockSize size to be buffered in memory\", " \ + "\"type\": \"integer\", \"default\": \"10\", " \ + "\"order\": \"12\", \"displayName\" : \"Memory Buffer Size\" ," \ + "\"readonly\": \"false\" } " \ "}"; volatile std::sig_atomic_t signalReceived = 0; @@ -773,6 +781,7 @@ ConfigCategory SendingProcess::fetchConfiguration(const std::string& defaultConf #endif ConfigCategory configuration; + ConfigCategory advancedConfiguration; try { // Create category, with "default" values only DefaultConfigCategory category(categoryName, @@ -791,19 +800,36 @@ ConfigCategory SendingProcess::fetchConfiguration(const std::string& defaultConf } // Create/Update hierarchical configuration categories - createConfigCategories(category, PARENT_CONFIGURATION_KEY, categoryName, CONFIG_CATEGORY_DESCRIPTION); + createConfigCategories(category, + PARENT_CONFIGURATION_KEY, + categoryName, + CONFIG_CATEGORY_DESCRIPTION); + + // Create advanced configuration category + string advancedCatName = categoryName + string("Advanced"); + DefaultConfigCategory defConfigAdvanced(advancedCatName, + sendingAdvancedConfig); + // Set/Updaqte advanced configuration category + this->getManagementClient()->addCategory(defConfigAdvanced, true); + // Set advanced configuration category as child pf parent categoryName + vector children1; + children1.push_back(advancedCatName); + this->getManagementClient()->addChildCategories(categoryName, children1); // Get the category with values and defaults configuration = this->getManagementClient()->getCategory(categoryName); + // Get the advanced category with values and defaults + advancedConfiguration = this->getManagementClient()->getCategory(advancedCatName); + /** - * Handle the sending process parameters here + * Handle the sending process parameters here: + * fetch the Advanced configuration */ - - string blockSize = configuration.getValue("blockSize"); - string duration = configuration.getValue("duration"); - string sleepInterval = configuration.getValue("sleepInterval"); - string memoryBufferSize = configuration.getValue("memoryBufferSize"); + string blockSize = advancedConfiguration.getValue("blockSize"); + string duration = advancedConfiguration.getValue("duration"); + string sleepInterval = advancedConfiguration.getValue("sleepInterval"); + string memoryBufferSize = advancedConfiguration.getValue("memoryBufferSize"); // Handles the case in which the stream_id is not defined // in the configuration and sets it to not defined (0) @@ -845,7 +871,8 @@ ConfigCategory SendingProcess::fetchConfiguration(const std::string& defaultConf m_data_source_t = ""; } - // Sets the m_memory_buffer_size = 1 in case of an invalid value from the configuration like for example "A432" + // Sets the m_memory_buffer_size = 1 in case of an invalid value + // from the configuration like for example "A432" m_memory_buffer_size = strtoul(memoryBufferSize.c_str(), NULL, 10); if (m_memory_buffer_size < 1) { From 805ccb9a2bedd4a0ffce74dac6556d00ad04d72a Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Tue, 12 Mar 2019 12:36:38 +0000 Subject: [PATCH 85/86] FOGL-2569 Fix uninitialised memory access --- C/common/config_category.cpp | 1 + C/common/include/config_category.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/C/common/config_category.cpp b/C/common/config_category.cpp index 9bb7c9b486..5e6469ebf4 100644 --- a/C/common/config_category.cpp +++ b/C/common/config_category.cpp @@ -797,6 +797,7 @@ ConfigCategory::CategoryItem::CategoryItem(const string& name, const Value& item) { m_name = name; + m_itemType = UnknownType; if (! item.IsObject()) { throw new ConfigMalformed(); diff --git a/C/common/include/config_category.h b/C/common/include/config_category.h index fee7bf94e8..7ba9031534 100644 --- a/C/common/include/config_category.h +++ b/C/common/include/config_category.h @@ -53,7 +53,7 @@ class ConfigCategories { class ConfigCategory { public: - enum ItemType { StringItem, EnumerationItem, JsonItem, BoolItem, NumberItem, DoubleItem, ScriptItem, CategoryType}; + enum ItemType { UnknownType, StringItem, EnumerationItem, JsonItem, BoolItem, NumberItem, DoubleItem, ScriptItem, CategoryType}; ConfigCategory(const std::string& name, const std::string& json); ConfigCategory() {}; From bb2b226cfbb7439a861847af1bcfe4fed605d979 Mon Sep 17 00:00:00 2001 From: dianomicbot Date: Tue, 12 Mar 2019 16:14:35 +0000 Subject: [PATCH 86/86] VERSION changed --- VERSION | 2 +- docs/91_version_history.rst | 7 +++++++ docs/92_downloads.rst | 18 +++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/VERSION b/VERSION index 90bd74bd9a..4861170611 100755 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -foglamp_version=1.5.0 +foglamp_version=1.5.1 foglamp_schema=26 diff --git a/docs/91_version_history.rst b/docs/91_version_history.rst index 414cf0f737..00f7369a65 100644 --- a/docs/91_version_history.rst +++ b/docs/91_version_history.rst @@ -25,6 +25,13 @@ Version History FogLAMP v1 ========== +v1.5.1 +------- + +Release Date: 2019-03-12 + + + v1.5.0 ------- diff --git a/docs/92_downloads.rst b/docs/92_downloads.rst index b5eaa9928d..c25837f578 100644 --- a/docs/92_downloads.rst +++ b/docs/92_downloads.rst @@ -27,17 +27,17 @@ https://github.com/foglamp/storage-postgres -.. |intel 1.5.0 Ubuntu 16.04| raw:: html +.. |intel 1.5.1 Ubuntu 16.04| raw:: html - v1.5.0 Ubuntu 16.04 + v1.5.1 Ubuntu 16.04 -.. |intel 1.5.0 Ubuntu 18.04| raw:: html +.. |intel 1.5.1 Ubuntu 18.04| raw:: html - v1.5.0 Ubuntu 18.04 + v1.5.1 Ubuntu 18.04 -.. |arm 1.5.0| raw:: html +.. |arm 1.5.1| raw:: html - v1.5.0 ARM + v1.5.1 ARM @@ -54,14 +54,14 @@ We have created Debian for Intel and ARM architectures. The packages have been t FogLAMP Debian Packages for Intel --------------------------------- -- |intel 1.5.0 Ubuntu 16.04| -- |intel 1.5.0 Ubuntu 18.04| +- |intel 1.5.1 Ubuntu 16.04| +- |intel 1.5.1 Ubuntu 18.04| FogLAMP Debian Packages for ARM ------------------------------- -- |arm 1.5.0| +- |arm 1.5.1| Download/Clone from GitHub