From 119666751fe7b1d8fd189a47e8fc231e31998deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Mon, 24 Sep 2018 19:00:58 +0200 Subject: [PATCH 1/6] Add first statistics version and implement multiple api method call protection --- debian/control | 12 +- .../nymea-remoteproxy-statistics.install.in | 1 + .../jsonrpc/authenticationhandler.cpp | 3 +- libnymea-remoteproxy/jsonrpcserver.cpp | 13 +- libnymea-remoteproxy/proxyclient.cpp | 10 ++ libnymea-remoteproxy/proxyclient.h | 3 + .../nymea-remoteproxy-statistics | 151 ++++++++++++++++++ nymea-remoteproxy.pri | 2 +- .../nymea-remoteproxy-tests-offline.cpp | 58 ++++++- .../nymea-remoteproxy-tests-offline.h | 3 + 10 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 debian/nymea-remoteproxy-statistics.install.in create mode 100755 nymea-remoteproxy-statistics/nymea-remoteproxy-statistics diff --git a/debian/control b/debian/control index 1afd2c2..8f90ba4 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: nymea-remoteproxy Section: utils Priority: options Maintainer: Simon Stürz -Build-depends: debhelper (>= 0.0.0), +Build-depends: debhelper (>= 9.0.0), dh-systemd, libqt5websockets5-dev, libncurses5-dev, @@ -94,3 +94,13 @@ Depends: ${shlibs:Depends}, libncurses5, Description: The nymea remote proxy monitor tool The nymea remote proxy server tests + +Package: nymea-remoteproxy-statistics +Architecture: any +Depends: ${shlibs:Depends}, + ${misc:Depends}, + python, + python-gnuplot, +Description: The nymea remote proxy statistic util + The nymea remote proxy server statistic tool for generating + server statistic reports. diff --git a/debian/nymea-remoteproxy-statistics.install.in b/debian/nymea-remoteproxy-statistics.install.in new file mode 100644 index 0000000..5802abb --- /dev/null +++ b/debian/nymea-remoteproxy-statistics.install.in @@ -0,0 +1 @@ +usr/bin/nymea-remoteproxy-statistics diff --git a/libnymea-remoteproxy/jsonrpc/authenticationhandler.cpp b/libnymea-remoteproxy/jsonrpc/authenticationhandler.cpp index 75ab875..3b46aac 100644 --- a/libnymea-remoteproxy/jsonrpc/authenticationhandler.cpp +++ b/libnymea-remoteproxy/jsonrpc/authenticationhandler.cpp @@ -39,7 +39,8 @@ AuthenticationHandler::AuthenticationHandler(QObject *parent) : "id the other tunnel client can understand. Once the authentication was successfull, you " "can wait for the RemoteProxy.TunnelEstablished notification. If you send any data before " "getting this notification, the server will close the connection. If the tunnel client does " - "not show up within 10 seconds, the server will close the connection."); + "not show up within 10 seconds, the server will close the connection. This method can only be " + "called once, otherwise the connection will be killed."); params.insert("uuid", JsonTypes::basicTypeToString(JsonTypes::String)); params.insert("name", JsonTypes::basicTypeToString(JsonTypes::String)); params.insert("token", JsonTypes::basicTypeToString(JsonTypes::String)); diff --git a/libnymea-remoteproxy/jsonrpcserver.cpp b/libnymea-remoteproxy/jsonrpcserver.cpp index eccca76..7fb85f1 100644 --- a/libnymea-remoteproxy/jsonrpcserver.cpp +++ b/libnymea-remoteproxy/jsonrpcserver.cpp @@ -38,7 +38,8 @@ JsonRpcServer::JsonRpcServer(QObject *parent) : params.clear(); returns.clear(); setDescription("Hello", "Once connected to this server, a client can get information about the server by saying Hello. " - "The response informs the client about this proxy server."); + "The response informs the client about this proxy server. This method can only be called once, " + "otherwise the connection will be killed."); setParams("Hello", params); returns.insert("server", JsonTypes::basicTypeToString(JsonTypes::String)); returns.insert("name", JsonTypes::basicTypeToString(JsonTypes::String)); @@ -47,7 +48,8 @@ JsonRpcServer::JsonRpcServer(QObject *parent) : setReturns("Hello", returns); params.clear(); returns.clear(); - setDescription("Introspect", "Introspect this API."); + setDescription("Introspect", "Introspect this API. This method can only be called once, " + "otherwise the connection will be killed."); setParams("Introspect", params); returns.insert("methods", JsonTypes::basicTypeToString(JsonTypes::Object)); returns.insert("types", JsonTypes::basicTypeToString(JsonTypes::Object)); @@ -291,6 +293,13 @@ void JsonRpcServer::processData(ProxyClient *proxyClient, const QByteArray &data return; } + // Verfiy if this method was already called by this client + if (!proxyClient->validateMethodCall(message.value("method").toString())) { + sendErrorResponse(proxyClient, commandId, "Multiple method call not allowed. The method" + message.value("method").toString() + "has already been called by this client."); + proxyClient->killConnection("Multiple method call."); + return; + } + JsonReply *reply; QMetaObject::invokeMethod(handler, method.toLatin1().data(), Q_RETURN_ARG(JsonReply*, reply), Q_ARG(QVariantMap, params), Q_ARG(ProxyClient *, proxyClient)); if (reply->type() == JsonReply::TypeAsync) { diff --git a/libnymea-remoteproxy/proxyclient.cpp b/libnymea-remoteproxy/proxyclient.cpp index ed9c854..1b628cb 100644 --- a/libnymea-remoteproxy/proxyclient.cpp +++ b/libnymea-remoteproxy/proxyclient.cpp @@ -179,6 +179,16 @@ void ProxyClient::killConnection(const QString &reason) m_interface->killClientConnection(m_clientId, reason); } +bool ProxyClient::validateMethodCall(const QString &method) +{ + // Note: each method is allowed only once. If the method was already called, return false + if (m_calledMethods.contains(method)) + return false; + + m_calledMethods.append(method); + return true; +} + QDebug operator<<(QDebug debug, ProxyClient *proxyClient) { debug.nospace() << "ProxyClient("; diff --git a/libnymea-remoteproxy/proxyclient.h b/libnymea-remoteproxy/proxyclient.h index d09783e..4d2691d 100644 --- a/libnymea-remoteproxy/proxyclient.h +++ b/libnymea-remoteproxy/proxyclient.h @@ -78,6 +78,7 @@ class ProxyClient : public QObject // Actions for this client void sendData(const QByteArray &data); void killConnection(const QString &reason); + bool validateMethodCall(const QString &method); private: TransportInterface *m_interface = nullptr; @@ -100,6 +101,8 @@ class ProxyClient : public QObject quint64 m_rxDataCount = 0; quint64 m_txDataCount = 0; + QStringList m_calledMethods; + signals: void authenticated(); void tunnelConnected(); diff --git a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics new file mode 100755 index 0000000..f1a5a4a --- /dev/null +++ b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics @@ -0,0 +1,151 @@ +#!/usr/bin/env python + +# -*- coding: UTF-8 -*- + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# Copyright (C) 2018 Simon Stuerz # +# # +# This file is part of nymea-remoteproxy. # +# # +# nymea-remoteproxy is free software: you can redistribute it and/or modi # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, version 3 of the License. # +# # +# nymea-remoteproxy is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with nymea-remoteproxy. If not, see . # +# # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + +import sys +import os +import shutil +import traceback +import argparse +import Gnuplot + +__version__ = "0.1.0" + +#-------------------------------------------------------------------------------- +def prepareWorkingDirectory(workDirectory, logDirectory, daysCount): + print("Prepare working directory %s" % workDirectory) + + if os.path.isdir(workDirectory): + print("Clean up old working directory.") + shutil.rmtree(workDirectory) + + os.makedirs(workDirectory) + + tunnelLogFiles = [] + statisticsLogFiles = [] + + print("Reading files from %s" % logDirectory) + for fileName in os.listdir(logDirectory): + if "nymea-remoteproxy-tunnels" in fileName: + tunnelLogFiles.append(fileName) + + if "nymea-remoteproxy-statistics" in fileName: + statisticsLogFiles.append(fileName) + + # Write data file for processing + tunnelLogFiles = tunnelLogFiles[-(daysCount + 1):] + tunnelOutputFileName = workDirectory + "/tunnels.dat" + tunnelOutputFile = open(tunnelOutputFileName, "a") + + for fileName in sorted(tunnelLogFiles): + print("%s" % fileName) + inputFile = open(logDirectory + "/" + fileName) + for line in inputFile.readlines(): + tunnelOutputFile.write(line) + + tunnelOutputFile.close() + + statisticsLogFiles = statisticsLogFiles[-(daysCount + 1):] + statisticsOutputFileName = workDirectory + "/statistics.dat" + statisticsOutputFile = open(statisticsOutputFileName, "a") + + for fileName in sorted(statisticsLogFiles): + print("%s" % fileName) + inputFile = open(logDirectory + "/" + fileName) + for line in inputFile.readlines(): + statisticsOutputFile.write(line) + + statisticsOutputFile.close() + + +#-------------------------------------------------------------------------------- +def plotStatistics(workDirectory, imageWidth, imageHeight): + print("Create statistics plots into %s" % workDirectory) + + plot = Gnuplot.Gnuplot() + plot.title("Server connections") + plot.xlabel("Time") + plot.ylabel("Count") + + plot("set term png size %s, %s" % (imageWidth, imageHeight)) + plot("set output '%s'" % (workDirectory + "/statistics.png")) + plot("set xdata time") + plot("set timefmt '%s'") + plot("set xtics format '%d.%m.%y %H:%M:%S'") + plot("set yrange [0:*]") + plot("set grid") + plot("plot '%s' using 1:2 with lines title 'Tunnels'," + " '%s' using 1:3 with lines title 'Connections'" % (workDirectory + "/statistics.dat", workDirectory + "/statistics.dat")) + + +def sendReportMail(): + print("Sending report email") + + + +#-------------------------------------------------------------------------------- +# Main +#-------------------------------------------------------------------------------- + +if __name__ == "__main__": + + # Process arguments + parser = argparse.ArgumentParser(description='The nymea-remoteproxy-statistics tool allowes to process the log files and create statistic reports.') + parser.add_argument('-v','--version', action='version', version=__version__) + parser.add_argument('-l', '--logs', metavar='DIRECTORY', type=str, default='/var/log', help='the path to the log directoy of the nymea-remoteproxy server. Default is "/var/log".') + parser.add_argument('-d', '--days', metavar='DAYS', type=int, default='1', help='the amount of past days included into the report. Default is 1.') + args = parser.parse_args() + + # Variables + daysCount = args.days + logDirectory = os.path.abspath(args.logs) + workDirectory = os.path.abspath("/tmp/nymea-remoteproxy-statistics") + + print("Using log dir %s" % logDirectory) + print("Amount of days for report: %s" % daysCount) + + prepareWorkingDirectory(workDirectory, logDirectory, daysCount) + plotStatistics(workDirectory, 5000, 500) + + + + + + + + + + + + + + + + + + + + + + diff --git a/nymea-remoteproxy.pri b/nymea-remoteproxy.pri index 3bd922e..7b1dbeb 100644 --- a/nymea-remoteproxy.pri +++ b/nymea-remoteproxy.pri @@ -5,7 +5,7 @@ QT -= gui SERVER_NAME=nymea-remoteproxy API_VERSION_MAJOR=0 API_VERSION_MINOR=3 -SERVER_VERSION=0.1.7 +SERVER_VERSION=0.1.8 DEFINES += SERVER_NAME_STRING=\\\"$${SERVER_NAME}\\\" \ SERVER_VERSION_STRING=\\\"$${SERVER_VERSION}\\\" \ diff --git a/tests/test-offline/nymea-remoteproxy-tests-offline.cpp b/tests/test-offline/nymea-remoteproxy-tests-offline.cpp index 76a8d3e..6696b12 100644 --- a/tests/test-offline/nymea-remoteproxy-tests-offline.cpp +++ b/tests/test-offline/nymea-remoteproxy-tests-offline.cpp @@ -275,7 +275,7 @@ void RemoteProxyOfflineTests::websocketPing() qDebug() << pongSpy.at(0).at(0) << pongSpy.at(0).at(1); QVERIFY(pongSpy.at(0).at(1).toByteArray() == pingMessage); - + client->deleteLater(); // Clean up stopServer(); } @@ -352,7 +352,6 @@ void RemoteProxyOfflineTests::authenticate_data() QTest::addColumn("name"); QTest::addColumn("token"); QTest::addColumn("nonce"); - QTest::addColumn("timeout"); QTest::addColumn("expectedError"); @@ -550,6 +549,61 @@ void RemoteProxyOfflineTests::authenticateSendData() stopServer(); } +void RemoteProxyOfflineTests::multipleApiCall_data() +{ + QTest::addColumn("method"); + QTest::addColumn("params"); + + QVariantMap authParams; + authParams.insert("uuid", "uuid"); + authParams.insert("name", "name"); + authParams.insert("token", "token"); + authParams.insert("nonce", "nonce"); + + QTest::newRow("Hello") << "RemoteProxy.Hello" << QVariantMap(); + QTest::newRow("Introspect") << "RemoteProxy.Introspect" << QVariantMap(); + QTest::newRow("Authenticate") << "Authentication.Authenticate" << authParams; +} + +void RemoteProxyOfflineTests::multipleApiCall() +{ + QFETCH(QString, method); + QFETCH(QVariantMap, params); + + // Start the server + startServer(); + + // Make the method call + QWebSocket *client = new QWebSocket("bad-client"); + connect(client, &QWebSocket::sslErrors, this, &BaseTest::sslErrors); + QSignalSpy spyConnection(client, SIGNAL(connected())); + client->open(Engine::instance()->webSocketServer()->serverUrl()); + spyConnection.wait(); + + m_commandCounter++; + + QVariantMap request; + request.insert("id", m_commandCounter); + request.insert("method", method); + request.insert("params", params); + QJsonDocument jsonDoc = QJsonDocument::fromVariant(request); + + QSignalSpy dataSpy(client, SIGNAL(textMessageReceived(QString))); + client->sendTextMessage(QString(jsonDoc.toJson(QJsonDocument::Compact))); + dataSpy.wait(); + + // Call the same method a second time and make sure the client will be disconnected + QSignalSpy disconnectedSpy(client, SIGNAL(disconnected())); + dataSpy.clear(); + client->sendTextMessage(QString(jsonDoc.toJson(QJsonDocument::Compact))); + disconnectedSpy.wait(100); + QVERIFY(disconnectedSpy.count() == 1); + + // Clean up + client->deleteLater(); + stopServer(); +} + void RemoteProxyOfflineTests::clientConnection() { // Start the server diff --git a/tests/test-offline/nymea-remoteproxy-tests-offline.h b/tests/test-offline/nymea-remoteproxy-tests-offline.h index 28b0d2a..d77fc4a 100644 --- a/tests/test-offline/nymea-remoteproxy-tests-offline.h +++ b/tests/test-offline/nymea-remoteproxy-tests-offline.h @@ -61,6 +61,9 @@ private slots: void authenticateNonce(); void authenticateSendData(); + void multipleApiCall_data(); + void multipleApiCall(); + // Client lib void clientConnection(); void remoteConnection(); From edfd01d425d5de32f236275a69e907b28a004bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Thu, 18 Oct 2018 13:52:05 +0200 Subject: [PATCH 2/6] Add sha3 256 hash for user name --- debian/control | 5 +- libnymea-remoteproxy/logengine.cpp | 3 +- libnymea-remoteproxy/proxyclient.cpp | 1 + .../nymea-remoteproxy-statistics | 114 ++++++++++++------ 4 files changed, 81 insertions(+), 42 deletions(-) diff --git a/debian/control b/debian/control index 8f90ba4..a9bc19c 100644 --- a/debian/control +++ b/debian/control @@ -12,9 +12,10 @@ Package: nymea-remoteproxy Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, - awscli, libnymea-remoteproxy (= ${binary:Version}), -Suggests: nymea-remoteproxy-monitor (= ${binary:Version}) +Recommends: nymea-remoteproxy-monitor (= ${binary:Version}), + nymea-remoteproxy-statistics (= ${binary:Version}) +Suggests: nymea-remoteproxy-client (= ${binary:Version}) Description: The nymea remote proxy server The nymea remote proxy server diff --git a/libnymea-remoteproxy/logengine.cpp b/libnymea-remoteproxy/logengine.cpp index 8a3dfdd..a866e4b 100644 --- a/libnymea-remoteproxy/logengine.cpp +++ b/libnymea-remoteproxy/logengine.cpp @@ -23,6 +23,7 @@ #include "loggingcategories.h" #include +#include namespace remoteproxy { @@ -48,7 +49,7 @@ void LogEngine::logTunnel(const TunnelConnection &tunnel) QStringList logString; logString << createTimestamp(); logString << QString::number(tunnel.creationTime()); - logString << tunnel.clientOne()->userName(); + logString << QString::fromUtf8(QCryptographicHash::hash(tunnel.clientOne()->userName().toUtf8(), QCryptographicHash::Sha3_256)); logString << tunnel.clientOne()->peerAddress().toString(); logString << tunnel.clientTwo()->peerAddress().toString(); logString << QString::number(tunnel.clientOne()->rxDataCount() + tunnel.clientOne()->txDataCount()); diff --git a/libnymea-remoteproxy/proxyclient.cpp b/libnymea-remoteproxy/proxyclient.cpp index 1b628cb..4380b01 100644 --- a/libnymea-remoteproxy/proxyclient.cpp +++ b/libnymea-remoteproxy/proxyclient.cpp @@ -195,6 +195,7 @@ QDebug operator<<(QDebug debug, ProxyClient *proxyClient) if (!proxyClient->name().isEmpty()) { debug.nospace() << proxyClient->name() << ", "; } + debug.nospace() << proxyClient->interface()->serverName(); debug.nospace() << ", " << proxyClient->clientId().toString(); debug.nospace() << ", " << proxyClient->userName(); diff --git a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics index f1a5a4a..e3dc4af 100755 --- a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics +++ b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics @@ -29,11 +29,32 @@ import shutil import traceback import argparse import Gnuplot +import time +import datetime +import tzlocal +import pytz -__version__ = "0.1.0" +__version__ = "0.1.1" #-------------------------------------------------------------------------------- -def prepareWorkingDirectory(workDirectory, logDirectory, daysCount): +def convertLineToTimezone(line, timezoneString): + if timezone is "UTC": + return line + + lineTokens = line.split(' ') + print("Line tokens", lineTokens) + timestamp = lineTokens[0] + utcTime = datetime.datetime.fromtimestamp(int(timestamp)) + print("Convert timestamp", timestamp, utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") , "to timezone", timezoneString) + + timezoneObject = pytz.timezone(timezoneString) + timezoneOffset = timezoneObject.utcoffset() + convertedTimestamp = timezoneObject.localize(utcTime) + print("Result", timezoneOffset, utcTime.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)"), "-->", convertedTimestamp.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)")) + + +#-------------------------------------------------------------------------------- +def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): print("Prepare working directory %s" % workDirectory) if os.path.isdir(workDirectory): @@ -41,44 +62,72 @@ def prepareWorkingDirectory(workDirectory, logDirectory, daysCount): shutil.rmtree(workDirectory) os.makedirs(workDirectory) - + tunnelLogFiles = [] statisticsLogFiles = [] + # Read the file list print("Reading files from %s" % logDirectory) for fileName in os.listdir(logDirectory): if "nymea-remoteproxy-tunnels" in fileName: tunnelLogFiles.append(fileName) - + if "nymea-remoteproxy-statistics" in fileName: statisticsLogFiles.append(fileName) - + + # Sort file lists for beeing able to get the last n days logs + tunnelLogFiles.sort() + statisticsLogFiles.sort() + # Write data file for processing tunnelLogFiles = tunnelLogFiles[-(daysCount + 1):] tunnelOutputFileName = workDirectory + "/tunnels.dat" tunnelOutputFile = open(tunnelOutputFileName, "a") - + for fileName in sorted(tunnelLogFiles): print("%s" % fileName) inputFile = open(logDirectory + "/" + fileName) for line in inputFile.readlines(): tunnelOutputFile.write(line) - + tunnelOutputFile.close() statisticsLogFiles = statisticsLogFiles[-(daysCount + 1):] statisticsOutputFileName = workDirectory + "/statistics.dat" statisticsOutputFile = open(statisticsOutputFileName, "a") - + for fileName in sorted(statisticsLogFiles): print("%s" % fileName) inputFile = open(logDirectory + "/" + fileName) for line in inputFile.readlines(): statisticsOutputFile.write(line) - + statisticsOutputFile.close() +#-------------------------------------------------------------------------------- +def createStatistics(workDirectory): + print("Preparing statistics for different plots") + + statisticsFile = open(workDirectory + "/statistics.dat", "r") + tunnelsFile = open(workDirectory + "/tunnels.dat", "r") + + hourlyTunnelsList = [] + userList = [] + + + # Init lists + for x in range(0, 23): + hourlyTunnelsList.append(0) + + print("Initialized hour statistics", hourlyTunnelsList) + + # Walk troug lines and calculate statistics + #for line in statisticsFile: + + + + #-------------------------------------------------------------------------------- def plotStatistics(workDirectory, imageWidth, imageHeight): print("Create statistics plots into %s" % workDirectory) @@ -87,7 +136,7 @@ def plotStatistics(workDirectory, imageWidth, imageHeight): plot.title("Server connections") plot.xlabel("Time") plot.ylabel("Count") - + plot("set term png size %s, %s" % (imageWidth, imageHeight)) plot("set output '%s'" % (workDirectory + "/statistics.png")) plot("set xdata time") @@ -97,15 +146,15 @@ def plotStatistics(workDirectory, imageWidth, imageHeight): plot("set grid") plot("plot '%s' using 1:2 with lines title 'Tunnels'," " '%s' using 1:3 with lines title 'Connections'" % (workDirectory + "/statistics.dat", workDirectory + "/statistics.dat")) - + def sendReportMail(): print("Sending report email") - + #-------------------------------------------------------------------------------- -# Main +# Main #-------------------------------------------------------------------------------- if __name__ == "__main__": @@ -113,39 +162,26 @@ if __name__ == "__main__": # Process arguments parser = argparse.ArgumentParser(description='The nymea-remoteproxy-statistics tool allowes to process the log files and create statistic reports.') parser.add_argument('-v','--version', action='version', version=__version__) - parser.add_argument('-l', '--logs', metavar='DIRECTORY', type=str, default='/var/log', help='the path to the log directoy of the nymea-remoteproxy server. Default is "/var/log".') + parser.add_argument('-l', '--logs', metavar='DIR', type=str, default='/var/log', help='the path to the log directoy of the nymea-remoteproxy server. Default is "/var/log".') parser.add_argument('-d', '--days', metavar='DAYS', type=int, default='1', help='the amount of past days included into the report. Default is 1.') + parser.add_argument('-t', '--timezone', metavar='ZONE', type=str, default='UTC', help='the timezone the statistics should be created for. Default is UTC.') args = parser.parse_args() - - # Variables + + # Variables daysCount = args.days logDirectory = os.path.abspath(args.logs) workDirectory = os.path.abspath("/tmp/nymea-remoteproxy-statistics") + timezone = args.timezone print("Using log dir %s" % logDirectory) + print("Using working dir %s" % workDirectory) print("Amount of days for report: %s" % daysCount) + print("Creating statistics for timezone: %s" % timezone) - prepareWorkingDirectory(workDirectory, logDirectory, daysCount) - plotStatistics(workDirectory, 5000, 500) - - - - - - - - - - - - - - - - - - - - - + convertLineToTimezone("1538870830 1538870830 jenkins@guh.io 37.61.202.234 37.61.202.234 13", "Europe/Vienna") + prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone) + createStatistics(workDirectory) + plotStatistics(workDirectory, 5000, 500) + + From 2384e3f843c9bbba68347af2f679e121f9e322ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Fri, 19 Oct 2018 15:24:41 +0200 Subject: [PATCH 3/6] Fix emitting twice the disconnected signal and cover it with tests: Fixes #2 --- libnymea-remoteproxy/websocketserver.cpp | 3 ++- .../remoteproxyconnection.cpp | 5 ++-- .../websocketconnection.cpp | 1 - nymea-remoteproxy.conf | 4 +-- .../nymea-remoteproxy-tests-offline.cpp | 25 ++++++++++--------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libnymea-remoteproxy/websocketserver.cpp b/libnymea-remoteproxy/websocketserver.cpp index e0f94a3..0d15f02 100644 --- a/libnymea-remoteproxy/websocketserver.cpp +++ b/libnymea-remoteproxy/websocketserver.cpp @@ -114,8 +114,9 @@ void WebSocketServer::onClientConnected() void WebSocketServer::onClientDisconnected() { QWebSocket *client = static_cast(sender()); - QUuid clientId = m_clientList.key(client); + client->abort(); + QUuid clientId = m_clientList.key(client); qCDebug(dcWebSocketServer()) << "Client disconnected:" << client << client->peerAddress().toString() << clientId.toString() << client->closeReason(); m_clientList.take(clientId)->deleteLater(); diff --git a/libnymea-remoteproxyclient/remoteproxyconnection.cpp b/libnymea-remoteproxyclient/remoteproxyconnection.cpp index 0d3ba40..0233e2f 100644 --- a/libnymea-remoteproxyclient/remoteproxyconnection.cpp +++ b/libnymea-remoteproxyclient/remoteproxyconnection.cpp @@ -184,14 +184,11 @@ void RemoteProxyConnection::onConnectionChanged(bool isConnected) { if (isConnected) { qCDebug(dcRemoteProxyClientConnection()) << "Connected to proxy server."; - setState(StateConnected); - setState(StateInitializing); JsonReply *reply = m_jsonClient->callHello(); connect(reply, &JsonReply::finished, this, &RemoteProxyConnection::onHelloFinished); } else { qCDebug(dcRemoteProxyClientConnection()) << "Disconnected from proxy server."; - setState(StateDisconnected); cleanUp(); } } @@ -270,6 +267,8 @@ void RemoteProxyConnection::onHelloFinished() m_proxyServerVersion = responseParams.value("version").toString(); m_proxyServerApiVersion = responseParams.value("apiVersion").toString(); + qCDebug(dcRemoteProxyClientConnection()) << "Connected to" << m_serverName << m_proxyServerName << m_proxyServerVersion << "API:" << m_proxyServerApiVersion; + setState(StateReady); } diff --git a/libnymea-remoteproxyclient/websocketconnection.cpp b/libnymea-remoteproxyclient/websocketconnection.cpp index b0f2455..8c1ccf8 100644 --- a/libnymea-remoteproxyclient/websocketconnection.cpp +++ b/libnymea-remoteproxyclient/websocketconnection.cpp @@ -85,7 +85,6 @@ void WebSocketConnection::onStateChanged(QAbstractSocket::SocketState state) setConnected(true); break; default: - setConnected(false); break; } diff --git a/nymea-remoteproxy.conf b/nymea-remoteproxy.conf index 9266c60..0030368 100644 --- a/nymea-remoteproxy.conf +++ b/nymea-remoteproxy.conf @@ -2,7 +2,7 @@ name=nymea-remoteproxy writeLogs=false logFile=/var/log/nymea-remoteproxy.log -logEngineEnabled=false +logEngineEnabled=true monitorSocket=/tmp/nymea-remoteproxy-monitor.sock jsonRpcTimeout=10000 authenticationTimeout=8000 @@ -20,7 +20,7 @@ certificateKey=/etc/ssl/private/ssl-cert-snakeoil.key certificateChain= [WebSocketServer] -host=127.0.0.1 +host=0.0.0.0 port=443 [TcpServer] diff --git a/tests/test-offline/nymea-remoteproxy-tests-offline.cpp b/tests/test-offline/nymea-remoteproxy-tests-offline.cpp index 6696b12..08e9068 100644 --- a/tests/test-offline/nymea-remoteproxy-tests-offline.cpp +++ b/tests/test-offline/nymea-remoteproxy-tests-offline.cpp @@ -166,9 +166,10 @@ void RemoteProxyOfflineTests::monitorServer() // TODO: verify monitor data - QSignalSpy disconnectedSpy(monitor, &QLocalSocket::connected); + QSignalSpy disconnectedSpy(monitor, &QLocalSocket::disconnected); monitor->disconnectFromServer(); disconnectedSpy.wait(200); + QVERIFY(disconnectedSpy.count() == 1); // Clean up monitor->deleteLater(); @@ -247,9 +248,9 @@ void RemoteProxyOfflineTests::websocketBinaryData() // Send binary data and make sure the server disconnects this socket QSignalSpy spyDisconnected(client, SIGNAL(disconnected())); - client->sendBinaryMessage("trying to upload stuff...stuff...more stuff... other stuff"); - spyConnection.wait(200); - QVERIFY(spyConnection.count() == 1); + client->sendBinaryMessage("Trying to upload stuff...stuff...more stuff... other stuff"); + spyDisconnected.wait(200); + QVERIFY(spyDisconnected.count() == 1); // Clean up stopServer(); @@ -596,7 +597,7 @@ void RemoteProxyOfflineTests::multipleApiCall() QSignalSpy disconnectedSpy(client, SIGNAL(disconnected())); dataSpy.clear(); client->sendTextMessage(QString(jsonDoc.toJson(QJsonDocument::Compact))); - disconnectedSpy.wait(100); + disconnectedSpy.wait(); QVERIFY(disconnectedSpy.count() == 1); // Clean up @@ -645,9 +646,9 @@ void RemoteProxyOfflineTests::clientConnection() QSignalSpy spyDisconnected(connection, &RemoteProxyConnection::disconnected); connection->disconnectServer(); // FIXME: check why it waits the full time here - spyDisconnected.wait(500); + spyDisconnected.wait(200); - QVERIFY(spyDisconnected.count() >= 1); + QVERIFY(spyDisconnected.count() == 1); QVERIFY(!connection->isConnected()); connection->deleteLater(); @@ -823,8 +824,8 @@ void RemoteProxyOfflineTests::trippleConnection() connectionOneDisconnectedSpy.wait(200); connectionTwoDisconnectedSpy.wait(200); - QVERIFY(connectionOneDisconnectedSpy.count() >= 1); - QVERIFY(connectionTwoDisconnectedSpy.count() >= 1); + QVERIFY(connectionOneDisconnectedSpy.count() == 1); + QVERIFY(connectionTwoDisconnectedSpy.count() == 1); QVERIFY(connectionOne->state() == RemoteProxyConnection::StateDisconnected); QVERIFY(connectionTwo->state() == RemoteProxyConnection::StateDisconnected); @@ -882,10 +883,10 @@ void RemoteProxyOfflineTests::duplicateUuid() QVERIFY(connectionTwo->authenticate(m_testToken)); disconnectSpyOne.wait(200); - QVERIFY(disconnectSpyOne.count() >= 1); + QVERIFY(disconnectSpyOne.count() == 1); disconnectSpyTwo.wait(200); - QVERIFY(disconnectSpyTwo.count() >= 1); + QVERIFY(disconnectSpyTwo.count() == 1); connectionOne->deleteLater(); connectionTwo->deleteLater(); @@ -966,7 +967,7 @@ void RemoteProxyOfflineTests::inactiveTimeout() // Now wait for disconnected connectionDisconnectedSpy.wait(); - QVERIFY(connectionDisconnectedSpy.count() >= 1); + QVERIFY(connectionDisconnectedSpy.count() == 1); // Clean up stopServer(); From 9a41b347f7e16cbed26ec39a6a3e5a1d73d08e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Fri, 19 Oct 2018 17:41:51 +0200 Subject: [PATCH 4/6] Fix user logs hash string and add LegEngine debug category --- libnymea-remoteproxy/logengine.cpp | 6 +++++- libnymea-remoteproxy/loggingcategories.cpp | 1 + libnymea-remoteproxy/loggingcategories.h | 1 + server/main.cpp | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/libnymea-remoteproxy/logengine.cpp b/libnymea-remoteproxy/logengine.cpp index a866e4b..29e4d12 100644 --- a/libnymea-remoteproxy/logengine.cpp +++ b/libnymea-remoteproxy/logengine.cpp @@ -49,11 +49,13 @@ void LogEngine::logTunnel(const TunnelConnection &tunnel) QStringList logString; logString << createTimestamp(); logString << QString::number(tunnel.creationTime()); - logString << QString::fromUtf8(QCryptographicHash::hash(tunnel.clientOne()->userName().toUtf8(), QCryptographicHash::Sha3_256)); + logString << QString::fromUtf8(QCryptographicHash::hash(tunnel.clientOne()->userName().toLatin1(), QCryptographicHash::Sha3_256).toHex()); logString << tunnel.clientOne()->peerAddress().toString(); logString << tunnel.clientTwo()->peerAddress().toString(); logString << QString::number(tunnel.clientOne()->rxDataCount() + tunnel.clientOne()->txDataCount()); + qCDebug(dcLogEngine()) << "Logging tunnel" << logString; + QTextStream textStream(&m_tunnelsFile); textStream << logString.join(" ") << endl; } @@ -70,6 +72,8 @@ void LogEngine::logStatistics(int tunnelCount, int connectionCount, int troughpu logString << QString::number(connectionCount); logString << QString::number(troughput); + qCDebug(dcLogEngine()) << "Logging statisitcs" << logString; + QTextStream textStream(&m_statisticsFile); textStream << logString.join(" ") << endl; diff --git a/libnymea-remoteproxy/loggingcategories.cpp b/libnymea-remoteproxy/loggingcategories.cpp index ab5ce1e..d870439 100644 --- a/libnymea-remoteproxy/loggingcategories.cpp +++ b/libnymea-remoteproxy/loggingcategories.cpp @@ -23,6 +23,7 @@ Q_LOGGING_CATEGORY(dcApplication, "Application") Q_LOGGING_CATEGORY(dcEngine, "Engine") +Q_LOGGING_CATEGORY(dcLogEngine, "LogEngine") Q_LOGGING_CATEGORY(dcJsonRpc, "JsonRpc") Q_LOGGING_CATEGORY(dcJsonRpcTraffic, "JsonRpcTraffic") Q_LOGGING_CATEGORY(dcWebSocketServer, "WebSocketServer") diff --git a/libnymea-remoteproxy/loggingcategories.h b/libnymea-remoteproxy/loggingcategories.h index 4149807..3432bcf 100644 --- a/libnymea-remoteproxy/loggingcategories.h +++ b/libnymea-remoteproxy/loggingcategories.h @@ -27,6 +27,7 @@ Q_DECLARE_LOGGING_CATEGORY(dcApplication) Q_DECLARE_LOGGING_CATEGORY(dcEngine) +Q_DECLARE_LOGGING_CATEGORY(dcLogEngine) Q_DECLARE_LOGGING_CATEGORY(dcJsonRpc) Q_DECLARE_LOGGING_CATEGORY(dcJsonRpcTraffic) Q_DECLARE_LOGGING_CATEGORY(dcWebSocketServer) diff --git a/server/main.cpp b/server/main.cpp index 7243623..a62a9fb 100644 --- a/server/main.cpp +++ b/server/main.cpp @@ -123,6 +123,7 @@ int main(int argc, char *argv[]) s_loggingFilters.insert("AwsCredentialsProvider", true); // Only with verbose enabled + s_loggingFilters.insert("LogEngine", false); s_loggingFilters.insert("JsonRpcTraffic", false); s_loggingFilters.insert("ProxyServerTraffic", false); s_loggingFilters.insert("AuthenticationProcess", false); @@ -181,6 +182,7 @@ int main(int argc, char *argv[]) } if (parser.isSet(verboseOption)) { + s_loggingFilters["LogEngine"] = true; s_loggingFilters["JsonRpcTraffic"] = true; s_loggingFilters["ProxyServerTraffic"] = true; s_loggingFilters["AuthenticationProcess"] = true; From e46702e807512361bedafd0d147d34e9b28d93ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Tue, 6 Nov 2018 12:09:22 +0100 Subject: [PATCH 5/6] Update statistics tool --- .../config/nymea-remoteproxy.conf | 0 data/logrotate/nymea-remoteproxy | 10 +++ debian/control | 1 + .../nymea-remoteproxy-statistics.install.in | 2 +- debian/nymea-remoteproxy.install.in | 3 +- .../nymea-remoteproxy-statistics | 67 ++++++++++++------- 6 files changed, 58 insertions(+), 25 deletions(-) rename nymea-remoteproxy.conf => data/config/nymea-remoteproxy.conf (100%) create mode 100644 data/logrotate/nymea-remoteproxy diff --git a/nymea-remoteproxy.conf b/data/config/nymea-remoteproxy.conf similarity index 100% rename from nymea-remoteproxy.conf rename to data/config/nymea-remoteproxy.conf diff --git a/data/logrotate/nymea-remoteproxy b/data/logrotate/nymea-remoteproxy new file mode 100644 index 0000000..0e0c25a --- /dev/null +++ b/data/logrotate/nymea-remoteproxy @@ -0,0 +1,10 @@ +/var/log/nymea-remoteproxy.log { + rotate 14 + daily + dateext + compress + delaycompress + missingok + notifempty + create 755 root root +} diff --git a/debian/control b/debian/control index a9bc19c..4f0fecd 100644 --- a/debian/control +++ b/debian/control @@ -6,6 +6,7 @@ Build-depends: debhelper (>= 9.0.0), dh-systemd, libqt5websockets5-dev, libncurses5-dev, + lcov Standards-Version: 3.9.3 Package: nymea-remoteproxy diff --git a/debian/nymea-remoteproxy-statistics.install.in b/debian/nymea-remoteproxy-statistics.install.in index 5802abb..4e9cba6 100644 --- a/debian/nymea-remoteproxy-statistics.install.in +++ b/debian/nymea-remoteproxy-statistics.install.in @@ -1 +1 @@ -usr/bin/nymea-remoteproxy-statistics +nymea-remoteproxy-statistics/nymea-remoteproxy-statistics usr/bin diff --git a/debian/nymea-remoteproxy.install.in b/debian/nymea-remoteproxy.install.in index 888197e..bbb65e9 100644 --- a/debian/nymea-remoteproxy.install.in +++ b/debian/nymea-remoteproxy.install.in @@ -1,2 +1,3 @@ usr/bin/nymea-remoteproxy -nymea-remoteproxy.conf etc/nymea/ +data/config/nymea-remoteproxy.conf etc/nymea/ +data/logrotate/nymea-remoteproxy etc/logrotate.d/ diff --git a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics index e3dc4af..ba5318c 100755 --- a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics +++ b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics @@ -31,8 +31,6 @@ import argparse import Gnuplot import time import datetime -import tzlocal -import pytz __version__ = "0.1.1" @@ -40,18 +38,18 @@ __version__ = "0.1.1" def convertLineToTimezone(line, timezoneString): if timezone is "UTC": return line - + lineTokens = line.split(' ') print("Line tokens", lineTokens) timestamp = lineTokens[0] - utcTime = datetime.datetime.fromtimestamp(int(timestamp)) + utcTime = datetime.datetime.utcfromtimestamp(int(timestamp)) print("Convert timestamp", timestamp, utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") , "to timezone", timezoneString) timezoneObject = pytz.timezone(timezoneString) timezoneOffset = timezoneObject.utcoffset() convertedTimestamp = timezoneObject.localize(utcTime) print("Result", timezoneOffset, utcTime.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)"), "-->", convertedTimestamp.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)")) - + #-------------------------------------------------------------------------------- def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): @@ -107,26 +105,41 @@ def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): #-------------------------------------------------------------------------------- def createStatistics(workDirectory): - print("Preparing statistics for different plots") - + print("Preparing statistics for different plots") + statisticsFile = open(workDirectory + "/statistics.dat", "r") tunnelsFile = open(workDirectory + "/tunnels.dat", "r") - + hourlyTunnelsList = [] userList = [] - - + # Init lists - for x in range(0, 23): + for x in range(0, 24): hourlyTunnelsList.append(0) + + print("Initialized hour statistics %s" % hourlyTunnelsList) + + # Walk trough lines and calculate statistics + for line in tunnelsFile.readlines(): + lineTokens = line.split(' ') + # print("Line tokens: %s" % lineTokens) + tunnelCloseTime = int(lineTokens[0]) + tunnelCreationTime = int(lineTokens[1]) + tunnelUser = lineTokens[2] + tunnelTraffic = int(lineTokens[5]) + tunnelDuration = tunnelCloseTime - tunnelCreationTime; - print("Initialized hour statistics", hourlyTunnelsList) - - # Walk troug lines and calculate statistics - #for line in statisticsFile: + utcTime = datetime.datetime.utcfromtimestamp(int(tunnelCreationTime)) + creationTimeString = utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") + creationHour = int(utcTime.strftime("%H")) + hourlyTunnelsList[creationHour] = hourlyTunnelsList[creationHour] + 1 + + print("User: %s %s -> %s -> Duration: %s -> Traffic %s" % (tunnelUser, creationTimeString, creationHour, tunnelDuration, tunnelTraffic)) + - + print("Hourly statistics %s" % hourlyTunnelsList) + #-------------------------------------------------------------------------------- def plotStatistics(workDirectory, imageWidth, imageHeight): @@ -137,7 +150,7 @@ def plotStatistics(workDirectory, imageWidth, imageHeight): plot.xlabel("Time") plot.ylabel("Count") - plot("set term png size %s, %s" % (imageWidth, imageHeight)) + plot("set term png enhanced font 'Ubuntu,10' size %s, %s" % (imageWidth, imageHeight)) plot("set output '%s'" % (workDirectory + "/statistics.png")) plot("set xdata time") plot("set timefmt '%s'") @@ -164,24 +177,32 @@ if __name__ == "__main__": parser.add_argument('-v','--version', action='version', version=__version__) parser.add_argument('-l', '--logs', metavar='DIR', type=str, default='/var/log', help='the path to the log directoy of the nymea-remoteproxy server. Default is "/var/log".') parser.add_argument('-d', '--days', metavar='DAYS', type=int, default='1', help='the amount of past days included into the report. Default is 1.') - parser.add_argument('-t', '--timezone', metavar='ZONE', type=str, default='UTC', help='the timezone the statistics should be created for. Default is UTC.') + parser.add_argument('--width', metavar='WIDTH', type=int, default='5000', help='the width of the plot images. Default is 5000.') + parser.add_argument('--height', metavar='HEIGHT', type=int, default='500', help='the height of the plot images. Default is 500.') + # parser.add_argument('-t', '--timezone', metavar='ZONE', type=str, default='UTC', help='the timezone the statistics should be created for. Default is UTC.') args = parser.parse_args() # Variables daysCount = args.days + imageWidth = args.width + imageHeight = args.height logDirectory = os.path.abspath(args.logs) workDirectory = os.path.abspath("/tmp/nymea-remoteproxy-statistics") - timezone = args.timezone - + timezone = "UTC" + + print("-----------------------------------------------------------------------") print("Using log dir %s" % logDirectory) print("Using working dir %s" % workDirectory) print("Amount of days for report: %s" % daysCount) + print("Image size: %sx%s" % (imageWidth, imageHeight)) print("Creating statistics for timezone: %s" % timezone) + print("-----------------------------------------------------------------------") + + # TODO: convert the logs to a certain timezone is not supported yet + #convertLineToTimezone("1538870830 1538870830 jenkins@guh.io 37.61.202.234 37.61.202.234 13", "Europe/Vienna") - convertLineToTimezone("1538870830 1538870830 jenkins@guh.io 37.61.202.234 37.61.202.234 13", "Europe/Vienna") - prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone) createStatistics(workDirectory) - plotStatistics(workDirectory, 5000, 500) + plotStatistics(workDirectory, imageWidth, imageHeight) From d7112ad4c69b8d8c4513b1be2d7610f9e68421f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20St=C3=BCrz?= Date: Mon, 10 Dec 2018 11:47:57 +0100 Subject: [PATCH 6/6] Add hourly statistics and ignore jenkins parameter for statistics generation --- .../nymea-remoteproxy-statistics | 243 ++++++++++++++---- 1 file changed, 200 insertions(+), 43 deletions(-) diff --git a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics index ba5318c..0107d6f 100755 --- a/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics +++ b/nymea-remoteproxy-statistics/nymea-remoteproxy-statistics @@ -34,29 +34,85 @@ import datetime __version__ = "0.1.1" +# Report time string +reportStartTimeString = "" +reportEndTimeString = "" + +# Statitsitcs data +hourlyTunnelsList = [] +userList = [] + +# Count variables +totalUserCount = 0 +totalConnectionCount = 0 +totalTunnelTraffic = 0 +totalTunnelCount = 0 +totalTunnelDuration = 0 + + +#-------------------------------------------------------------------------- +def printInfo(info): + print('[+] ' + info) + + +#-------------------------------------------------------------------------- +def printWarning(warning): + print('[-] Warning: ' + warning) + + +#-------------------------------------------------------------------------- +def printError(error): + print('[!] Error: ' + error) + + +#-------------------------------------------------------------------------------- +def humanReadableSize(sizeBytes): + if sizeBytes == 1: + return "1 byte" + + suffixesTable = [('bytes',0),('KB',0),('MB',1),('GB',2),('TB',2), ('PB',2)] + + num = float(sizeBytes) + for suffix, precision in suffixesTable: + if num < 1024.0: + break + num /= 1024.0 + + if precision == 0: + formattedSize = "%d" % num + else: + formattedSize = str(round(num, ndigits=precision)) + + return "%s %s" % (formattedSize, suffix) + + #-------------------------------------------------------------------------------- def convertLineToTimezone(line, timezoneString): if timezone is "UTC": return line lineTokens = line.split(' ') - print("Line tokens", lineTokens) + printInfo("Line tokens", lineTokens) timestamp = lineTokens[0] utcTime = datetime.datetime.utcfromtimestamp(int(timestamp)) - print("Convert timestamp", timestamp, utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") , "to timezone", timezoneString) + printInfo("Convert timestamp", timestamp, utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") , "to timezone", timezoneString) timezoneObject = pytz.timezone(timezoneString) timezoneOffset = timezoneObject.utcoffset() convertedTimestamp = timezoneObject.localize(utcTime) - print("Result", timezoneOffset, utcTime.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)"), "-->", convertedTimestamp.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)")) + printInfo("Result", timezoneOffset, utcTime.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)"), "-->", convertedTimestamp.strftime("%Y-%m-%d %H:%M:%S.%f%z (%Z)")) #-------------------------------------------------------------------------------- -def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): - print("Prepare working directory %s" % workDirectory) +def prepareWorkingDirectory(): + + global reportStartTimeString + global reportEndTimeString + + printInfo("Prepare working directory %s" % workDirectory) if os.path.isdir(workDirectory): - print("Clean up old working directory.") + printInfo("Clean up old working directory.") shutil.rmtree(workDirectory) os.makedirs(workDirectory) @@ -65,7 +121,7 @@ def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): statisticsLogFiles = [] # Read the file list - print("Reading files from %s" % logDirectory) + printInfo("Reading files from %s" % logDirectory) for fileName in os.listdir(logDirectory): if "nymea-remoteproxy-tunnels" in fileName: tunnelLogFiles.append(fileName) @@ -78,46 +134,80 @@ def prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone): statisticsLogFiles.sort() # Write data file for processing - tunnelLogFiles = tunnelLogFiles[-(daysCount + 1):] - tunnelOutputFileName = workDirectory + "/tunnels.dat" + if not useAllLogs: + tunnelLogFiles = tunnelLogFiles[-(daysCount + 1):] + tunnelOutputFile = open(tunnelOutputFileName, "a") for fileName in sorted(tunnelLogFiles): - print("%s" % fileName) + printInfo("%s" % fileName) inputFile = open(logDirectory + "/" + fileName) for line in inputFile.readlines(): tunnelOutputFile.write(line) tunnelOutputFile.close() - statisticsLogFiles = statisticsLogFiles[-(daysCount + 1):] - statisticsOutputFileName = workDirectory + "/statistics.dat" + if not useAllLogs: + statisticsLogFiles = statisticsLogFiles[-(daysCount + 1):] + statisticsOutputFile = open(statisticsOutputFileName, "a") for fileName in sorted(statisticsLogFiles): - print("%s" % fileName) + printInfo("%s" % fileName) inputFile = open(logDirectory + "/" + fileName) for line in inputFile.readlines(): statisticsOutputFile.write(line) statisticsOutputFile.close() + # Get the start and entime for this report + statisticsOutputFile = open(statisticsOutputFileName, "r") + lineList = statisticsOutputFile.readlines() + statisticsOutputFile.close() + + firstLine = lineList[0] + lastLine = lineList[-1] + printInfo("First line '%s'" % firstLine) + lineTokens = firstLine.split(' ') + utcTime = datetime.datetime.utcfromtimestamp(int(lineTokens[0])) + reportStartTimeString = utcTime.strftime("%d.%m.%Y %H:%M:%S") + + printInfo("Last line '%s'" % lastLine) + lineTokens = lastLine.split(' ') + utcTime = datetime.datetime.utcfromtimestamp(int(lineTokens[0])) + reportEndTimeString = utcTime.strftime("%d.%m.%Y %H:%M:%S") + + printInfo("Statistics time window %s - %s" % (reportStartTimeString, reportEndTimeString)) + + #-------------------------------------------------------------------------------- -def createStatistics(workDirectory): - print("Preparing statistics for different plots") +def createStatistics(): + # Global variables + global reportStartTimeString + global reportEndTimeString + + global hourlyTunnelsList + global userList + + global totalUserCount + global totalConnectionCount + global totalTunnelTraffic + global totalTunnelCount + global totalTunnelDuration + + printInfo("Create statistics out of data") statisticsFile = open(workDirectory + "/statistics.dat", "r") tunnelsFile = open(workDirectory + "/tunnels.dat", "r") - hourlyTunnelsList = [] - userList = [] - # Init lists for x in range(0, 24): hourlyTunnelsList.append(0) - print("Initialized hour statistics %s" % hourlyTunnelsList) + printInfo("Initialized hour statistics %s" % hourlyTunnelsList) + + # # Walk trough lines and calculate statistics for line in tunnelsFile.readlines(): @@ -129,28 +219,78 @@ def createStatistics(workDirectory): tunnelTraffic = int(lineTokens[5]) tunnelDuration = tunnelCloseTime - tunnelCreationTime; + # Fixme: get hash for this user + if ignoreJenkins and tunnelUser == "jenkins@guh.io": + continue + + utcTime = datetime.datetime.utcfromtimestamp(int(tunnelCreationTime)) creationTimeString = utcTime.strftime("%Y.%m.%d %H:%M:%S (%Z)") creationHour = int(utcTime.strftime("%H")) hourlyTunnelsList[creationHour] = hourlyTunnelsList[creationHour] + 1 - print("User: %s %s -> %s -> Duration: %s -> Traffic %s" % (tunnelUser, creationTimeString, creationHour, tunnelDuration, tunnelTraffic)) + if not tunnelUser in userList: + userList.append(tunnelUser) + + totalTunnelCount += 1 + totalTunnelDuration += tunnelDuration + totalTunnelTraffic += tunnelTraffic + totalUserCount = len(userList) + + #printInfo("User: %s %s -> %s -> Duration: %s -> Traffic %s" % (tunnelUser, creationTimeString, creationHour, tunnelDuration, tunnelTraffic)) - print("Hourly statistics %s" % hourlyTunnelsList) + printInfo("-----------------------------------------------------------------------") + printInfo("Statistics time window %s - %s" % (reportStartTimeString, reportEndTimeString)) + printInfo("Hourly statistics: %s" % hourlyTunnelsList) + printInfo("User count: %s" % totalUserCount) + printInfo("Tunnel count: %s" % totalTunnelCount) + printInfo("Tunnel durations: %s [s]" % totalTunnelDuration) + printInfo("Tunnel traffic: %s [B] %s" % (totalTunnelTraffic, humanReadableSize(totalTunnelTraffic))) + printInfo("-----------------------------------------------------------------------") + + hourlyFile = open(workDirectory + "/hourly.dat", "a") + for x in range(0, 24): + line = "%s %s\n" % (x, hourlyTunnelsList[x]) + hourlyFile.write(line) + #-------------------------------------------------------------------------------- -def plotStatistics(workDirectory, imageWidth, imageHeight): - print("Create statistics plots into %s" % workDirectory) +def plotHourlyStatistics(): + global reportStartTimeString + global reportEndTimeString + + printInfo("Create houly plots into %s" % workDirectory) plot = Gnuplot.Gnuplot() - plot.title("Server connections") + plot.title("Hourly connections %s - %s" % (reportStartTimeString, reportEndTimeString)) plot.xlabel("Time") plot.ylabel("Count") - plot("set term png enhanced font 'Ubuntu,10' size %s, %s" % (imageWidth, imageHeight)) + plot("set term png enhanced font 'Ubuntu,12' size %s, %s" % (imageWidth, imageHeight)) + plot("set output '%s'" % (workDirectory + "/hourly.png")) + plot("set xrange [0:23]") + plot("set xtics 1") + plot("set yrange [0:*]") + plot("set grid") + plot("plot '%s' using 1:2 smooth cspline title 'Tunnels'" % (workDirectory + "/hourly.dat")) + + +#-------------------------------------------------------------------------------- +def plotStatistics(): + global reportStartTimeString + global reportEndTimeString + + printInfo("Create statistics plots into %s" % workDirectory) + + plot = Gnuplot.Gnuplot() + plot.title("Server connections %s - %s" % (reportStartTimeString, reportEndTimeString)) + plot.xlabel("Time") + plot.ylabel("Count") + + plot("set term png enhanced font 'Ubuntu,12' size %s, %s" % (imageWidth, imageHeight)) plot("set output '%s'" % (workDirectory + "/statistics.png")) plot("set xdata time") plot("set timefmt '%s'") @@ -162,8 +302,10 @@ def plotStatistics(workDirectory, imageWidth, imageHeight): def sendReportMail(): - print("Sending report email") - + printInfo("Sending report email") + + + #-------------------------------------------------------------------------------- @@ -174,11 +316,13 @@ if __name__ == "__main__": # Process arguments parser = argparse.ArgumentParser(description='The nymea-remoteproxy-statistics tool allowes to process the log files and create statistic reports.') + parser.add_argument('-a', '--all', action='store_true', help='create the statistic of all available log files. This parameter overrides the \"-d, --days\" paramter.') + parser.add_argument('-i', '--ignorejenkins', action='store_true', help='ignore the logs from the jenkins tests.') parser.add_argument('-v','--version', action='version', version=__version__) parser.add_argument('-l', '--logs', metavar='DIR', type=str, default='/var/log', help='the path to the log directoy of the nymea-remoteproxy server. Default is "/var/log".') parser.add_argument('-d', '--days', metavar='DAYS', type=int, default='1', help='the amount of past days included into the report. Default is 1.') - parser.add_argument('--width', metavar='WIDTH', type=int, default='5000', help='the width of the plot images. Default is 5000.') - parser.add_argument('--height', metavar='HEIGHT', type=int, default='500', help='the height of the plot images. Default is 500.') + parser.add_argument('--width', metavar='WIDTH', type=int, default='2000', help='the width of the plot images. Default is 2000.') + parser.add_argument('--height', metavar='HEIGHT', type=int, default='1000', help='the height of the plot images. Default is 1000.') # parser.add_argument('-t', '--timezone', metavar='ZONE', type=str, default='UTC', help='the timezone the statistics should be created for. Default is UTC.') args = parser.parse_args() @@ -187,22 +331,35 @@ if __name__ == "__main__": imageWidth = args.width imageHeight = args.height logDirectory = os.path.abspath(args.logs) + ignoreJenkins = args.ignorejenkins + useAllLogs = args.all workDirectory = os.path.abspath("/tmp/nymea-remoteproxy-statistics") timezone = "UTC" - print("-----------------------------------------------------------------------") - print("Using log dir %s" % logDirectory) - print("Using working dir %s" % workDirectory) - print("Amount of days for report: %s" % daysCount) - print("Image size: %sx%s" % (imageWidth, imageHeight)) - print("Creating statistics for timezone: %s" % timezone) - print("-----------------------------------------------------------------------") - - # TODO: convert the logs to a certain timezone is not supported yet - #convertLineToTimezone("1538870830 1538870830 jenkins@guh.io 37.61.202.234 37.61.202.234 13", "Europe/Vienna") - - prepareWorkingDirectory(workDirectory, logDirectory, daysCount, timezone) - createStatistics(workDirectory) - plotStatistics(workDirectory, imageWidth, imageHeight) + # Final logs for parsing + tunnelOutputFileName = workDirectory + "/tunnels.dat" + statisticsOutputFileName = workDirectory + "/statistics.dat" + printInfo("-----------------------------------------------------------------------") + printInfo("Using log dir %s" % logDirectory) + printInfo("Using working dir %s" % workDirectory) + + if useAllLogs: + printInfo("Using all available logs") + else: + printInfo("Amount of days for report: %s" % daysCount) + + + if ignoreJenkins: + printInfo("Ignore jenkins user from tests.") + else: + printInfo("Logs include connections from jenkins user from tests.") + + printInfo("Image size: %sx%s" % (imageWidth, imageHeight)) + printInfo("-----------------------------------------------------------------------") + + prepareWorkingDirectory() + createStatistics() + plotStatistics() + plotHourlyStatistics()