Skip to content

Commit

Permalink
Merged !58, Networking merge request new
Browse files Browse the repository at this point in the history
  • Loading branch information
klauck committed Apr 3, 2017
1 parent 576b445 commit 69d6cac
Show file tree
Hide file tree
Showing 34 changed files with 2,436 additions and 41 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,5 @@ benchmark.json
.idea
*.xcodeproj
*.xcworkspace
src/lib/network/generated
.vscode
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
[submodule "third_party/googletest"]
path = third_party/googletest
url = https://github.com/google/googletest.git
[submodule "third_party/grpc"]
path = third_party/grpc
url = https://github.com/grpc/grpc
[submodule "third_party/benchmark"]
path = third_party/benchmark
url = https://github.com/google/benchmark.git
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,25 @@ get via `git submodule update --init`
install via homebrew: brew install tbb
install via apt: apt-get install libtbb-dev

### development command line tools
install via `xcode-select --install` / `apt install build-essential`

### autoconf
install via homebrew / packet manager

### automake
install via homebrew / packet manager (installed as default by Ubuntu)

### libtool
install via homebrew / packet manager

### pkg-config
install via homebrew / packet manager (installed as default by Ubuntu)

### get and compile protoc and gRPC
get via `submodule update --init --recursive`
installation guide on github

### llvm (optional)
install via homebrew / packet manager
used for AddressSanitizer
Expand Down
22 changes: 16 additions & 6 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,14 @@ if echo $REPLY | grep -E '^[Yy]$' > /dev/null; then
echo "Installing dependencies (this may take a while)..."
if brew update >/dev/null; then
# python2.7 is preinstalled on macOS
if brew install premake boost gcc clang-format gcovr tbb; then
if git submodule update --init; then
echo "Installation successful"
if brew install premake boost gcc clang-format gcovr tbb autoconf automake libtool pkg-config; then
if git submodule update --init --recursive; then
if CPPFLAGS="-Wno-deprecated-declarations" CFLAGS="-Wno-deprecated-declarations -Wno-implicit-function-declaration -Wno-shift-negative-value" make static --directory=third_party/grpc REQUIRE_CUSTOM_LIBRARIES_opt=true; then
echo "Installation successful"
else
echo "Error during gRPC installation."
exit 1
fi
else
echo "Error during installation."
exit 1
Expand All @@ -32,9 +37,14 @@ if echo $REPLY | grep -E '^[Yy]$' > /dev/null; then
if cat /etc/lsb-release | grep DISTRIB_ID | grep Ubuntu >/dev/null; then
echo "Installing dependencies (this may take a while)..."
if sudo apt-get update >/dev/null; then
if sudo apt-get install -y premake4 libboost-all-dev clang-format gcovr python2.7 gcc-6 clang llvm libnuma-dev libnuma1 libtbb-dev; then
if git submodule update --init; then
echo "Installation successful."
if sudo apt-get install -y premake4 libboost-all-dev clang-format gcovr python2.7 gcc-6 clang llvm libnuma-dev libnuma1 libtbb-dev build-essential autoconf libtool; then
if git submodule update --init --recursive; then
if CPPFLAGS="-Wno-deprecated-declarations" CFLAGS="-Wno-deprecated-declarations -Wno-implicit-function-declaration -Wno-shift-negative-value" make static --directory=third_party/grpc REQUIRE_CUSTOM_LIBRARIES_opt=true; then
echo "Installation successful"
else
echo "Error during gRPC installation."
exit 1
fi
else
echo "Error during installation."
exit 1
Expand Down
55 changes: 42 additions & 13 deletions premake4.lua
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ else
end
end

-- Generate C++ source files from protobuf grammar files
os.execute("echo \"Generating protobuf and grpc files...\"")
os.execute("[ -d src/lib/network/generated ] || mkdir src/lib/network/generated")
os.execute("./third_party/grpc/bins/opt/protobuf/protoc --cpp_out=./src/lib/network/generated -I=\"./src/lib/network/protos/\" ./src/lib/network/protos/opossum.proto")
os.execute("./third_party/grpc/bins/opt/protobuf/protoc --grpc_out=./src/lib/network/generated --plugin=protoc-gen-grpc=./third_party/grpc/bins/opt/grpc_cpp_plugin -I=\"./src/lib/network/protos/\" ./src/lib/network/protos/opossum.proto")

solution "opossum"
configurations { "Debug", "Release" }
flags { "FatalWarnings", "ExtraWarnings" }
Expand Down Expand Up @@ -106,8 +112,8 @@ solution "opossum"

project "googletest"
kind "StaticLib"
files { "third_party/googletest/googletest/src/gtest-all.cc" }
includedirs { "third_party/googletest/googletest", "third_party/googletest/googletest/include" }
files { "third_party/googletest/googletest/src/gtest-all.cc" }

project "googlebenchmark"
kind "StaticLib"
Expand All @@ -120,25 +126,45 @@ project "googlebenchmark"

project "opossum"
kind "StaticLib"
files { "src/lib/**.hpp", "src/lib/**.cpp", "src/bin/server.cpp" }
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
files { "src/lib/**.hpp", "src/lib/**.cpp" }

project "opossum-asan"
kind "StaticLib"
buildoptions {"-fsanitize=address -fno-omit-frame-pointer"}
linkoptions {"-fsanitize=address"}
files { "src/lib/**.hpp", "src/lib/**.cpp", "src/bin/server.cpp" }
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
files { "src/lib/**.hpp", "src/lib/**.cpp", "src/bin/server_main.cpp" }

project "opossumCoverage"
kind "StaticLib"
buildoptions { "-fprofile-arcs -ftest-coverage" }
linkoptions { "-lgcov --coverage" }
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
files { "src/lib/**.hpp", "src/lib/**.cpp" }

-- Static lib for the opossum protobuf and grpc code generated from opossum.proto (see action 'protoc' below)
project "opossumProtobuf"
kind "StaticLib"
buildoptions ("-Wno-unused-parameter -Wno-deprecated-declarations")
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
files { "src/lib/network/generated/**.pb.cc" }

-- Exemplary opossum client, showing how to use grpc and protobuf at client-side
project "client"
kind "ConsoleApp"
links { "opossumProtobuf", "protobuf", "grpc++", "grpc", "z", "boost_program_options" }
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
libdirs { "third_party/grpc/libs/opt/", "third_party/grpc/libs/opt/protobuf" }
files { "src/bin/client.cpp" }

project "server"
kind "ConsoleApp"
links { "opossum" }
links { "opossum", "opossumProtobuf", "protobuf", "grpc++", "grpc", "z", "boost_program_options" } -- z is needed on macos to link grpc
includedirs { "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
libdirs { "third_party/grpc/libs/opt/", "third_party/grpc/libs/opt/protobuf" }
links(libs)
files { "src/bin/server.cpp" }
files { "src/bin/server_main.cpp" }

project "playground"
kind "ConsoleApp"
Expand All @@ -149,19 +175,21 @@ project "playground"
project "test"
kind "ConsoleApp"

links { "opossum", "googletest" }
links { "opossum", "googletest", "opossumProtobuf", "protobuf", "grpc++", "grpc", "z" }
includedirs { "third_party/googletest/googletest/include", "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
libdirs { "third_party/grpc/libs/opt/", "third_party/grpc/libs/opt/protobuf" }
links(libs)
files { "src/test/**.hpp", "src/test/**.cpp" }
includedirs { "third_party/googletest/googletest/include" }
postbuildcommands { "./build/test" }

project "asan"
kind "ConsoleApp"

links { "opossum-asan", "googletest" }
links { "opossum-asan", "googletest", "opossumProtobuf", "protobuf", "grpc++", "grpc", "z" }
links(libs)
files { "src/test/**.hpp", "src/test/**.cpp" }
includedirs { "third_party/googletest/googletest/include" }
includedirs { "third_party/googletest/googletest/include", "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
libdirs { "third_party/grpc/libs/opt/", "third_party/grpc/libs/opt/protobuf" }
buildoptions {"-fsanitize=address -fno-omit-frame-pointer"}
linkoptions { "-fsanitize=address" }
postbuildcommands { "./build/asan" }
Expand All @@ -177,13 +205,14 @@ project "benchmark"
project "coverage"
kind "ConsoleApp"

links { "opossumCoverage", "googletest" }
links { "opossumCoverage", "googletest", "opossumProtobuf", "protobuf", "grpc++", "grpc", "z" }
links(libs)
linkoptions {"--coverage"}
files { "src/test/**.hpp", "src/test/**.cpp" }
buildoptions { "-fprofile-arcs -ftest-coverage" }
includedirs { "third_party/googletest/googletest/include" }
postbuildcommands { "./build/coverage && rm -fr coverage; mkdir coverage && gcovr -s -r . --exclude=\"(.*types*.|.*test*.)\" --html --html-details -o coverage/index.html" }
includedirs { "third_party/googletest/googletest/include", "third_party/grpc/include/", "third_party/grpc/third_party/protobuf/src/" }
libdirs { "third_party/grpc/libs/opt/", "third_party/grpc/libs/opt/protobuf" }
postbuildcommands { "./build/coverage && rm -fr coverage; mkdir coverage && gcovr -s -r . --exclude=\"(.*types*.|.*test*.|.*\.pb\.|third_party)\" --html --html-details -o coverage/index.html" }

newoption {
trigger = "compiler",
Expand Down Expand Up @@ -229,7 +258,7 @@ function premake.generate(obj, filename, callback)
-- "make clean" should also call "premake4 clean"
os.execute("awk '\\\
/help:/ {\\\
print \"\tpremake4 clean\"\\\
print \"\tpremake4 clean\\n\trm -r src/lib/network/generated\"\\\
}\\\
{ print }' Makefile > Makefile.awk && mv Makefile.awk Makefile")
end
Expand Down
168 changes: 168 additions & 0 deletions src/bin/client.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#include "client.hpp"

#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/config.hpp>
#include <boost/program_options/environment_iterator.hpp>
#include <boost/program_options/eof_iterator.hpp>
#include <boost/program_options/errors.hpp>
#include <boost/program_options/option.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/positional_options.hpp>
#include <boost/program_options/value_semantic.hpp>
#include <boost/program_options/variables_map.hpp>
#include <boost/program_options/version.hpp>

#include <iomanip>
#include <iostream>
#include <memory>
#include <string>

namespace po = boost::program_options;

namespace opossum {

// Sample client showing how to use protobuf and grpc at client-side
// This client uses a synchronous, blocking grpc-call, for an async client sample, see network-tests
OpossumClient::OpossumClient(std::shared_ptr<Channel> channel) : _stub(proto::OpossumService::NewStub(channel)) {}

// Assembles the client's payload, sends it and presents the response back from the server.
void OpossumClient::query(std::string& table_name, std::string& column_name, std::string& filter_op,
std::string& filter) {
// Data we are sending to the server.
proto::Request request;

proto::GetTableOperator* get_table = nullptr;
auto root_op_variant = request.mutable_root_operator();

if (!column_name.empty() && !filter_op.empty() && !filter.empty()) {
// Init a TableScan (protobuf allocates and manages the needed resources)
proto::TableScanOperator* table_scan = root_op_variant->mutable_table_scan();
table_scan->set_column_name(column_name);
table_scan->set_filter_operator(filter_op);
proto::Variant* variant = table_scan->mutable_value();
variant->set_value_int(std::stoi(filter));
// Add a GetTable operator as input operator for TableScan
get_table = table_scan->mutable_input_operator()->mutable_get_table();
}

if (!get_table) {
// Init a GetTable operator if there is no TableScan already
get_table = root_op_variant->mutable_get_table();
}
get_table->set_table_name(table_name);

// Container for the data we expect from the server.
proto::Response response;

// Context for the client. It could be used to convey extra information to
// the server and/or tweak certain RPC behaviors.
ClientContext context;

// The actual RPC (synchronous).
Status status = _stub->Query(&context, request, &response);

// Act upon the status of the actual RPC.
if (status.ok()) {
print_response_table(response);
} else {
std::cout << "RPC failed" << std::endl;
}
}

void OpossumClient::print_variant(const proto::Variant& variant) const {
std::cout << "|" << std::setw(20);
switch (variant.variant_case()) {
case proto::Variant::kValueInt:
std::cout << variant.value_int();
break;
case proto::Variant::kValueFloat:
std::cout << variant.value_float();
break;
case proto::Variant::kValueString:
std::cout << variant.value_string();
break;
case proto::Variant::kValueDouble:
std::cout << variant.value_double();
break;
case proto::Variant::kValueLong:
std::cout << variant.value_long();
break;
default:
throw std::runtime_error("Unknown AllTypeVariant in operator_translator");
}
std::cout << std::setw(0);
}

void OpossumClient::print_response_table(proto::Response& response) const {
if (response.result_case() == proto::Response::kError) {
std::cout << "Error in request: " << response.error() << std::endl;
return;
}

const auto table = response.response_table();
std::cout << "=== Columns"
<< " === " << std::endl;
for (int i = 0; i < table.column_type_size(); ++i) {
std::cout << "|" << std::setw(20) << table.column_type(i) << std::setw(0);
}
std::cout << "|" << std::endl;
for (int i = 0; i < table.column_name_size(); ++i) {
std::cout << "|" << std::setw(20) << table.column_name(i) << std::setw(0);
}
std::cout << "|" << std::endl;
std::cout << "=== Values"
<< " === " << std::endl;
for (int row_id = 0; row_id < table.row_size(); ++row_id) {
auto row = table.row(row_id);
for (int i = 0; i < row.value_size(); ++i) {
auto variant = row.value(i);
print_variant(variant);
}
std::cout << "|" << std::endl;
}
}

} // namespace opossum

int main(int argc, char** argv) {
po::options_description desc("Allowed options");
auto options = desc.add_options();
options("help", "print help message");
options("table_name", po::value<std::string>(), "opossum table name (required)");
options("address", po::value<std::string>()->default_value("0.0.0.0:50051"), "IP:PORT");
options("column_name", po::value<std::string>()->default_value(""), "column name for table scan");
options("filter_op", po::value<std::string>()->default_value(""), "filter operation for table scan, e.g. = > >= ...");
options("filter_val", po::value<std::string>()->default_value(""), "filter value for table scan");

po::positional_options_description pd;
pd.add("table_name", 1);
pd.add("column_name", 1);
pd.add("filter_op", 1);
pd.add("filter_val", 1);

po::variables_map variables;
po::store(po::command_line_parser(argc, argv).options(desc).positional(pd).run(), variables);
po::notify(variables);

if (variables.count("help") || variables.count("table_name") == 0) {
std::cout << desc << std::endl;
return 1;
}

auto address = variables["address"].as<std::string>();
auto table_name = variables["table_name"].as<std::string>();
auto column_name = variables["column_name"].as<std::string>();
auto filter_op = variables["filter_op"].as<std::string>();
auto filter = variables["filter_val"].as<std::string>();

// Instantiate the client. It requires a channel, out of which the actual RPCs
// are created. This channel models a connection to an endpoint. We indicate that the channel isn't authenticated (use
// of InsecureChannelCredentials()).
opossum::OpossumClient client(grpc::CreateChannel(address, grpc::InsecureChannelCredentials()));

std::cout << "Sending query to " << address << std::endl;
client.query(table_name, column_name, filter_op, filter);

return 0;
}
Loading

0 comments on commit 69d6cac

Please sign in to comment.