Completes and isolates logical block feature

- Improved cohesion in logical_blocks::scheduler classes
- It add unit tests for the logical block scheduler.
- Logical blocks is not an optional feature enabled by using:

    ./configure --enable-lblocks

- Logical blocks code is compiled selectively (To avoid ZK dependency).
- Organized unit tests in independent binaries.
- Added automatic release script which will release the tag using travis
  after PR is accepted and merged to master. An example is on the last
  line of this commit message.

release: v1.8.3

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,9 @@
 This PR Fixes DICL/EclipseMETA#(Which issue?)
 
+
 ## BRIEF
 
+
 ## STATUS
 - [ ] Its implemented.
 - [ ] It compiles.
@@ -15,4 +17,5 @@ You might want to use the following command:
     $ git rebase -i #hash key of base commit     
 
 ---
-## EXTRA
+
+- This pull request will release: X.Y.Z

.github/make_release.sh

@@ -0,0 +1,7 @@
+release_no=$(grep -oP "release: \K(v\d*.\d*.\d*)" <(git log -1 --pretty=%B))
+
+if [ "$?" = "0" ]; then
+  cmd="git tag -a $release_no -m \"`git log -1 --pretty=%B`\"" 
+  echo "$cmd"
+  eval $cmd
+fi

.travis.yml

@@ -64,4 +64,6 @@ after_success: |
         git add .
         git commit -m "API: $TRAVIS_TAG"
         git push origin gh-pages
+        bash .github/make_release.sh
+        git push --tags
   fi

Makefile.am

@@ -60,14 +60,21 @@ eclipse_node_SOURCES = src/targets/node_main.cc \
                        src/blocknode/block_node_router.cc \
                        src/fileleader/directory.cc \
                        src/fileleader/file_leader.cc \
-                       src/fileleader/file_leader_router.cc \
-                       src/stats/io_monitor_invoker.cc \
+                       src/fileleader/file_leader_router.cc
+
+eclipse_node_LDADD   = $(LDADD) 
+
+if LOGICAL_BLOCKS_FEATURE 
+
+eclipse_node_SOURCES+= src/stats/io_monitor_invoker.cc \
                        src/stats/zk_listener.cc \
                        src/stats/scheduler_factory.cc \
                        src/stats/scheduler_score_based.cc \
                        src/stats/scheduler_simple.cc
 
-eclipse_node_LDADD   = $(LDADD) -lzookeeper_mt
+eclipse_node_LDADD  +=  -lzookeeper_mt
+
+endif
 
 veloxdfs_SOURCES     = src/targets/client.cc \
                        src/client/cli_driver.cc

configure.ac

@@ -21,15 +21,17 @@ AX_CXX_COMPILE_STDCXX([14], [noext], [mandatory])
 AC_ARG_ENABLE([samples],
     [AS_HELP_STRING([--disable-samples], [copy sample of eclipse.json])])
 
-# Configure options: --enable-debug[=no].
 AC_ARG_ENABLE([debug],
   [AS_HELP_STRING([--enable-debug], [enable debug code (default is no)])],
     [debug="$withval"], [debug=no])
 
+AC_ARG_ENABLE([lblocks],
+    [AS_HELP_STRING([--enable-lblocks], [enable logical blocks experimental feature (Needs zookeeper)])])
+
 AM_CONDITIONAL(COPY_SAMPLES, [test "$enable_samples" != no ])
+AM_CONDITIONAL(LOGICAL_BLOCKS_FEATURE, [test "x$enable_lblocks" = xyes])
 AX_BOOST_STATIC_LINKAGE
 
-
 #}}}
 # Dependencies checkings {{{
 # Checks for typedefs, structures, and compiler characteristics.
@@ -63,6 +65,11 @@ AC_DEFINE_UNQUOTED([ECLIPSE_CONF_PATH], ["$sysconfdirfull"])
 
 AH_TEMPLATE([IO_STAT_SCRIPT],[IOSTATS path])
 AC_DEFINE_UNQUOTED([IO_STAT_SCRIPT], ["${bindirfull}/read_io_stats.sh"])
+
+AS_IF([test "x$enable_lblocks" = xyes], [
+      AH_TEMPLATE([LOGICAL_BLOCKS_FEATURE],["Enables logical blocks feature" ]) 
+      AC_DEFINE_UNQUOTED([LOGICAL_BLOCKS_FEATURE], [1])
+     ])
 #}}}
 #OUTPUT{{{
 AC_OUTPUT

m4/AX_BOOST_STATIC_LINKAGE.m4

@@ -7,5 +7,5 @@ AC_DEFUN([AX_BOOST_STATIC_LINKAGE],
 AC_ARG_ENABLE([boost_static],
   [AS_HELP_STRING([--enable-boost-static], [statically link boost])])
 
-AM_CONDITIONAL(BOOST_STATIC, [test x$enable_boost_static = xyes])
+AM_CONDITIONAL(BOOST_STATIC, [test "x$enable_boost_static" = xyes])
 ])

src/client/cli_driver.cc

@@ -140,6 +140,12 @@ void cli_driver::file_show (std::string file) {
 // }}}
 // file_show_optimized {{{
 void cli_driver::file_show_optimized(std::string file) {
+
+#ifndef LOGICAL_BLOCKS_FEATURE
+  cout << "ERROR! LOGICAL BLOCKS FEATURE IS DISABLED IN THIS BUILD" << endl;
+  exit(EXIT_FAILURE);
+#endif
+
   vec_str nodes = GET_VEC_STR("network.nodes");
   Histogram boundaries(nodes.size(), 100);
   boundaries.initialize();

src/fileleader/file_leader.cc

@@ -3,7 +3,11 @@
 #include "../messages/boost_impl.hh"
 #include "../messages/filedescription.hh"
 #include "../common/logical_block_metadata.hh"
+
+#ifdef LOGICAL_BLOCKS_FEATURE
 #include "../stats/logical_blocks_scheduler.hh"
+#endif
+
 #include <set>
 
 using namespace eclipse;
@@ -186,11 +190,17 @@ bool FileLeader::format () {
 // }}}
 // find_best_arrangement {{{
 void FileLeader::find_best_arrangement(messages::FileDescription* file_desc) {
+#ifdef LOGICAL_BLOCKS_FEATURE
   using namespace eclipse::logical_blocks_schedulers;
   auto nodes = context.settings.get<vec_str>("network.nodes");
 
-  auto scheduler = scheduler_factory(GET_STR("addons.block_scheduler"), boundaries.get());
+  map<string, string> opts;
+  opts["alpha"] = GET_STR("addons.alpha");
+  opts["beta"]  = GET_STR("addons.beta");
+
+  auto scheduler = scheduler_factory(GET_STR("addons.block_scheduler"), boundaries.get(), opts);
 
   scheduler->generate(*file_desc, nodes);
+#endif
 }
 // }}}

src/stats/logical_blocks_scheduler.hh

@@ -7,9 +7,11 @@
 #include <vector>
 #include <string>
 #include <memory>
+#include <map>
 
 namespace eclipse {
 namespace logical_blocks_schedulers {
+typedef std::map<std::string, std::string> SCHEDULER_OPTS;
 
 class scheduler;
 
@@ -18,10 +20,12 @@ class scheduler {
     virtual void generate(messages::FileDescription& file_desc, std::vector<std::string> nodes) = 0;
 
     std::shared_ptr<stats_listener> listener;
+    std::map<std::string, std::string> options;
     Histogram* boundaries;
 };
 
-std::shared_ptr<eclipse::logical_blocks_schedulers::scheduler> scheduler_factory(std::string, Histogram*);
+std::shared_ptr<eclipse::logical_blocks_schedulers::scheduler> 
+    scheduler_factory(std::string, Histogram*, SCHEDULER_OPTS = SCHEDULER_OPTS());
 
 class scheduler_simple: public scheduler{
   public:

src/stats/scheduler_factory.cc

@@ -3,12 +3,14 @@
 #include "../common/context_singleton.hh"
 #include "zk_listener.hh"
 #include <memory>
+#include <map>
+
+using namespace std;
 
 namespace eclipse {
 namespace logical_blocks_schedulers {
 
-using namespace std;
-std::shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries) {
+shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries, SCHEDULER_OPTS options) {
   shared_ptr<scheduler> sch;
 
   if (type == "scheduler_simple") {
@@ -23,6 +25,7 @@ std::shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries)
 
   sch->boundaries = boundaries;
   sch->listener = make_shared<zk_listener>();
+  sch->options = options;
 
   return sch;
 }

src/stats/scheduler_score_based.cc

@@ -1,7 +1,7 @@
 #include "logical_blocks_scheduler.hh"
 #include "../messages/blockinfo.hh"
-#include "../common/context_singleton.hh"
 
+#include <algorithm>
 #include <set>
 
 using namespace std;
@@ -10,8 +10,8 @@ using namespace eclipse::messages;
 using namespace eclipse::logical_blocks_schedulers;
 
 void scheduler_score_based::generate(FileDescription& file_desc, std::vector<std::string> nodes) {
-  double alpha = atof(GET_STR("addons.alpha").c_str());
-  double beta  = atof(GET_STR("addons.beta").c_str());
+  double alpha = atof(options["alpha"].c_str());
+  double beta  = atof(options["beta"].c_str());
 
   auto io_vec = listener->get_io_stats();
 

src/stats/scheduler_simple.cc

@@ -1,6 +1,6 @@
 #include "logical_blocks_scheduler.hh"
 #include "../messages/blockinfo.hh"
-#include "../common/context_singleton.hh"
+#include <algorithm>
 
 #include <set>
 

src/targets/node_main.cc

@@ -4,7 +4,11 @@
 #include <network/simple_router.hh>
 #include <fileleader/file_leader_router.hh>
 #include <blocknode/block_node_router.hh>
+
+#ifdef LOGICAL_BLOCKS_FEATURE
 #include <src/stats/io_monitor_invoker.hh>
+#endif
+
 #include <memory>
 
 using namespace eclipse;
@@ -26,7 +30,10 @@ int main (int argc, char ** argv) {
 
   external_net->establish();
 
+#ifdef LOGICAL_BLOCKS_FEATURE
   invoke_io_reporter();
+#endif
+
   context.run();
   context.join();
 

tests/Makefile.am

@@ -1,36 +1,59 @@
 AM_CPPFLAGS_T    = $(AM_CPPFLAGS) -include unittest++/UnitTest++.h
-LDADD_T          = $(LDADD) -l:libUnitTest++.a -ldl -lsqlite3
-
-# Input your tests files here
-check_PROGRAMS   = units metadata 
-check_SCRIPTS    = integration
-
-TESTS            = units 
+LDADD_T          = $(LDADD) -l:libUnitTest++.a
 
 check_DATA       = $(top_srcdir)/doc/eclipse.json
 CLEANFILES       = eclipse.json
+check_PROGRAMS   = 
+TESTS            = 
+
+#<<<<<<<<<<<<<<<<Input your tests files here>>>>>>>>>>>>>>>>>>>>
+
+# INTEGRATION TEST
+# ---------------------------------------------------------------------
+check_SCRIPTS     = integration 
+check-integration: integration
+	bash integration.sh
+
+integration:
+	cp $(top_srcdir)/tests/integration_test.sh integration.sh
+	chmod +x integration.sh
 
 
-# Input the dependencies of the test files here
-units_LDADD        = $(LDADD_T)
-units_CPPFLAGS     = $(AM_CPPFLAGS_T)
-units_SOURCES      = $(messages_files) \
+# MESSAGES TEST
+# ---------------------------------------------------------------------
+check_PROGRAMS    += messages
+TESTS             += messages
+messages_LDADD     = $(LDADD_T) -ldl -lsqlite3
+messages_CPPFLAGS  = $(AM_CPPFLAGS_T)
+messages_SOURCES   = $(messages_files) \
                      tests/main.cc \
                      tests/messages_test.cc \
                      src/network/asyncchannel.cc \
                      src/network/server.cc \
                      src/nodes/machine.cc \
-                     src/nodes/node.cc 
+                     src/nodes/node.cc
 
-metadata_LDADD     = $(LDADD_T)
+# METADATA TEST
+# ---------------------------------------------------------------------
+check_PROGRAMS    += metadata 
+metadata_LDADD     = $(LDADD_T) -ldl -lsqlite3  
 metadata_CPPFLAGS  = $(AM_CPPFLAGS_T)
 metadata_SOURCES   = tests/metadata_test.cc \
                      src/fileleader/directory.cc \
                      $(messages_files)
 
-check-integration: integration
-	bash integration.sh
-
-integration:
-	cp $(top_srcdir)/tests/integration_test.sh integration.sh
-	chmod +x integration.sh
+# LOGICAL BLOCKS TEST
+# ---------------------------------------------------------------------
+if LOGICAL_BLOCKS_FEATURE
+check_PROGRAMS   += lblocks
+TESTS            += lblocks
+
+lblocks_LDADD     = $(LDADD_T) -lzookeeper_mt
+lblocks_CPPFLAGS  = $(AM_CPPFLAGS_T)
+lblocks_SOURCES   = tests/main.cc \
+                    tests/stats_scheduler_test.cc \
+                    src/stats/scheduler_factory.cc \
+                    src/stats/scheduler_score_based.cc \
+                    src/stats/scheduler_simple.cc \
+                    src/stats/zk_listener.cc
+endif

tests/stats_scheduler_test.cc

@@ -0,0 +1,69 @@
+#include <settings.hh>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <messages/filedescription.hh>
+#include <common/histogram.hh>
+#include <stats/logical_blocks_scheduler.hh>
+
+using namespace std;
+using namespace eclipse::logical_blocks_schedulers;
+using namespace eclipse::messages;
+
+namespace {
+
+struct listener_1_busy : public eclipse::stats_listener {
+  listener_1_busy() = default;
+  ~listener_1_busy() = default;
+
+  std::vector<double> get_io_stats() {
+    return {0.01, 0.01, 0.01, 0.99};
+  }
+};
+
+struct stats_fixture {
+  std::vector<std::string> nodes = {"1", "2", "3", "4"};
+  map<string, string> opts;
+  FileDescription fd;
+
+  stats_fixture () { 
+    opts["alpha"] = "0.5";
+    opts["beta"]  = "0.5";
+
+    fd.name = "file";
+    fd.blocks = {"file_1", "file_2", "file_3", "file_4", "file_5"};
+    fd.hash_keys = {123123, 124123, 32323, 4242, 424245};
+    fd.block_size = {123123, 124123, 32323, 4242, 424245};
+    fd.block_hosts= {"1", "2", "3", "4", "4"};
+  }
+};
+
+}
+
+SUITE(STATS_TESTS) {
+
+  TEST_FIXTURE(stats_fixture, simple_test) {
+    Histogram boundaries (4, 100);
+    auto scheduler = scheduler_factory("scheduler_simple", &boundaries, opts);
+    scheduler->listener.reset(new listener_1_busy());
+    scheduler->generate(fd, nodes);
+
+    CHECK_EQUAL(fd.logical_blocks.size(), 4);
+    CHECK_EQUAL(fd.logical_blocks[0].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[1].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[2].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[3].physical_blocks.size(), 2);
+  }
+
+  TEST_FIXTURE(stats_fixture, score_based_tests) {
+    Histogram boundaries (4, 100);
+    auto scheduler = scheduler_factory("scheduler_score_based", &boundaries, opts);
+    scheduler->listener.reset(new listener_1_busy());
+    scheduler->generate(fd, nodes);
+
+    CHECK_EQUAL(3, fd.logical_blocks.size());
+    CHECK_EQUAL(1, fd.logical_blocks[0].physical_blocks.size());
+    CHECK_EQUAL(2, fd.logical_blocks[1].physical_blocks.size());
+    CHECK_EQUAL(2, fd.logical_blocks[2].physical_blocks.size());
+  }
+}