Merge branch 'train' into devel

Conflicts: .travis.yml makefile src/Testing/Downscaler.cpp
metno · Aug 19, 2015 · 57418ec · 57418ec
2 parents be9754b + b2b6301
commit 57418ec
Show file tree

Hide file tree

Showing 68 changed files with 1,739 additions and 333 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,8 @@
 *.o
 gridpp
 gridpp_debug
+gridpp_train
+gridpp_train_debug
 *.nc
 gmon.out
 tags

diff --git a/.travis.yml b/.travis.yml
@@ -9,6 +9,8 @@ before_install:
  - cp -a .pbuilderrc $HOME
  - sudo apt-get update -qq
  - sudo apt-get install pbuilder aptitude
+ - sudo apt-get install libgsl0-dev
+ - sudo apt-get install libblas-dev
  - sudo pbuilder create --debootstrapopts --variant=buildd
 
 install:

diff --git a/README.rst b/README.rst
@@ -36,7 +36,6 @@ Calibrators include:
 
 
 
-
 Installation Instructions
 -------------------------
 

diff --git a/debian/control b/debian/control
@@ -3,7 +3,7 @@ Maintainer: Thomas Nipen <[email protected]>
 Section: misc
 Priority: optional
 Standards-Version: 3.9.2
-Build-Depends: debhelper (>= 9.0.0), cmake, libnetcdf-dev (>= 4.1.1-6), libboost-dev, libgtest-dev
+Build-Depends: debhelper (>= 9.0.0), cmake, libnetcdf-dev (>= 4.1.1-6), libboost-dev, libgtest-dev, libgsl0-dev, libblas-dev
 Homepage: https://github.com/metno/gridpp
 Vcs-Git: [email protected]:metno/gridpp.git
 Vcs-Browser: https://github.com/metno/gridpp

diff --git a/makefile b/makefile
@@ -6,11 +6,11 @@ CFLAGS   = -Wall -Wno-reorder -Wno-sign-compare
 
 # Flags for optimized compilation
 CFLAGS_O = -O3 -fopenmp $(CFLAGS)
-LIBS_O   = -lnetcdf_c++
+LIBS_O   = -lnetcdf_c++ -lgsl -lblas
 
 # Flags for debug compilation
 CFLAGS_D = -g -pg -rdynamic -fprofile-arcs -ftest-coverage -coverage -DDEBUG $(CFLAGS)
-LIBS_D   = -lnetcdf_c++ -L build/gtest -lgtest -lpthread
+LIBS_D   = -lnetcdf_c++ -lgsl -lblas -L build/gtest -lgtest -lpthread
 
 # Don't change below here
 SRCDIR   = src/
@@ -34,6 +34,9 @@ DRVOBJ_D	= $(BUILDDIR_D)/Driver/Gpp.o
 KFSRC  	= src/Driver/Kf.cpp
 KFOBJ_O	= $(BUILDDIR_O)/Driver/Kf.o
 KFOBJ_D	= $(BUILDDIR_D)/Driver/Kf.o
+TRAINRC  = src/Driver/Train.cpp
+TRAINOBJ_O= $(BUILDDIR_O)/Driver/Train.o
+TRAINOBJ_D= $(BUILDDIR_D)/Driver/Train.o
 SRC     	= $(CORESRC) $(CALSRC) $(FILESRC) $(DOWNSRC) $(PARSRC)
 HEADERS 	= $(SRC:.cpp=.h)
 OBJ0_O   = $(patsubst src/%,$(BUILDDIR_O)/%,$(SRC))
@@ -69,12 +72,18 @@ gridpp: $(OBJ_O) $(DRVOBJ_O) makefile
 kalmanFilter: $(OBJ_O) $(KFOBJ_O) makefile
 	$(CC) $(CFLAGS_O) $(LFLAGS) $(OBJ_O) $(KFOBJ_O) $(LIBS_O) -o $@
 
+gridpp_train: $(OBJ_O) $(TRAINOBJ_O) makefile
+	$(CC) $(CFLAGS_O) $(LFLAGS) $(OBJ_O) $(TRAINOBJ_O) $(LIBS_O) -o $@
+
 gridpp_debug: $(OBJ_D) $(DRVOBJ_D) makefile gtest
 	$(CC) $(CFLAGS_D) $(LFLAGS) $(OBJ_D) $(DRVOBJ_D) $(LIBS_D) -o $@
 
 kalmanFilter_debug: $(OBJ_D) $(KFOBJ_D) makefile gtest
 	$(CC) $(CFLAGS_D) $(LFLAGS) $(OBJ_D) $(KFOBJ_D) $(LIBS_D) -o $@
 
+gridpp_train_debug: $(OBJ_D) $(TRAINOBJ_D) makefile gtest
+	$(CC) $(CFLAGS_D) $(LFLAGS) $(OBJ_D) $(TRAINOBJ_D) $(LIBS_D) -o $@
+
 test: gtest $(TESTS)
 	./runAllTests.sh
 

diff --git a/src/Calibrator/Accumulate.cpp b/src/Calibrator/Accumulate.cpp
@@ -2,15 +2,15 @@
 #include "../Util.h"
 #include "../File/File.h"
 CalibratorAccumulate::CalibratorAccumulate(Variable::Type iVariable, const Options& iOptions) :
-      Calibrator(NULL, iOptions),
+      Calibrator(iOptions),
       mInputVariable(iVariable),
       mOutputVariable(Variable::PrecipAcc) {
    std::string outputVariable;
    if(iOptions.getValue("outputVariable", outputVariable)) {
       mOutputVariable = Variable::getType(outputVariable);
    }
 }
-bool CalibratorAccumulate::calibrateCore(File& iFile) const {
+bool CalibratorAccumulate::calibrateCore(File& iFile, const ParameterFile* iParameterFile) const {
    int nLat = iFile.getNumLat();
    int nLon = iFile.getNumLon();
    int nEns = iFile.getNumEns();

diff --git a/src/Calibrator/Accumulate.h b/src/Calibrator/Accumulate.h
@@ -10,7 +10,7 @@ class CalibratorAccumulate : public Calibrator {
       static std::string description();
       std::string name() const {return "accumulate";};
    private:
-      bool calibrateCore(File& iFile) const;
+      bool calibrateCore(File& iFile, const ParameterFile* iParameterFile) const;
       Variable::Type mInputVariable;
       Variable::Type mOutputVariable;
 };

diff --git a/src/Calibrator/Calibrator.cpp b/src/Calibrator/Calibrator.cpp
@@ -7,17 +7,17 @@
 #include "../ParameterFile/ParameterFile.h"
 #include "../File/File.h"
 
-Calibrator::Calibrator(const ParameterFile* iParameterFile, const Options& iOptions) : Scheme(iOptions), mParameterFile(iParameterFile) {
+Calibrator::Calibrator(const Options& iOptions) : Scheme(iOptions) {
 
 }
-Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParameterFile, const Options& iOptions) {
+Calibrator* Calibrator::getScheme(std::string iName, const Options& iOptions) {
 
    if(iName == "zaga") {
       std::string variable;
       if(!iOptions.getValue("variable", variable)) {
          Util::error("Calibrator 'zaga' needs variable");
       }
-      CalibratorZaga* c = new CalibratorZaga(iParameterFile, Variable::getType(variable), iOptions);
+      CalibratorZaga* c = new CalibratorZaga(Variable::getType(variable), iOptions);
 
       return c;
    }
@@ -37,6 +37,14 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       CalibratorAccumulate* c = new CalibratorAccumulate(Variable::getType(variable), iOptions);
       return c;
    }
+   else if(iName == "gaussian") {
+      std::string variable;
+      if(!iOptions.getValue("variable", variable)) {
+         Util::error("Calibrator 'gaussian' needs variable");
+      }
+      CalibratorGaussian* c = new CalibratorGaussian(Variable::getType(variable), iOptions);
+      return c;
+   }
    else if(iName == "neighbourhood") {
       std::string variable;
       if(!iOptions.getValue("variable", variable)) {
@@ -46,7 +54,7 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       return c;
    }
    else if(iName == "phase") {
-      CalibratorPhase* c = new CalibratorPhase(iParameterFile, iOptions);
+      CalibratorPhase* c = new CalibratorPhase(iOptions);
       float minPrecip;
       if(iOptions.getValue("minPrecip", minPrecip)) {
          c->setMinPrecip(minPrecip);
@@ -63,7 +71,7 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       if(!iOptions.getValue("variable", variable)) {
          Util::error("Calibrator 'windDirection' needs variable");
       }
-      CalibratorWindDirection* c = new CalibratorWindDirection(iParameterFile, Variable::getType(variable), iOptions);
+      CalibratorWindDirection* c = new CalibratorWindDirection(Variable::getType(variable), iOptions);
 
       return c;
    }
@@ -72,7 +80,7 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       if(!iOptions.getValue("variable", variable)) {
          Util::error("Calibrator 'kriging' needs variable");
       }
-      CalibratorKriging* c = new CalibratorKriging(Variable::getType(variable), iParameterFile, iOptions);
+      CalibratorKriging* c = new CalibratorKriging(Variable::getType(variable), iOptions);
 
       return c;
    }
@@ -104,7 +112,7 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       if(!iOptions.getValue("variable", variable)) {
          Util::error("Calibrator 'regression' needs variable");
       }
-      CalibratorQq* c = new CalibratorQq(iParameterFile, Variable::getType(variable), iOptions);
+      CalibratorQq* c = new CalibratorQq(Variable::getType(variable), iOptions);
 
       return c;
    }
@@ -113,7 +121,7 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       if(!iOptions.getValue("variable", variable)) {
          Util::error("Calibrator 'regression' needs variable");
       }
-      CalibratorRegression* c = new CalibratorRegression(iParameterFile, Variable::getType(variable), iOptions);
+      CalibratorRegression* c = new CalibratorRegression(Variable::getType(variable), iOptions);
 
       return c;
    }
@@ -122,8 +130,8 @@ Calibrator* Calibrator::getScheme(std::string iName, const ParameterFile* iParam
       return NULL;
    }
 }
-bool Calibrator::calibrate(File& iFile) const {
-   return calibrateCore(iFile);
+bool Calibrator::calibrate(File& iFile, const ParameterFile* iParameterFile) const {
+   return calibrateCore(iFile, iParameterFile);
 }
 
 void Calibrator::shuffle(const std::vector<float>& iBefore, std::vector<float>& iAfter) {
@@ -153,6 +161,23 @@ void Calibrator::shuffle(const std::vector<float>& iBefore, std::vector<float>&
    }
 }
 
-const ParameterFile* Calibrator::getParameterFile() {
-   return mParameterFile;
+Parameters Calibrator::train(const TrainingData& iData, int iOffset) const {
+   Util::error("Cannot train method. Not yet implemented.");
+}
+
+std::string Calibrator::getDescriptions() {
+   std::stringstream ss;
+   ss << CalibratorZaga::description() << std::endl;
+   ss << CalibratorCloud::description() << std::endl;
+   ss << CalibratorQc::description() << std::endl;
+   ss << CalibratorQq::description() << std::endl;
+   ss << CalibratorQnh::description() << std::endl;
+   ss << CalibratorAccumulate::description() << std::endl;
+   ss << CalibratorWindDirection::description() << std::endl;
+   ss << CalibratorNeighbourhood::description() << std::endl;
+   ss << CalibratorPhase::description() << std::endl;
+   ss << CalibratorRegression::description() << std::endl;
+   ss << CalibratorKriging::description() << std::endl;
+   ss << CalibratorGaussian::description() << std::endl;
+   return ss.str();
 }
diff --git a/src/Calibrator/Calibrator.h b/src/Calibrator/Calibrator.h
@@ -3,6 +3,8 @@
 #include <string>
 #include <vector>
 #include "../Scheme.h"
+#include "../Parameters.h"
+#include "../TrainingData.h"
 class File;
 class Options;
 class ParameterFile;
@@ -11,14 +13,14 @@ class ParameterFile;
 class Calibrator : public Scheme {
    public:
       //! The calibrator does not free the memory of iParameterFile
-      Calibrator(const ParameterFile* iParameterFile, const Options& iOptions);
+      Calibrator(const Options& iOptions);
       virtual ~Calibrator() {};
       //! \brief Calibrate one or more fields in iFile
       //! @return true if calibration was successful, false otherwise
-      bool calibrate(File& iFile) const;
+      bool calibrate(File& iFile, const ParameterFile* iParameterFile=NULL) const;
 
       //! Instantiates a calibrator with name iName
-      static Calibrator* getScheme(std::string iName, const ParameterFile* iParameterFile, const Options& iOptions);
+      static Calibrator* getScheme(std::string iName, const Options& iOptions);
 
       //! \brief Ensure that values in iAfter are in the same order as in iBefore.
       //! If missing values are encountered in iBefore or iAfter, then iAfter is left unchanged.
@@ -28,19 +30,19 @@ class Calibrator : public Scheme {
       //! Returns the name of this calibrator
       virtual std::string name() const = 0;
 
-      //! Returns the parameter file given to this calibrator. Note that a NULL value may be
-      //! returned.
-      const ParameterFile* getParameterFile();
+      static std::string getDescriptions();
+
+      virtual Parameters train(const TrainingData& iData, int iOffset) const;
    protected:
-      virtual bool calibrateCore(File& iFile) const = 0;
-      const ParameterFile* mParameterFile;
+      virtual bool calibrateCore(File& iFile, const ParameterFile* iParameterFile) const = 0;
    private:
 };
 #include "Zaga.h"
 #include "Cloud.h"
 // #include "Wind.h"
 #include "Temperature.h"
 #include "Accumulate.h"
+#include "Gaussian.h"
 #include "Neighbourhood.h"
 #include "Qc.h"
 #include "Qnh.h"

diff --git a/src/Calibrator/Cloud.cpp b/src/Calibrator/Cloud.cpp
@@ -2,11 +2,11 @@
 #include "../Util.h"
 #include "../File/File.h"
 CalibratorCloud::CalibratorCloud(Variable::Type iVariable, const Options& iOptions) :
-      Calibrator(NULL, iOptions),
+      Calibrator(iOptions),
       mCloudType(iVariable),
       mPrecipType(Variable::Precip) {
 }
-bool CalibratorCloud::calibrateCore(File& iFile) const {
+bool CalibratorCloud::calibrateCore(File& iFile, const ParameterFile* iParameterFile) const {
    int nLat = iFile.getNumLat();
    int nLon = iFile.getNumLon();
    int nEns = iFile.getNumEns();

diff --git a/src/Calibrator/Cloud.h b/src/Calibrator/Cloud.h
@@ -10,7 +10,7 @@ class CalibratorCloud : public Calibrator {
       static std::string description();
       std::string name() const {return "cloud";};
    private:
-      bool calibrateCore(File& iFile) const;
+      bool calibrateCore(File& iFile, const ParameterFile* iParameterFile) const;
       Variable::Type mPrecipType;
       Variable::Type mCloudType;
 };
Original file line number	Diff line number	Diff line change
Expand Up		@@ -36,7 +36,6 @@ Calibrators include:




		Installation Instructions
		-------------------------

Expand Down