Merge pull request #617 from bsura/transform_refactor

PERCENTILE, DOWNSAMPLE, MOVING Transform changes.

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/CullAboveValueMapping.java

@@ -34,7 +34,7 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.Maps;
 import com.salesforce.dva.argus.system.SystemAssert;
-import org.apache.commons.math.stat.descriptive.rank.Percentile;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/CullBelowValueMapping.java

@@ -34,7 +34,7 @@
 import com.google.common.base.Predicate;
 import com.google.common.collect.Maps;
 import com.salesforce.dva.argus.system.SystemAssert;
-import org.apache.commons.math.stat.descriptive.rank.Percentile;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/DeviationValueReducerOrMapping.java

@@ -33,7 +33,7 @@
 
 import com.salesforce.dva.argus.system.SystemAssert;
 import com.salesforce.dva.argus.system.SystemException;
-import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/DownsampleTransform.java

@@ -35,17 +35,15 @@
 import com.salesforce.dva.argus.entity.Metric;
 import com.salesforce.dva.argus.service.metric.MetricReader;
 import com.salesforce.dva.argus.system.SystemAssert;
-import org.apache.commons.math.stat.descriptive.moment.Mean;
-import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
-import org.apache.commons.math.stat.descriptive.summary.Sum;
+import org.apache.commons.math3.stat.descriptive.moment.Mean;
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
+import org.apache.commons.math3.stat.descriptive.summary.Sum;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.TreeMap;
 
 /**
@@ -78,6 +76,7 @@ public static Double downsamplerReducer(List<Double> values, String reducerType)
                 operands.add(value);
             }
         }
+
         InternalReducerType type = InternalReducerType.fromString(reducerType);
         switch (type) {
             case AVG:
@@ -93,8 +92,10 @@ public static Double downsamplerReducer(List<Double> values, String reducerType)
             case COUNT:
             	values.removeAll(Collections.singleton(null));
             	return (double) values.size();
+            case PERCENTILE:
+            	return new Percentile().evaluate(Doubles.toArray(operands), Double.parseDouble(reducerType.substring(1)));
             default:
-                throw new UnsupportedOperationException(reducerType);
+                throw new UnsupportedOperationException("Illegal type: " + reducerType + ". Please provide a valid type.");
         }
     }
 
@@ -126,10 +127,12 @@ public List<Metric> transform(List<Metric> metrics) {
 
     @Override
     public List<Metric> transform(List<Metric> metrics, List<String> constants) {
-        SystemAssert.requireArgument(metrics != null, "Cannot transform empty metric/metrics");
+        SystemAssert.requireArgument(metrics != null, "Cannot transform null metrics");
+
         if (metrics.isEmpty()) {
             return metrics;
         }
+
         SystemAssert.requireArgument(constants.size() == 1,
             "Downsampler Transform can only have exactly one constant which is downsampler expression");
         SystemAssert.requireArgument(constants.get(0).contains("-"), "This downsampler expression is not valid.");
@@ -142,11 +145,8 @@ public List<Metric> transform(List<Metric> metrics, List<String> constants) {
         String windowSizeStr = expArr[0];
         Long windowSize = getWindowInSeconds(windowSizeStr) * 1000;
         String windowUnit = windowSizeStr.substring(windowSizeStr.length() - 1);
-        // init downsample type
-        Set<String> typeSet = new HashSet<String>(Arrays.asList("avg", "min", "max", "sum", "dev", "count"));
         String downsampleType = expArr[1];
 
-        SystemAssert.requireArgument(typeSet.contains(downsampleType), "Please input a valid type.");
         for (Metric metric : metrics) {
             metric.setDatapoints(createDownsampleDatapoints(metric.getDatapoints(), windowSize, downsampleType, windowUnit));
         }

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/FillCalculateTransform.java

@@ -31,6 +31,7 @@
 
 package com.salesforce.dva.argus.service.metric.transform;
 
+import com.google.common.primitives.Doubles;
 import com.salesforce.dva.argus.entity.Metric;
 import com.salesforce.dva.argus.system.SystemAssert;
 import java.util.ArrayList;
@@ -40,6 +41,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
+
 /**
  * Creates a constant line based on the calculated value.<br/>
  * <tt>FILL_CALCULATE(<expr>, <interval>, <interval>, <constant>)</tt>
@@ -89,7 +92,7 @@ private static Double calculateResult(Metric metric, String calculationType) {
         if (matcher.matches()) {
             Integer target = Integer.valueOf(matcher.group(1));
 
-            result = PercentileTransform.calculateNthPercentile(valueList, target);
+            result = new Percentile().evaluate(Doubles.toArray(valueList), target);
         } else {
             switch (calculationType) {
                 case "min":

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/InternalReducerType.java

@@ -53,6 +53,7 @@ public enum InternalReducerType {
     DEVIATION("dev"),
     SUM("sum"),
     MEDIAN("median"),
+    PERCENTILE(""),
 	COUNT("count");
 
     //~ Instance fields ******************************************************************************************************************************
@@ -84,8 +85,13 @@ public static InternalReducerType fromString(String name) {
                     return type;
                 }
             }
+
+            if(name.matches("^p\\d{1,2}$")) {
+        		return InternalReducerType.PERCENTILE;
+        	}
         }
-        throw new IllegalArgumentException(name);
+
+        throw new IllegalArgumentException("Illegal type: " + name + ". Please provide a valid type.");
     }
 
     //~ Methods **************************************************************************************************************************************

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/MetricFilterWithInteralReducerTransform.java

@@ -34,8 +34,8 @@
 import com.google.common.primitives.Doubles;
 import com.salesforce.dva.argus.entity.Metric;
 import com.salesforce.dva.argus.system.SystemAssert;
-import org.apache.commons.math.stat.descriptive.moment.Mean;
-import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
+import org.apache.commons.math3.stat.descriptive.moment.Mean;
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;

ArgusCore/src/main/java/com/salesforce/dva/argus/service/metric/transform/MetricZipperTransform.java

@@ -50,6 +50,8 @@ public class MetricZipperTransform implements Transform {
 
     private final ValueZipper valueZipper;
     private final String defaultScope;
+    protected static String FULLJOIN = "UNION";
+    protected Boolean fulljoinIndicator = false;
 
     //~ Constructors *********************************************************************************************************************************
 
@@ -72,18 +74,23 @@ public String getResultScopeName() {
 
     @Override
     public List<Metric> transform(List<Metric> metrics, List<String> constants) {
-        SystemAssert.requireArgument(constants == null || constants.isEmpty(), "Zipper transform doesn't support constants!");
+        SystemAssert.requireArgument(constants != null && constants.size() == 1 && FULLJOIN.equals(constants.get(0).toUpperCase()), 
+        		"Zipper transforms only support UNION indicator as a constant!");
+
+        fulljoinIndicator = true;
         return transform(metrics);
     }
 
     @Override
     public List<Metric> transform(List<Metric> metrics) {
-        SystemAssert.requireArgument(metrics != null, "Cannot transform empty metric/metrics");
+        SystemAssert.requireArgument(metrics != null, "Cannot transform null metrics.");
         if (metrics.isEmpty()) {
             return metrics;
         }
+
         SystemAssert.requireArgument(metrics.size() >= 2 && metrics.get(metrics.size() - 1) != null,
             "Cannot transform without a base metric as second param!");
+
         return zip(metrics.subList(0, metrics.size() - 1), metrics.get(metrics.size() - 1));
     }
 
@@ -134,6 +141,19 @@ public Map<Long, Double> zip(Map<Long, Double> originalDatapoints, Map<Long, Dou
 
             zippedDP.put(originalKey, this.valueZipper.zip(originalVal, baseVal));
         }
+
+        // if a point exists in the baseDP but does not exist in the original set, 
+        // then only add it to the result when fullJoinIndicator is true.
+        if(fulljoinIndicator) {
+        	for (Map.Entry<Long, Double> baseDP : baseDatapoints.entrySet()) {
+                Long baseDPKey = baseDP.getKey();
+
+                if(!zippedDP.containsKey(baseDPKey)) {
+                    zippedDP.put(baseDPKey, this.valueZipper.zip(null, baseDP.getValue()));
+                }
+            }
+        }
+
         return zippedDP;
     }