Develop (#88)

* Argus metric name serialized as part of TSDB metric name.

* Fixing namespace service, no longer exposing underlying implementation. Some code cleanup.

* Remove commented out code and add some documentation.

* Update alert expression max length to 2048. (#83)

* Anomaly Detection Transforms (#69)

* Update Angular Selectize dependency

* Implement anomaly detection using Gaussian distribution

* Convert to abstract class for Gaussian-based anomaly detection transforms

* Implementation and unit tests for anomaly detection (Gaussian) using probability density

* Implementation and unit tests for anomaly detection (Gaussian) using z-score

* Add Gaussian anomaly detection transforms to TransformFactory and MetricReader

* Add back original argus-build.properties file

* Add back argus-build.properties to gitignore

* Update anomaly detection transform grammar with shorter names

* Correct message for UnsupportedOperationException

* Add Override annotation to all implementations of abstract methods

* Move important comment into method Javadoc

* Remove index.html from repo

* Normalize anomaly detection results. New abstract class for anomaly detection transforms.

* Update Gaussian anomaly detection tests with normalized results

* Contextual anomaly detection with unit tests

* K-means anomaly detection transform

* Update K-means transform to use Weka 3.6.14 for license compliance

* Removing dead code.

ArgusCore/src/main/java/com/salesforce/dva/argus/entity/TSDBEntity.java

@@ -256,7 +256,6 @@ private String getTag(String key, boolean isUserTag) {
     public static enum ReservedField {
 
         META("meta"),
-        METRIC("metric"),
         UNITS("units"),
         DISPLAY_NAME("displayName");
 

ArgusCore/src/main/java/com/salesforce/dva/argus/service/NamespaceService.java

@@ -43,11 +43,6 @@
  */
 public interface NamespaceService extends Service {
 
-    //~ Instance fields ******************************************************************************************************************************
-
-    /** The namespace prefix used to delineate the namespace. */
-    String NAMEPSACE_PREFIX = "-__-";
-
     //~ Methods **************************************************************************************************************************************
 
     /**

ArgusCore/src/main/java/com/salesforce/dva/argus/service/TSDBService.java

@@ -79,34 +79,6 @@ public interface TSDBService extends Service {
      * @return  The query results. Will never be null, but may be empty.
      */
     List<Annotation> getAnnotations(List<AnnotationQuery> queries);
-
-    /**
-     * Constructs a native TSDB metric name from supplied scope and namespace
-     *
-     * @param scope - TSDB scope of a metric
-     * @param namespace - namespace for a metric   
-     *
-     * @return  The native TSDB metric name for this metric.  This method should never return null.
-     */
-     String constructTSDBMetricName(String scope, String namespace);
-
-    /**
-     * Calculates the scope of a metric given it's native TSDB name.
-     *
-     * @param   tsdbMetricName  The native TSDB metric name.
-     *
-     * @return  The scope of the metric.  May return null.
-     */
-    String getScopeFromTSDBMetric(String tsdbMetricName);
-
-    /**
-     * Calculates the namespace of a metric given it's native TSDB name.
-     *
-     * @param   tsdbMetricName  The TSDB metric name.
-     *
-     * @return  The namespace of the metric.  May return null.
-     */    
-    String getNamespaceFromTSDBMetric(String tsdbMetricName);
 
     /**
      * Enumeration of time window for a query

ArgusCore/src/main/java/com/salesforce/dva/argus/service/jpa/DefaultNamespaceService.java

@@ -106,34 +106,29 @@ public List<Namespace> findNamespacesByOwner(PrincipalUser owner) {
     /**
      * Creates a new namespace.
      *
-     * @param   newNamespace  The namespace to create.  Cannot be null.
+     * @param   namespace  The namespace to create.  Cannot be null.
      *
-     * @return  The update namespace object having the ID field populated.
+     * @return  The updated namespace object having the ID field populated.
      *
      * @throws  SystemException  If a duplicate namespace exists.
      */
     @Override
-    public Namespace createNamespace(Namespace newNamespace) {
+    public Namespace createNamespace(Namespace namespace) {
         requireNotDisposed();
-        requireArgument(newNamespace != null, "null namespace cannot be created.");
-        if (!_validateQualifier(newNamespace.getQualifier())) {
+        requireArgument(namespace != null, "null namespace cannot be created.");
+
+        if (!_validateQualifier(namespace.getQualifier())) {
             throw new SystemException(new IllegalArgumentException(
                     "Illegal characters found while generating namespace. Cannot generate a namespace with this qualifier."));
         }
-
-        String qualifierWithPrefix =
-            (newNamespace.getQualifier().length() >= NAMEPSACE_PREFIX.length() &&
-                newNamespace.getQualifier().substring(0, NAMEPSACE_PREFIX.length()).equals(NAMEPSACE_PREFIX))
-            ? newNamespace.getQualifier() : NAMEPSACE_PREFIX + newNamespace.getQualifier();
-        Namespace namespace = Namespace.findByQualifier(emf.get(), qualifierWithPrefix);
-
-        if (namespace != null) {
+
+        if (Namespace.findByQualifier(emf.get(), namespace.getQualifier()) != null) {
             throw new SystemException(new IllegalArgumentException("Namespace already exists. Please try a different namespace."));
         }
-        newNamespace.setQualifier(qualifierWithPrefix);
-        newNamespace = updateNamespace(newNamespace);
-        _logger.debug("Generated namespace {}.", newNamespace);
-        return newNamespace;
+
+        namespace = updateNamespace(namespace);
+        _logger.debug("Generated namespace {}.", namespace);
+        return namespace;
     }
 
     private boolean _validateQualifier(String qualifierSuffix) {

ArgusCore/src/main/java/com/salesforce/dva/argus/service/tsdb/AnnotationQuery.java

@@ -35,6 +35,7 @@
 import com.salesforce.dva.argus.entity.Metric;
 import com.salesforce.dva.argus.entity.TSDBEntity.ReservedField;
 import com.salesforce.dva.argus.system.SystemException;
+
 import java.io.UnsupportedEncodingException;
 import java.text.MessageFormat;
 import java.util.HashMap;
@@ -263,21 +264,23 @@ protected void setMetric(String metric) {
      * @throws  UnsupportedEncodingException  If UTF-8 is not supported.
      */
     protected String toTagParameterArray(Map<String, String> tags) throws UnsupportedEncodingException {
+    	if(tags == null || tags.isEmpty()) {
+    		return "";
+    	}
+
         StringBuilder sb = new StringBuilder(encode("{", "UTF-8"));
+        for (Map.Entry<String, String> tagEntry : tags.entrySet()) {
+            sb.append(tagEntry.getKey()).append("=");
 
-        if (tags != null && !tags.isEmpty()) {
-            for (Map.Entry<String, String> tagEntry : tags.entrySet()) {
-                sb.append(tagEntry.getKey()).append("=");
-
-                String tagV = tagEntry.getValue().replaceAll("\\|", encode("|", "UTF-8"));
+            String tagV = tagEntry.getValue().replaceAll("\\|", encode("|", "UTF-8"));
 
-                sb.append(tagV).append(",");
-            }
+            sb.append(tagV).append(",");
         }
         sb.replace(sb.length() - 1, sb.length(), encode("}", "UTF-8"));
+
         return sb.toString();
     }
-
+    
     @Override
     public int hashCode() {
         final int prime = 31;
@@ -363,12 +366,14 @@ public String toString() {
         String pattern = "start={0,number,#}&end={1,number,#}&m=avg:{2}{3}&ms=true&show_tsuids=true";
         long start = Math.max(0, getStartTimestamp() - 1);
         long end = Math.max(start, getEndTimestamp() + 1);
+
         String scope = DefaultTSDBService.toAnnotationKey(_scope, _metric, _type, _tags);
+        //When creating the corresponding argus Metric for the annotations, _type is used as metric name. 
+        String tsdbMetricName = DefaultTSDBService.constructTSDBMetricName(new Metric(scope, _type));
         Map<String, String> tags = new HashMap<>(getTags());
-
-        tags.put(ReservedField.METRIC.getKey(), _type);
+
         try {
-            return MessageFormat.format(pattern, start, end, scope, toTagParameterArray(tags));
+            return MessageFormat.format(pattern, start, end, tsdbMetricName, toTagParameterArray(tags));
         } catch (UnsupportedEncodingException ex) {
             throw new SystemException(ex);
         }

ArgusCore/src/main/java/com/salesforce/dva/argus/service/tsdb/CachedTSDBService.java

@@ -579,21 +579,6 @@ public void putAnnotations(List<Annotation> annotations) {
     public List<Annotation> getAnnotations(List<AnnotationQuery> queries) {
         return _defaultTsdbService.getAnnotations(queries);
     }
-
-    @Override
-    public String constructTSDBMetricName(String scope, String namespace) {
-        return _defaultTsdbService.constructTSDBMetricName(scope, namespace);
-    }
-
-    @Override
-    public String getScopeFromTSDBMetric(String tsdbMetricName) {
-	return _defaultTsdbService.getScopeFromTSDBMetric(tsdbMetricName);
-    }
-
-    @Override
-    public String getNamespaceFromTSDBMetric(String tsdbMetricName) {
-	return _defaultTsdbService.getNamespaceFromTSDBMetric(tsdbMetricName);
-    }
 
     private void instrumentQueryLatency(final MonitorService monitorService, final AnnotationQuery query, final long start) {
 		String timeWindow = QueryTimeWindow.getWindow(query.getEndTimestamp() - query.getStartTimestamp());

ArgusCore/src/main/java/com/salesforce/dva/argus/service/tsdb/DefaultTSDBService.java

@@ -78,7 +78,8 @@ public class DefaultTSDBService extends DefaultService implements TSDBService {
     private static final int CHUNK_SIZE = 50;
     private static final int TSDB_DATAPOINTS_WRITE_MAX_SIZE = 100;
     private static final String QUERY_LATENCY_COUNTER = "query.latency";
-    private static final String QUERY_COUNT_COUNTER = "query.count";    
+    private static final String QUERY_COUNT_COUNTER = "query.count";
+    static final String DELIMITER = "-__-";
 
     //~ Instance fields ******************************************************************************************************************************
 
@@ -158,17 +159,67 @@ private static String toAnnotationKey(Annotation annotation) {
 
         return toAnnotationKey(scope, metric, type, tags);
     }
+
+    /**
+     * We construct OpenTSDB metric name as a combination of Argus metric, scope and namespace as follows:
+     * 			
+     * 			metric(otsdb) = metric(argus)<DELIMITER>scope(argus)<DELIMITER>namespace(argus)
+     * 
+     * @param metric
+     * @return OpenTSDB metric name constructed from scope, metric and namespace.
+     */
+    public static String constructTSDBMetricName(Metric metric) {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(metric.getMetric()).append(DELIMITER).append(metric.getScope());
 
-    private ObjectMapper getMapper() {
-        ObjectMapper mapper = new ObjectMapper();
-        SimpleModule module = new SimpleModule();
+        if (metric.getNamespace() != null && !metric.getNamespace().isEmpty()) {
+            sb.append(DELIMITER).append(metric.getNamespace());
+        }
+        return sb.toString();
+    }
 
-        module.addSerializer(Metric.class, new MetricTransform.Serializer(this));
-        module.addDeserializer(Metric.class, new MetricTransform.Deserializer(this));
-        module.addSerializer(AnnotationWrapper.class, new AnnotationTransform.Serializer());
-        module.addDeserializer(AnnotationWrappers.class, new AnnotationTransform.Deserializer());
-        mapper.registerModule(module);
-        return mapper;
+    /**
+     * Given otsdb metric name, return argus metric.
+     * We construct OpenTSDB metric name as a combination of Argus metric, scope and namespace as follows:
+     * 			
+     * 			metric(otsdb) = metric(argus)<DELIMITER>scope(argus)<DELIMITER>namespace(argus)
+     * 
+     * 
+     * @param tsdbMetricName
+     * @return Argus metric name.
+     */
+    public static String getMetricFromTSDBMetric(String tsdbMetricName) {
+    	return tsdbMetricName.split(DELIMITER)[0];
+    }
+
+    /**
+     * Given otsdb metric name, return argus scope.
+     * We construct OpenTSDB metric name as a combination of Argus metric, scope and namespace as follows:
+     * 			
+     * 			metric(otsdb) = metric(argus)<DELIMITER>scope(argus)<DELIMITER>namespace(argus)
+     * 
+     * 
+     * @param tsdbMetricName
+     * @return Argus scope.
+     */
+    public static String getScopeFromTSDBMetric(String tsdbMetricName) {
+    	return tsdbMetricName.split(DELIMITER)[1];
+    }
+
+    /**
+     * Given otsdb metric name, return argus namespace.
+     * We construct OpenTSDB metric name as a combination of Argus metric, scope and namespace as follows:
+     * 			
+     * 			metric(otsdb) = metric(argus)<DELIMITER>scope(argus)<DELIMITER>namespace(argus)
+     * 
+     * 
+     * @param tsdbMetricName
+     * @return Argus namespace. 
+     */
+    public static String getNamespaceFromTSDBMetric(String tsdbMetricName) {
+    	String[] splits = tsdbMetricName.split(DELIMITER);
+        return (splits.length == 3) ? splits[2] : null;
     }
 
     //~ Methods **************************************************************************************************************************************
@@ -302,26 +353,17 @@ public List<Annotation> getAnnotations(List<AnnotationQuery> queries) {
         }
         return annotations;
     }
+
+    private ObjectMapper getMapper() {
+        ObjectMapper mapper = new ObjectMapper();
+        SimpleModule module = new SimpleModule();
 
-    @Override
-    public String constructTSDBMetricName(String scope, String namespace) {
-        StringBuilder sb = new StringBuilder(scope);
-
-        if (namespace != null && !namespace.isEmpty()) {
-            sb.append(namespace);
-        }
-        return sb.toString();
-    }
-
-    @Override
-    public String getScopeFromTSDBMetric(String tsdbMetricName) {
-	return tsdbMetricName.split(NamespaceService.NAMEPSACE_PREFIX)[0];
-    }
-
-    @Override
-    public String getNamespaceFromTSDBMetric(String tsdbMetricName) {
-	String[] splits = tsdbMetricName.split(NamespaceService.NAMEPSACE_PREFIX);
-        return (splits.length == 2) ? NamespaceService.NAMEPSACE_PREFIX + splits[1] : null;
+        module.addSerializer(Metric.class, new MetricTransform.Serializer(this));
+        module.addDeserializer(Metric.class, new MetricTransform.Deserializer(this));
+        module.addSerializer(AnnotationWrapper.class, new AnnotationTransform.Serializer());
+        module.addDeserializer(AnnotationWrappers.class, new AnnotationTransform.Deserializer());
+        mapper.registerModule(module);
+        return mapper;
     }
 
     /* Writes objects in chunks. */

ArgusCore/src/main/java/com/salesforce/dva/argus/service/tsdb/MetricQuery.java

@@ -34,9 +34,9 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.salesforce.dva.argus.entity.TSDBEntity.ReservedField;
 import com.salesforce.dva.argus.system.SystemAssert;
 import com.salesforce.dva.argus.system.SystemException;
+
 import java.io.UnsupportedEncodingException;
 import java.text.MessageFormat;
 import java.util.HashMap;
@@ -177,10 +177,12 @@ public void setDownsamplingPeriod(Long downsamplingPeriod) {
      */
     @JsonIgnore
     public String getTSDBMetricName() {
-        StringBuilder sb = new StringBuilder(getScope());
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(getMetric()).append(DefaultTSDBService.DELIMITER).append(getScope());
 
         if (_namespace != null && !_namespace.isEmpty()) {
-            sb.append(getNamespace());
+            sb.append(DefaultTSDBService.DELIMITER).append(getNamespace());
         }
         return sb.toString();
     }
@@ -256,8 +258,7 @@ public String toString() {
         sb.append(getTSDBMetricName());
 
         Map<String, String> tags = new HashMap<>(getTags());
-
-        tags.put(ReservedField.METRIC.getKey(), getMetric());
+
         try {
             return MessageFormat.format(pattern, start, end, sb.toString(), toTagParameterArray(tags));
         } catch (UnsupportedEncodingException ex) {

ArgusCore/src/main/java/com/salesforce/dva/argus/service/tsdb/MetricTransform.java

@@ -89,15 +89,16 @@ public Metric deserialize(JsonParser jp, DeserializationContext dc) throws IOExc
 
             Map<String, String> meta = fromMeta(tags.get(ReservedField.META.getKey()));
             String tsdbMetricName = node.get("metric").asText();
-            String scope = tsdbService.getScopeFromTSDBMetric(tsdbMetricName);
-            String namespace = tsdbService.getNamespaceFromTSDBMetric(tsdbMetricName);
-
+
             // Post filtering metric , since in some cases TSDB metric can be empty https://github.com/OpenTSDB/opentsdb/issues/540
-            if (scope.isEmpty()) {
+            if (tsdbMetricName.isEmpty()) {
                 return null;
             }
 
-            String metric = tags.get(ReservedField.METRIC.getKey());
+            String scope = DefaultTSDBService.getScopeFromTSDBMetric(tsdbMetricName);
+            String metric = DefaultTSDBService.getMetricFromTSDBMetric(tsdbMetricName);
+            String namespace = DefaultTSDBService.getNamespaceFromTSDBMetric(tsdbMetricName);
+
             Map<String, String> userTags = new HashMap<>();
 
             for (Map.Entry<String, String> entry : tags.entrySet()) {
@@ -171,7 +172,7 @@ public void serialize(Metric metric, JsonGenerator jgen, SerializerProvider sp)
 
             for (Map.Entry<Long, String> entry : datapoints.entrySet()) {
                 jgen.writeStartObject();
-                jgen.writeStringField("metric", tsdbService.constructTSDBMetricName(metric.getScope(), metric.getNamespace()));
+                jgen.writeStringField("metric", DefaultTSDBService.constructTSDBMetricName(metric));
                 jgen.writeNumberField("timestamp", entry.getKey());
                 jgen.writeStringField("value", entry.getValue());
                 serializeTags(metric, jgen);
@@ -184,7 +185,7 @@ private void serializeTags(Metric metric, JsonGenerator jgen) throws IOException
 
             Map<String, String> tags = new HashMap<>(metric.getTags());
 
-            tags.put(ReservedField.METRIC.getKey(), metric.getMetric());
+            //tags.put(ReservedField.METRIC.getKey(), metric.getMetric());
             tags.put(ReservedField.META.getKey(), toMeta(metric));
             for (Map.Entry<String, String> tagEntry : tags.entrySet()) {
                 jgen.writeStringField(tagEntry.getKey(), tagEntry.getValue());