move to ds

Dockerfile.ps_collector

@@ -26,6 +26,7 @@ COPY configs/ps-throughput.conf /usr/share/logstash/configs/ps-throughput.conf
 COPY configs/ps-trace.conf /usr/share/logstash/configs/ps-trace.conf
 COPY configs/ps-intake.conf /usr/share/logstash/configs/ps-intake.conf
 COPY configs/es-output.conf /usr/share/logstash/configs/es-output.conf
+COPY configs/es-output-ds.conf /usr/share/logstash/configs/es-output-ds.conf
 COPY configs/nebraska.conf /usr/share/logstash/configs/nebraska.conf
 COPY configs/ps-debug.conf /usr/share/logstash/configs/ps-debug.conf
 

configs/es-output-ds.conf

@@ -0,0 +1,44 @@
+input { pipeline { address => "es-output" } }
+
+output {
+
+  # stdout {
+  #   codec => json
+  # }
+
+  if "_jsonparsefailure" in [tags] {
+    file {
+      path => "/var/log/logstash/json_parse_failure.txt"
+    }
+  }
+  else if "_rubyexception" in [tags] {
+    file {
+      path => "/var/log/logstash/ruby_exception.json"
+    }
+  }  
+  else if "_extra" in [tags] {
+    file {
+      path => "/var/log/logstash/extra.json"
+    }
+  }  
+  else {
+
+  #   file{
+  #     path => "/var/log/logstash/correct_%{[@metadata][type]}.json"
+  #   }
+
+    elasticsearch {
+      hosts => "atlas-kibana.mwt2.org"
+      data_stream => true
+      action => "create"
+      ssl => true
+      index => "%{[@metadata][type]}"
+      user => "uc_logstash_indexer"
+      password => "${LOGSTASH_PWD}"
+      document_id => "%{[@metadata][id]}"
+    }
+
+
+  }
+
+}

configs/ps-latencybg.conf

@@ -49,6 +49,10 @@ filter {
 }
 
 output {
-  pipeline { send_to => ["es-output"] }
-  pipeline { send_to => ["nebraska"] }
+  if [@metadata][type] == "ps_packetloss_write" {
+    pipeline { send_to => ["es-output-ds"] }
+  } else {
+    pipeline { send_to => ["es-output"] }
+    pipeline { send_to => ["nebraska"] }
+  }
 }

pipelines/ps-collector.yml

@@ -26,6 +26,9 @@
 - pipeline.id: es-output
   path.config: "/usr/share/logstash/configs/es-output.conf"
   pipeline.workers: 1
+- pipeline.id: es-output-ds
+  path.config: "/usr/share/logstash/configs/es-output-ds.conf"
+  pipeline.workers: 1
 - pipeline.id: nebraska
   path.config: "/usr/share/logstash/configs/nebraska.conf"
   pipeline.workers: 1