Worked with pivotal-hargun-suri to add a new 'json-cee' log formatter…

… type, allowing a rsyslog server to parse the JSON/CEE in the message.

Gopkg.toml

@@ -43,3 +43,7 @@
 [[constraint]]
   name = "gopkg.in/alecthomas/kingpin.v2"
   version = "~2.1.11"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/stvp/go-udp-testing"

Makefile

@@ -4,6 +4,8 @@ compile: linux32 linux64 darwin64
 
 test:
 	ginkgo -r -v .
+	# For tests that aren't ginkgo based
+	go test ./logging
 
 linux32:
 	 CGO_ENABLED=0 GOARCH=386 GOOS=linux go build --ldflags="-X main.version=${BUILD_NUMBER}" -o dist/linux/386/firehose-to-syslog_linux_386
@@ -22,4 +24,4 @@ docker-dev:
 	$(SHELL) ./Docker/build-dev.sh
 
 docker-final:
-	$(SHELL) ./Docker/build.sh     
+	$(SHELL) ./Docker/build.sh

README.md

@@ -67,8 +67,8 @@ Flags:
   --path-prof=""                 Set the Path to write profiling file
   --log-formatter-type=LOG-FORMATTER-TYPE
                                  Log formatter type to use. Valid options
-                                 are text, json. If none provided, defaults
-                                 to json.
+                                 are text, json, json-cee. If none
+                                 provided, defaults to json.
   --cert-pem-syslog=""           Certificate Pem file
   --ignore-missing-apps          Enable throttling on cache lookup for
                                  missing apps
@@ -85,7 +85,10 @@ Since v3 firehose-to-syslog support TLS syslog `--cert-pem-syslog` using PEM enc
 Please refer to https://github.com/RackSec/srslog/blob/master/script/gen-certs.py
 for Cert generation.
 
+#  JSON CEE rsyslog compatibility
+rsyslog supports parsing JSON log messages throught its [mmjsonparse module](https://www.rsyslog.com/doc/master/configuration/modules/mmjsonparse.html). This module expects a parsable JSON log message to be prepended with the value `@cee:`.
 
+For `LOG-FORMATTER-TYPE`, using `json-cee` will yield the same result as using `json` but the JSON sent will have `@cee:` prepended to it. The `json-cee` formatter type also ensures that a RFC3164 compatible log message is sent, which was found to be required by rsyslog when using the mmjsonparse module. 
 
 # Event documentation
 
@@ -107,7 +110,7 @@ We have 3 caching strategies:
     cd $GOPATH/src/github.com/cloudfoundry-community/firehose-to-syslog
 
     # Test
-	    ginkgo -r .
+	make test
 
     # Build binary
     go build
@@ -186,7 +189,7 @@ Showing top 10 nodes out of 44 (cum >= 20ms)
     cf set-env firehose-to-syslog FIREHOSE_SUBSCRIPTION_ID firehose-to-syslog
     cf set-env firehose-to-syslog FIREHOSE_CLIENT_ID  [your doppler.firehose enabled client id]
     cf set-env firehose-to-syslog FIREHOSE_CLIENT_SECRET  [your doppler.firehose enabled client secret]
-    cf set-env firehose-to-syslog LOG_FORMATTER_TYPE [Log formatter type to use. Valid options are : text, json]
+    cf set-env firehose-to-syslog LOG_FORMATTER_TYPE [Log formatter type to use. Valid options are : text, json, json-cee]
     ```
 1. Turn off the health check if you're staging to Diego.
     ```

cli.go

@@ -44,7 +44,7 @@ var (
 	orgs              = kingpin.Flag("orgs", "Forwarded on the app logs from theses organisations' example: --orgs=org1,org2").Default("").Envar("ORGS").String()
 	modeProf          = kingpin.Flag("mode-prof", "Enable profiling mode, one of [cpu, mem, block]").Default("").Envar("MODE_PROF").String()
 	pathProf          = kingpin.Flag("path-prof", "Set the Path to write profiling file").Default("").Envar("PATH_PROF").String()
-	logFormatterType  = kingpin.Flag("log-formatter-type", "Log formatter type to use. Valid options are text, json. If none provided, defaults to json.").Envar("LOG_FORMATTER_TYPE").String()
+	logFormatterType  = kingpin.Flag("log-formatter-type", "Log formatter type to use. Valid options are text, json, json-cee. If none provided, defaults to json.").Envar("LOG_FORMATTER_TYPE").String()
 	certPath          = kingpin.Flag("cert-pem-syslog", "Certificate Pem file").Envar("CERT_PEM").Default("").String()
 	ignoreMissingApps = kingpin.Flag("ignore-missing-apps", "Enable throttling on cache lookup for missing apps").Envar("IGNORE_MISSING_APPS").Default("false").Bool()
 	stripAppSuffixes  = kingpin.Flag("strip-app-name-suffixes", "Suffixes that should be stripped from application names, comma separated").Envar("STRIP_APP_NAME_SUFFIXES").Default("").String()

logging/logging_logrus.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"time"
 
 	syslog "github.com/RackSec/srslog"
 	logrus_syslog "github.com/shinji62/logrus-syslog-ng"
@@ -32,6 +33,18 @@ func NewLogging(SyslogServerFlag string, SysLogProtocolFlag string, LogFormatter
 	}
 }
 
+// This srslog formatter is based on srslog/formatter.go's RFC3164Formatter.
+// The default rsyslog input module expects RFC3164 formatted logs,  using srslog's DefaultFormatter with "@cee:" wasn't
+//  compatible enough to get it working.
+// some historical context on the efforts to standardize the syslog message
+//  format(s): https://www.rsyslog.com/doc/syslog_parsing.html
+func CeeFormatter(p syslog.Priority, hostname, tag, content string) string {
+	timestamp := time.Now().Format(time.Stamp)
+	msg := fmt.Sprintf("<%d>%s %s %s[%d]: @cee: %s",
+		p, timestamp, hostname, tag, os.Getpid(), content)
+	return msg
+}
+
 func (l *LoggingLogrus) Connect() bool {
 
 	success := false
@@ -55,6 +68,9 @@ func (l *LoggingLogrus) Connect() bool {
 		if err != nil {
 			LogError(fmt.Sprintf("Unable to connect to syslog server [%s]!\n", l.syslogServer), err.Error())
 		} else {
+			if l.logFormatterType == "json-cee" {
+				hook.(*logrus_syslog.SyslogHook).Writer.SetFormatter(CeeFormatter)
+			}
 			LogStd(fmt.Sprintf("Received hook to syslog server [%s]!\n", l.syslogServer), false)
 			l.Logger.Hooks.Add(hook)
 			success = true

logging/logging_suite_test.go

@@ -1,6 +1,11 @@
 package logging
 
 import (
+	"encoding/json"
+	"fmt"
+	"github.com/stvp/go-udp-testing"
+	"math/rand"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -13,6 +18,28 @@ func TestEvents(t *testing.T) {
 	RunSpecs(t, "Logging Suite")
 }
 
+// Can't use ginkgo because go-udp-testing requires to be passed testing.T
+func TestJSONCeeLoggingFormatter(t *testing.T) {
+	RegisterTestingT(t)
+
+	port := rand.Intn(65535 - 1080) + 1081
+	listeningUDPSocket := fmt.Sprintf("localhost:%d", port)
+
+	udp.SetAddr(listeningUDPSocket)
+	result := udp.ReceiveString(t, func() {
+		logging := NewLogging(listeningUDPSocket, "udp", "json-cee", "", false, true)
+		logging.Connect()
+		logging.ShipEvents(nil, "msg field content")
+	})
+
+	Ω(result).Should(ContainSubstring("@cee:"))
+
+	jsonPayload := strings.Split(result, "@cee:")[1]
+	var payload map[string]interface{}
+	json.Unmarshal([]byte(jsonPayload), &payload)
+	Ω(payload["msg"]).Should(Equal("msg field content"))
+}
+
 var _ = Describe("Logging", func() {
 	Describe("SetupLogging", func() {
 		Context("called with a Text formatter", func() {