replace byte buffer with ring buffer

core/common.go

@@ -1,11 +1,10 @@
 package core
 
 import (
-	"bytes"
 	"crypto/rand"
 	"errors"
 	"fmt"
-	"io"
+	"github.com/armon/circbuf"
 	"reflect"
 	"strings"
 	"time"
@@ -22,6 +21,9 @@ var (
 	ErrLocalImageNotFound = errors.New("couldn't find image on the host")
 )
 
+// maximum size of a stdout/stderr stream to be kept in memory and optional stored/sent via mail
+const maxStreamSize = 10 * 1024 * 1024
+
 type Job interface {
 	GetName() string
 	GetSchedule() string
@@ -135,15 +137,17 @@ type Execution struct {
 	Skipped   bool
 	Error     error
 
-	OutputStream, ErrorStream io.ReadWriter `json:"-"`
+	OutputStream, ErrorStream *circbuf.Buffer `json:"-"`
 }
 
 // NewExecution returns a new Execution, with a random ID
 func NewExecution() *Execution {
+	bufOut, _ := circbuf.NewBuffer(maxStreamSize)
+	bufErr, _ := circbuf.NewBuffer(maxStreamSize)
 	return &Execution{
 		ID:           randomID(),
-		OutputStream: bytes.NewBuffer(nil),
-		ErrorStream:  bytes.NewBuffer(nil),
+		OutputStream: bufOut,
+		ErrorStream:  bufErr,
 	}
 }
 

core/scheduler.go

@@ -3,7 +3,6 @@ package core
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"sync"
 
 	"github.com/robfig/cron"
@@ -108,10 +107,7 @@ func (w *jobWrapper) stop(ctx *Context, err error) {
 		errText = ctx.Execution.Error.Error()
 	}
 
-	output, err := ioutil.ReadAll(ctx.Execution.OutputStream)
-	if err != nil {
-		ctx.Logger.Errorf("Couldn't read command output")
-	}
+	output := ctx.Execution.OutputStream.Bytes()
 
 	if len(output) > 0 {
 		ctx.Log("Output: " + string(output))

go.mod

@@ -3,6 +3,7 @@ module github.com/mcuadros/ofelia
 go 1.11
 
 require (
+	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
 	github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible
 	github.com/fsouza/go-dockerclient v1.6.5

go.sum

@@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873 h1:93nQ7k53G
 github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg=
 github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
+github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs=
+github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625 h1:ckJgFhFWywOx+YLEMIJsTb+NV6NexWICk5+AMSuz3ss=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
@@ -109,6 +111,7 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -192,6 +195,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=

middlewares/mail.go

@@ -70,12 +70,12 @@ func (m *Mail) sendMail(ctx *core.Context) error {
 
 	base := fmt.Sprintf("%s_%s", ctx.Job.GetName(), ctx.Execution.ID)
 	msg.Attach(base+".stdout.log", gomail.SetCopyFunc(func(w io.Writer) error {
-		_, err := io.Copy(w, ctx.Execution.OutputStream)
+		_, err := w.Write(ctx.Execution.OutputStream.Bytes())
 		return err
 	}))
 
 	msg.Attach(base+".stderr.log", gomail.SetCopyFunc(func(w io.Writer) error {
-		_, err := io.Copy(w, ctx.Execution.ErrorStream)
+		_, err := w.Write(ctx.Execution.ErrorStream.Bytes())
 		return err
 	}))
 

middlewares/save.go

@@ -60,12 +60,12 @@ func (m *Save) saveToDisk(ctx *core.Context) error {
 	))
 
 	e := ctx.Execution
-	err := m.saveReaderToDisk(e.ErrorStream, fmt.Sprintf("%s.stderr.log", root))
+	err := m.saveReaderToDisk(bytes.NewReader(e.ErrorStream.Bytes()), fmt.Sprintf("%s.stderr.log", root))
 	if err != nil {
 		return err
 	}
 
-	err = m.saveReaderToDisk(e.OutputStream, fmt.Sprintf("%s.stdout.log", root))
+	err = m.saveReaderToDisk(bytes.NewReader(e.OutputStream.Bytes()), fmt.Sprintf("%s.stdout.log", root))
 	if err != nil {
 		return err
 	}