Rewriting

Adding Test
Moving to Go1.7

.gitignore

@@ -1,3 +1,4 @@
+internal-core
 firehose-to-syslog*
 events/events.test
 .project

README.md

@@ -148,8 +148,6 @@ Showing top 10 nodes out of 44 (cum >= 20ms)
          0     0%   100%       20ms 18.18%  bufio.(*Reader).fill
 ```
 
-For Mac OSX golang profiling do not work.
-
 # Push as an App to Cloud Foundry
 
 1. Create doppler.firehose enabled user

caching/caching.go

@@ -1,12 +1,8 @@
 package caching
 
 import (
-	"fmt"
-	"github.com/boltdb/bolt"
-	log "github.com/cloudfoundry-community/firehose-to-syslog/logging"
-	cfClient "github.com/cloudfoundry-community/go-cfclient"
-	json "github.com/pquerna/ffjson/ffjson"
 	"regexp"
+	"time"
 )
 
 type App struct {
@@ -18,116 +14,16 @@ type App struct {
 	OrgGuid   string
 }
 
-var gcfClient *cfClient.Client
-var appdb *bolt.DB
+//go:generate counterfeiter . Caching
 
-func CreateBucket() {
-	appdb.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
-		if err != nil {
-			return fmt.Errorf("create bucket: %s", err)
-		}
-		return nil
-
-	})
-
-}
-
-func FillDatabase(listApps []App) {
-
-	for _, app := range listApps {
-		appdb.Update(func(tx *bolt.Tx) error {
-			b, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
-			if err != nil {
-				return fmt.Errorf("create bucket: %s", err)
-			}
-
-			serialize, err := json.Marshal(app)
-
-			if err != nil {
-				return fmt.Errorf("Error Marshaling data: %s", err)
-			}
-			err = b.Put([]byte(app.Guid), serialize)
-
-			if err != nil {
-				return fmt.Errorf("Error inserting data: %s", err)
-			}
-			return nil
-		})
-
-	}
-
-}
-
-func GetAppByGuid(appGuid string) []App {
-	var apps []App
-	app := gcfClient.AppByGuid(appGuid)
-	apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
-	FillDatabase(apps)
-	return apps
-
-}
-
-func GetAllApp() []App {
-
-	log.LogStd("Retrieving Apps for Cache...", false)
-	var apps []App
-
-	defer func() {
-		if r := recover(); r != nil {
-			log.LogError("Recovered in caching.GetAllApp()", r)
-		}
-	}()
-
-	for _, app := range gcfClient.ListApps() {
-		log.LogStd(fmt.Sprintf("App [%s] Found...", app.Name), false)
-		apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
-	}
-
-	FillDatabase(apps)
-
-	log.LogStd(fmt.Sprintf("Found [%d] Apps!", len(apps)), false)
-
-	return apps
-}
-
-func GetAppInfo(appGuid string) App {
-
-	defer func() {
-		if r := recover(); r != nil {
-			log.LogError(fmt.Sprintf("Recovered from panic retrieving App Info for App Guid: %s", appGuid), r)
-		}
-	}()
-
-	var d []byte
-	var app App
-	appdb.View(func(tx *bolt.Tx) error {
-		log.LogStd(fmt.Sprintf("Looking for App %s in Cache!\n", appGuid), false)
-		b := tx.Bucket([]byte("AppBucket"))
-		d = cloneBytes(b.Get([]byte(appGuid)))
-		return nil
-	})
-	err := json.Unmarshal([]byte(d), &app)
-	if err != nil {
-		return App{}
-	}
-	return app
-}
-
-// cloneBytes returns a copy of a given slice.
-func cloneBytes(v []byte) []byte {
-	var clone = make([]byte, len(v))
-	copy(clone, v)
-	return clone
-}
-
-func SetCfClient(cfClient *cfClient.Client) {
-	gcfClient = cfClient
-
-}
-
-func SetAppDb(db *bolt.DB) {
-	appdb = db
+type Caching interface {
+	CreateBucket()
+	PerformPoollingCaching(time.Duration)
+	GetAppByGuid(string) []App
+	GetAllApp() []App
+	GetAppInfo(string) App
+	GetAppInfoCache(string) App
+	Close()
 }
 
 func IsNeeded(wantedEvents string) bool {

caching/caching_boltdb.go

@@ -0,0 +1,144 @@
+package caching
+
+import (
+	"fmt"
+	"github.com/boltdb/bolt"
+	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
+	cfClient "github.com/cloudfoundry-community/go-cfclient"
+	json "github.com/mailru/easyjson"
+	"log"
+	"os"
+	"time"
+)
+
+type CachingBolt struct {
+	GcfClient *cfClient.Client
+	Appdb     *bolt.DB
+}
+
+func NewCachingBolt(gcfClientSet *cfClient.Client, boltDatabasePath string) Caching {
+
+	//Use bolt for in-memory  - file caching
+	db, err := bolt.Open(boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		log.Fatal("Error opening bolt db: ", err)
+		os.Exit(1)
+
+	}
+
+	return &CachingBolt{
+		GcfClient: gcfClientSet,
+		Appdb:     db,
+	}
+}
+
+func (c *CachingBolt) CreateBucket() {
+	c.Appdb.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
+		if err != nil {
+			return fmt.Errorf("create bucket: %s", err)
+		}
+		return nil
+
+	})
+
+}
+
+func (c *CachingBolt) PerformPoollingCaching(tickerTime time.Duration) {
+	// Ticker Pooling the CC every X sec
+	ccPooling := time.NewTicker(tickerTime)
+
+	var apps []App
+	go func() {
+		for range ccPooling.C {
+			apps = c.GetAllApp()
+		}
+	}()
+
+}
+
+func (c *CachingBolt) fillDatabase(listApps []App) {
+	for _, app := range listApps {
+		c.Appdb.Update(func(tx *bolt.Tx) error {
+			b, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
+			if err != nil {
+				return fmt.Errorf("create bucket: %s", err)
+			}
+
+			serialize, err := json.Marshal(app)
+
+			if err != nil {
+				return fmt.Errorf("Error Marshaling data: %s", err)
+			}
+			err = b.Put([]byte(app.Guid), serialize)
+
+			if err != nil {
+				return fmt.Errorf("Error inserting data: %s", err)
+			}
+			return nil
+		})
+
+	}
+
+}
+
+func (c *CachingBolt) GetAppByGuid(appGuid string) []App {
+	var apps []App
+	app := c.GcfClient.AppByGuid(appGuid)
+	apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
+	c.fillDatabase(apps)
+	return apps
+
+}
+
+func (c *CachingBolt) GetAllApp() []App {
+
+	logging.LogStd("Retrieving Apps for Cache...", false)
+	var apps []App
+
+	defer func() {
+		if r := recover(); r != nil {
+			logging.LogError("Recovered in caching.GetAllApp()", r)
+		}
+	}()
+
+	for _, app := range c.GcfClient.ListApps() {
+		logging.LogStd(fmt.Sprintf("App [%s] Found...", app.Name), false)
+		apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
+	}
+
+	c.fillDatabase(apps)
+	logging.LogStd(fmt.Sprintf("Found [%d] Apps!", len(apps)), false)
+
+	return apps
+}
+
+func (c *CachingBolt) GetAppInfo(appGuid string) App {
+
+	var d []byte
+	var app App
+	c.Appdb.View(func(tx *bolt.Tx) error {
+		logging.LogStd(fmt.Sprintf("Looking for App %s in Cache!\n", appGuid), false)
+		b := tx.Bucket([]byte("AppBucket"))
+		d = b.Get([]byte(appGuid))
+		return nil
+	})
+	err := json.Unmarshal([]byte(d), &app)
+	if err != nil {
+		return App{}
+	}
+	return app
+}
+
+func (c *CachingBolt) Close() {
+	c.Appdb.Close()
+}
+
+func (c *CachingBolt) GetAppInfoCache(appGuid string) App {
+	if app := c.GetAppInfo(appGuid); app.Name != "" {
+		return app
+	} else {
+		c.GetAppByGuid(appGuid)
+	}
+	return c.GetAppInfo(appGuid)
+}

caching/caching_boltdb_test.go

@@ -0,0 +1,14 @@
+package caching_test
+
+import (
+	. "github.com/boltdb/bolt"
+	. "github.com/cloudfoundry-community/firehose-to-syslog/caching"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CachingBoltdb", func() {
+
+	var bdb *Caching
+
+})