Merge pull request #17 from uselagoon/add-persistent-storage

Feature: Incorporates a test for persistent storage

Dockerfile

@@ -8,9 +8,7 @@ RUN go get github.com/joho/godotenv
 
 RUN go build && chmod +x ./internal-services-test
 
-ENV SOLR_HOST=solr \
-    REDIS_HOST=redis \
-    OPENSEARCH_HOST=opensearch-2
+ENV STORAGE_LOCATION='/app/files'
 
 EXPOSE 3000
 

TESTING_dockercompose.md

@@ -76,6 +76,10 @@ docker compose exec -T commons sh -c "curl -kL http://go-web:3000/solr?service=s
 # solr-8 should be able to read/write data
 docker compose exec -T commons sh -c "curl -kL http://go-web:3000/solr?service=solr-8" | grep "SERVICE_HOST=solr-8"
 docker compose exec -T commons sh -c "curl -kL http://go-web:3000/solr?service=solr-8" | grep "LAGOON_TEST_VAR=internal-services-test"
+
+# persistent storage should be able to read/write data
+docker compose exec -T commons sh -c "curl -kL http://go-web:3000/storage?path=/app/files" | grep "STORAGE_PATH=/app/files/storage.txt"
+docker compose exec -T commons sh -c "curl -kL http://go-web:3000/storage?path=/app/files" | grep "LAGOON_TEST_VAR=internal-services-test"
 ```
 
 Destroy tests

docker-compose.yml

@@ -4,21 +4,25 @@ volumes:
   solr-7-data:
   solr-8-data:
   opensearch-2-data:
+  scratch:
 
 services:
   web:
     build:
       context: .
       dockerfile: Dockerfile
     labels:
-      lagoon.type: basic
+      lagoon.type: basic-persistent
     ports:
       - '3000:3000'
     container_name: go-web
     environment:
       - LAGOON_TEST_VAR=internal-services-test
       - LAGOON_GIT_SHA=SHA256
       - LAGOON_ENVIRONMENT_TYPE=development
+      - STORAGE_LOCATION=/app/files
+    volumes:
+      - scratch:/app/files
 
   mariadb-10-5:
     image: uselagoon/mariadb-10.5:latest

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/opensearch-project/opensearch-go/v2 v2.3.0
 	github.com/redis/go-redis/v9 v9.3.0
+	github.com/uselagoon/machinery v0.0.12
 	github.com/vanng822/go-solr v0.10.0
 	go.mongodb.org/mongo-driver v1.12.1
 )

go.sum

@@ -28,8 +28,9 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -56,6 +57,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/uselagoon/machinery v0.0.12 h1:TJnA+FrL1uEhRTjJ6dExiL4G7SOQ+hUfGuWDmbW2HBA=
+github.com/uselagoon/machinery v0.0.12/go.mod h1:h/qeMWQR4Qqu33x+8AulNDeolEwvb/G+aIsn/jyUtwk=
 github.com/vanng822/go-solr v0.10.0 h1:oygAxyFL2apSN8vddxDXoyho40z66Guu9nwH7ANr6wM=
 github.com/vanng822/go-solr v0.10.0/go.mod h1:FSglzTPzoNVKTXP+SqEQiiz284cKzcKpeRXmwPa81wc=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=

main.go

@@ -6,15 +6,10 @@ import (
 	"github.com/gorilla/mux"
 	"log"
 	"net/http"
-	"os"
 	"strings"
 	"time"
 )
 
-var (
-	localCheck = os.Getenv("LAGOON_ENVIRONMENT")
-)
-
 type funcType func() map[string]string
 
 func main() {
@@ -25,6 +20,7 @@ func main() {
 	r.HandleFunc("/solr", solrHandler)
 	r.HandleFunc("/mongo", mongoHandler)
 	r.HandleFunc("/opensearch", opensearchHandler)
+	r.HandleFunc("/storage", persistentStorageHandler)
 	r.HandleFunc("/", handleReq)
 	http.Handle("/", r)
 
@@ -76,12 +72,3 @@ func cleanRoute(basePath string) (string, string) {
 	lagoonService := strings.ToUpper(replaceHyphen)
 	return localService, lagoonService
 }
-
-// getEnv get key environment variable if exist otherwise return defalutValue
-func getEnv(key, defaultValue string) string {
-	value := os.Getenv(key)
-	if len(value) == 0 {
-		return defaultValue
-	}
-	return value
-}

mariadb.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 	_ "github.com/go-sql-driver/mysql"
+	machineryEnvVars "github.com/uselagoon/machinery/utils/variables"
 	"log"
 	"net/http"
 	"os"
@@ -18,11 +19,11 @@ var (
 func mariadbHandler(w http.ResponseWriter, r *http.Request) {
 	service := r.URL.Query().Get("service")
 	localService, lagoonService := cleanRoute(service)
-	mariadbUser := getEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
-	mariadbPassword := getEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
-	mariadbHost := getEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
-	mariadbPort := getEnv(fmt.Sprintf("%s_PORT", lagoonService), "3306")
-	mariadbDatabase := getEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
+	mariadbUser := machineryEnvVars.GetEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
+	mariadbPassword := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
+	mariadbHost := machineryEnvVars.GetEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
+	mariadbPort := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PORT", lagoonService), "3306")
+	mariadbDatabase := machineryEnvVars.GetEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
 
 	mariadbConnectionStr = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", mariadbUser, mariadbPassword, mariadbHost, mariadbPort, mariadbDatabase)
 	log.Print(fmt.Sprintf("Using %s as the connstring", mariadbConnectionStr))

mongo.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	machineryEnvVars "github.com/uselagoon/machinery/utils/variables"
 	"log"
 	"net/http"
 	"os"
@@ -25,11 +26,11 @@ var (
 func mongoHandler(w http.ResponseWriter, r *http.Request) {
 	service := r.URL.Query().Get("service")
 	localService, lagoonService := cleanRoute(service)
-	mongoUser := getEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
-	mongoPassword := getEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
-	mongoHost := getEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
-	mongoPort := getEnv(fmt.Sprintf("%s_PORT", lagoonService), "27017")
-	mongoDatabase := getEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
+	mongoUser := machineryEnvVars.GetEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
+	mongoPassword := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
+	mongoHost = machineryEnvVars.GetEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
+	mongoPort := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PORT", lagoonService), "27017")
+	mongoDatabase := machineryEnvVars.GetEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
 
 	if mongoHost != localService {
 		mongoConnectionStr = fmt.Sprintf("mongodb://%s:%s@%s:%s/%s", mongoUser, mongoPassword, mongoHost, mongoPort, mongoDatabase)

persistentstorage.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	machineryEnvVars "github.com/uselagoon/machinery/utils/variables"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func persistentStorageHandler(w http.ResponseWriter, r *http.Request) {
+	path := r.URL.Query().Get("path")
+	log.Print(fmt.Sprintf("Writing to %s", path))
+	fmt.Fprintf(w, persistentStorageConnector(path))
+}
+
+func persistentStorageConnector(route string) string {
+	if route != machineryEnvVars.GetEnv("STORAGE_LOCATION", "") {
+		return "Storage location is not defined - ensure format matches '/storage?path=[path]'"
+	}
+	path := route + "/storage.txt"
+	f, err := os.Create(path)
+	if err != nil {
+		log.Println(err)
+	}
+
+	for _, e := range os.Environ() {
+		if strings.Contains(e, "LAGOON_") {
+			_, err := f.WriteString(strconv.Quote(e) + "\n")
+			if err != nil {
+				log.Print(err)
+			}
+			e := f.Sync()
+			if e != nil {
+				log.Print(e)
+			}
+		}
+	}
+
+	fileBuffer, err := os.ReadFile(path)
+	var results = string(fileBuffer)
+	storagePath := fmt.Sprintf(`"STORAGE_PATH=%s"`, path)
+	storageResults := storagePath + "\n" + results
+	return storageResults
+}

postgres.go

@@ -3,6 +3,7 @@ package main
 import (
 	"database/sql"
 	"fmt"
+	machineryEnvVars "github.com/uselagoon/machinery/utils/variables"
 	"log"
 	"net/http"
 	"os"
@@ -20,11 +21,11 @@ var (
 func postgresHandler(w http.ResponseWriter, r *http.Request) {
 	service := r.URL.Query().Get("service")
 	localService, lagoonService := cleanRoute(service)
-	postgresUser := getEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
-	postgresPassword := getEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
-	postgresHost := getEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
-	postgresPort := getEnv(fmt.Sprintf("%s_PORT", lagoonService), "5432")
-	postgresDatabase := getEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
+	postgresUser := machineryEnvVars.GetEnv(fmt.Sprintf("%s_USERNAME", lagoonService), "lagoon")
+	postgresPassword := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PASSWORD", lagoonService), "lagoon")
+	postgresHost := machineryEnvVars.GetEnv(fmt.Sprintf("%s_HOST", lagoonService), localService)
+	postgresPort := machineryEnvVars.GetEnv(fmt.Sprintf("%s_PORT", lagoonService), "5432")
+	postgresDatabase := machineryEnvVars.GetEnv(fmt.Sprintf("%s_DATABASE", lagoonService), "lagoon")
 
 	postgresConnectionStr = fmt.Sprintf("user=%s password=%s dbname=%s sslmode=%s host=%s port=%s", postgresUser, postgresPassword, postgresDatabase, postgresSSL, postgresHost, postgresPort)
 	log.Print(fmt.Sprintf("Using %s as the connstring", postgresConnectionStr))