go script for report - understand if usefull?

cloudfront-logs-report/logOutput/logOutput.go

@@ -0,0 +1,118 @@
+package logOutput
+
+import (
+	"fmt"
+	"os"
+	"slices"
+	"strings"
+)
+
+type CsvOutput struct {
+	fileName              string
+	fieldsToSkip          []string
+	fieldIndexesToSkip    []int
+	fieldsCondition       []string
+	fieldIndexesCondition []int
+}
+
+func New(fileName string, fieldsToSkip []string, fieldsCondition []string) *CsvOutput {
+	return &CsvOutput{
+		fileName:        fileName,
+		fieldsToSkip:    fieldsToSkip,
+		fieldsCondition: fieldsCondition,
+	}
+}
+
+func (l *CsvOutput) AppendHead(headFields []string) {
+	l.fieldIndexesToSkip = getFieldIndexes(headFields, l.fieldsToSkip...)
+	l.fieldIndexesCondition = getFieldIndexes(headFields, l.fieldsCondition...)
+	headFields = removeIdexes(headFields, l.fieldIndexesToSkip)
+	if fileIsEmptyOrNotExists(l.fileName) {
+		appendToFile(l.fileName, fmt.Sprintf("%s\n", strings.Join(headFields, ",")))
+	}
+}
+
+func (l *CsvOutput) AppendLine(lineOfValues []string, condition func(map[string]string) bool) error {
+	fieldsToVerify := map[string]string{}
+	for idx, idxOfField := range l.fieldIndexesCondition {
+		if len(lineOfValues) > idxOfField {
+			fieldsToVerify[l.fieldsCondition[idx]] = lineOfValues[idxOfField]
+		}
+	}
+	if !condition(fieldsToVerify) {
+		return nil
+	}
+
+	lineOfValues = removeIdexes(lineOfValues, l.fieldIndexesToSkip)
+	for i, value := range lineOfValues {
+		if strings.Contains(value, ",") {
+			lineOfValues[i] = fmt.Sprintf("\"%s\"", value)
+		}
+	}
+	lineString := fmt.Sprintln(strings.Join(lineOfValues, ","))
+	if len(lineString) <= 1 {
+		return nil
+	}
+	if err := appendToFile(l.fileName, lineString); err != nil {
+		return fmt.Errorf("Failed to open file: %s", err)
+	}
+	return nil
+}
+
+func appendToFile(fileName string, line string) error {
+	file, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("Failed to open file: %s", err)
+	}
+	defer file.Close()
+
+	if _, err := file.WriteString(line); err != nil {
+		return fmt.Errorf("Failed to write in file: %s", err)
+	}
+	return nil
+}
+
+func fileIsEmptyOrNotExists(fileName string) bool {
+	file, err := os.Open(fileName)
+	if err != nil {
+		return true
+	}
+	defer file.Close()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return true
+	}
+
+	return stat.Size() == 0
+}
+
+func getFieldIndexes(fields []string, fieldNames ...string) []int {
+	positions := []int{}
+	for i, field := range fields {
+		if slices.Contains(fieldNames, field) {
+			positions = append(positions, i)
+		}
+	}
+	return positions
+}
+
+func SkipLines[T any](lines [][]T, indexField int, condition func(T) bool) [][]T {
+	filteredLines := [][]T{}
+	for _, line := range lines {
+		if !condition(line[indexField]) {
+			filteredLines = append(filteredLines, line)
+		}
+	}
+	return filteredLines
+}
+
+func removeIdexes[T any](line []T, indexesToFilter []int) []T {
+	filteredLine := []T{}
+	for i, value := range line {
+		if !slices.Contains(indexesToFilter, i) {
+			filteredLine = append(filteredLine, value)
+		}
+	}
+	return filteredLine
+}

cloudfront-logs-report/logStorage/logStorage.go

@@ -0,0 +1,78 @@
+package logStorage
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"log"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type LogStorage struct {
+	s3     *s3.Client
+	ctx    context.Context
+	bucket string
+	folder string
+}
+
+func New() *LogStorage {
+	ctx := context.Background()
+	sdkConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion("eu-west-1"))
+	if err != nil {
+		log.Fatalf("unable to load SDK config, %v", err)
+	}
+	s3Client := s3.NewFromConfig(sdkConfig)
+	return &LogStorage{
+		s3:     s3Client,
+		ctx:    ctx,
+		bucket: "angeloceccato-website-logs",
+		folder: "angeloceccato-website-content/",
+	}
+}
+
+func (logStorage *LogStorage) AllLogFiles() ([]string, error) {
+	listObjectVersionsInput := s3.ListObjectsV2Input{
+		Bucket: aws.String(logStorage.bucket),
+		Prefix: aws.String(logStorage.folder),
+	}
+	result, err := logStorage.s3.ListObjectsV2(logStorage.ctx, &listObjectVersionsInput)
+	if err != nil {
+		return nil, err
+	}
+	keys := []string{}
+	for _, content := range result.Contents {
+		keys = append(keys, aws.ToString(content.Key))
+	}
+	return keys, nil
+}
+
+func (logStorage *LogStorage) DownloadLogFile(key string) (string, error) {
+	getObjectInput := s3.GetObjectInput{
+		Bucket: aws.String(logStorage.bucket),
+		Key:    aws.String(key),
+	}
+	result, err := logStorage.s3.GetObject(logStorage.ctx, &getObjectInput)
+	if err != nil {
+		return "", err
+	}
+	defer result.Body.Close()
+	return gzipDecode(result.Body), nil
+}
+
+func gzipDecode(reader io.Reader) string {
+	gzreader, e1 := gzip.NewReader(reader)
+	if e1 != nil {
+		fmt.Println(e1)
+	}
+
+	output, e2 := io.ReadAll(gzreader)
+	if e2 != nil {
+		fmt.Println(e2)
+	}
+
+	return string(output)
+}

cloudfront-logs-report/logshape/logshape.go

@@ -6,14 +6,24 @@ import (
 
 type Log struct {
 	Fields []string
+	Values [][]string
 }
 
-func LogFormatter(logs string) Log {
-	lines := strings.Split(logs, "\n")
-	lines = lines[1:]
-	fieldsLineSplited := strings.Split(lines[0], "#Fields: ")
-	fieldsLine := fieldsLineSplited[1]
-	fields := strings.Split(fieldsLine, " ")
-	//fmt.Println(strings.Join(fields, ","))
-	return Log{Fields: fields}
+func LogParse(logs string) Log {
+	stringLines := strings.Split(logs, "\n")
+	stringLines = stringLines[1:]
+	fieldsStringLineSplited := strings.Split(stringLines[0], "#Fields: ")
+	fieldsStringLine := fieldsStringLineSplited[1]
+	fieldsFirstLine := strings.Split(fieldsStringLine, " ")
+
+	stringLinesWithValues := stringLines[1:]
+	valuesPerLine := [][]string{}
+	for _, line := range stringLinesWithValues {
+		valuesPerLine = append(valuesPerLine, strings.Split(line, "\t"))
+	}
+
+	return Log{
+		Fields: fieldsFirstLine,
+		Values: valuesPerLine,
+	}
 }

cloudfront-logs-report/logshape/logshape_test.go

@@ -19,7 +19,7 @@ func TestLogFormatter(t *testing.T) {
 2024-10-11	22:35:45	SEA900-P3	529	216.244.66.244	GET	dssqkj1981x2t.cloudfront.net	/robots.txt	200	-	Mozilla/5.0%20(compatible;%20DotBot/1.2;%20+https://opensiteexplorer.org/dotbot;%[email protected])	-	-	RefreshHit	Szyb5Cz7_JK0rENfdp8-jc7oJmoTbjNJZoJ_iKjI7F6Oj0xq9SCeog==	angeloceccato.com	https	222	0.273	-	TLSv1.2	ECDHE-RSA-AES128-GCM-SHA256	RefreshHit	HTTP/1.1	-	-	37933	0.273	RefreshHit	text/plain	69	-	-
 2024-10-11	22:35:44	SEA900-P3	578	216.244.66.244	GET	dssqkj1981x2t.cloudfront.net	/robots.txt	301	-	Mozilla/5.0%20(compatible;%20DotBot/1.2;%20+https://opensiteexplorer.org/dotbot;%[email protected])	-	-	Redirect	iG3alVWAwv4dYiX9xuaMLXFWOq3GGU2usKwUPRS3T8CNJkkXW9i_rA==	angeloceccato.com	http	222	0.001	-	-	-	Redirect	HTTP/1.1	-	-	49918	0.001	Redirect	text/html	167	-	-
 `
-	formatted := logshape.LogFormatter(log)
+	formatted := logshape.LogParse(log)
 
 	expectedFields := []string{
 		"date",
@@ -58,7 +58,45 @@ func TestLogFormatter(t *testing.T) {
 	}
 
 	if diff := deep.Equal(expectedFields, formatted.Fields); diff != nil {
-		//	t.Errorf("got %v, \n ##### \n want %v", formatted.Fields, expectedFields)
+		t.Error(diff)
+	}
+
+	expectedFirstLineValues := []string{
+		"2024-10-11",
+		"22:33:27",
+		"SIN2-P8",
+		"808",
+		"114.119.135.98",
+		"GET",
+		"dssqkj1981x2t.cloudfront.net",
+		"/graph/link/are-developers-needed-in-the-age-of-ai-link",
+		"302",
+		"https://angeloceccato.it/graph",
+		"Mozilla/5.0%20(Linux;%20Android%207.0;)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Mobile%20Safari/537.36%20(compatible;%20PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
+		"-",
+		"-",
+		"Miss",
+		"5fnl73FSgF4xlEgwShL_S0rJqoX2DYNzHSqDJV7QxiIGAH4rUj5zQQ==",
+		"angeloceccato.it",
+		"https",
+		"470",
+		"0.382",
+		"-",
+		"TLSv1.3",
+		"TLS_AES_128_GCM_SHA256",
+		"Miss",
+		"HTTP/1.1",
+		"-",
+		"-",
+		"25751",
+		"0.382",
+		"Miss",
+		"text/html;%20charset=utf-8",
+		"313",
+		"-",
+		"-",
+	}
+	if diff := deep.Equal(expectedFirstLineValues, formatted.Values[0]); diff != nil {
 		t.Error(diff)
 	}
 }