Merge pull request #80 from adamjsturge/dev

Merge Dev into Main mostly dependencies (and e2e testing)

.github/workflows/playwright.yml

@@ -12,6 +12,7 @@ on:
       - "*.go"
       - "e2e/**"
       - ".github/workflows/playwright.yml"
+      - "docker-compose.prod.yml"
   pull_request:
     branches: [ main, dev ]
     paths:
@@ -24,6 +25,7 @@ on:
       - "*.go"
       - "e2e/**"
       - ".github/workflows/playwright.yml"
+      - "docker-compose.prod.yml"
 jobs:
   e2e-tests:
     timeout-minutes: 60
@@ -45,26 +47,27 @@ jobs:
     - name: Install Playwright Browsers
       run:  npx playwright install --with-deps
       working-directory: e2e
-    - name: Show docker logs
-      run: docker logs xsshunter-go-xsshunter-go-1
-      working-directory: ./
-    # - name: NPM Install
-    #   run: npm install
-    #   working-directory: e2e
     - name: Get Password from Docker logs
-      id: logs
       run: |
-        LOGS=$(docker logs xsshunter-go-xsshunter-go-1)
-        PASSWORD=$(echo "$LOGS" | grep -oP 'PASSWORD: \K.*')
-        echo "::set-output name=password::$PASSWORD"
-        echo "TEMP_E2E_PLAYWRIGHT_PASSWORD=$PASSWORD" > e2e/.env
+        echo "TEMP_E2E_PLAYWRIGHT_PASSWORD=$(docker logs xsshunter-go-xsshunter-go-1 | grep -oP 'PASSWORD: \K.*')" > e2e/.env
     - name: Run Playwright tests
-      env:
-        TEMP_E2E_PLAYWRIGHT_PASSWORD: ${{ steps.logs.outputs.password }}
       run: npx playwright test
       working-directory: e2e
-    # - uses: actions/upload-artifact@v4
-    #   if: always()
-    #   with:
-    #     name: trace
-    #     path: ./e2e/test-results/xsshunter-Logging-in-Successfully-firefox-retry1/trace.zip
+    - name: Stop the xsshunter-go Docker container
+      run: docker compose -f docker-compose.prod.yml down xsshunter-go
+      working-directory: ./
+    - name: Add DATABASE_URL to env
+      run: echo -e "\nDATABASE_URL=postgres://xsshunter:xsshunter@xsshunter-postgres:5432/xsshunter?sslmode=disable" >> .env
+      working-directory: ./
+    - name: Start the xsshunter-go Docker container
+      run: docker compose -f docker-compose.prod.yml --env-file .env up -d
+      working-directory: ./
+    - name: Wait for the xsshunter-go Docker container to start
+      run: sleep 10
+      working-directory: ./
+    - name: Get Password from Docker logs for postgres test
+      run: |
+        echo "TEMP_E2E_PLAYWRIGHT_PASSWORD=$(docker logs xsshunter-go-xsshunter-go-1 | grep -oP 'PASSWORD: \K.*')" > e2e/.env
+    - name: Run Playwright tests with the database
+      run: npx playwright test
+      working-directory: e2e

Dockerfile

@@ -14,7 +14,7 @@ ARG GIT_BRANCH
 RUN BUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%S%z') && \
     go build -ldflags "-X 'main.version=${GIT_TAG}' -X 'main.gitCommit=${GIT_COMMIT}' -X 'main.gitBranch=${GIT_BRANCH}' -X 'main.buildDate=${BUILD_DATE}'" -o main
 
-FROM alpine:3.20.0 as prod
+FROM alpine:3.20.1 as prod
 WORKDIR /app
 
 COPY --from=builder /app/main .

api.go

@@ -34,9 +34,6 @@ func settingsHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Not authenticated", http.StatusUnauthorized)
 	}
 	if r.Method == "GET" {
-		// db := establish_database_connection()
-		// defer db.Close()
-
 		rows, err := db.Query("SELECT key, value FROM settings WHERE key IN ($1, $2, $3, $4)", CORRELATION_API_SECRET_SETTINGS_KEY, CHAINLOAD_URI_SETTINGS_KEY, PAGES_TO_COLLECT_SETTINGS_KEY, SEND_ALERTS_SETTINGS_KEY)
 		if err != nil {
 			http.Error(w, "Error querying database", http.StatusInternalServerError)
@@ -136,9 +133,6 @@ func payloadFiresHandler(w http.ResponseWriter, r *http.Request) {
 		limit := parameter_to_int(limit_string, 10)
 		offset := page * limit
 
-		// db := establish_database_connection()
-		// defer db.Close()
-
 		rows, err := db.Query("SELECT id, url, ip_address, referer, user_agent, cookies, title, dom, text, origin, screenshot_id, was_iframe, browser_timestamp, injection_requests_id FROM payload_fire_results ORDER BY created_at DESC LIMIT $1 OFFSET $2", limit, offset)
 		if err != nil {
 			http.Error(w, "Error querying database", http.StatusInternalServerError)
@@ -168,8 +162,6 @@ func payloadFiresHandler(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, "No ids to delete", http.StatusBadRequest)
 			return
 		}
-		// db := establish_database_connection()
-		// defer db.Close()
 
 		rows, err := db.Query("SELECT screenshot_id FROM payload_fire_results WHERE id IN ($1)", ids_to_delete)
 		if err != nil {
@@ -214,9 +206,6 @@ func collectedPagesHandler(w http.ResponseWriter, r *http.Request) {
 		limit := parameter_to_int(limit_string, 10)
 		offset := page * limit
 
-		// db := establish_database_connection()
-		// defer db.Close()
-
 		rows, err := db.Query("SELECT id, uri FROM collected_pages ORDER BY created_at DESC LIMIT $1 OFFSET $2", limit, offset)
 		if err != nil {
 			http.Error(w, "Error querying database", http.StatusInternalServerError)
@@ -311,9 +300,6 @@ func userPayloadsHandler(w http.ResponseWriter, r *http.Request) {
 		// limit := parameter_to_int(limit_string, 10)
 		// offset := page * limit
 
-		// db := establish_database_connection()
-		// defer db.Close()
-
 		rows, err := db.Query("SELECT id, payload, title, description, author, author_link FROM user_xss_payloads ORDER BY created_at ASC")
 		if err != nil {
 			http.Error(w, "Error querying database", http.StatusInternalServerError)
@@ -339,8 +325,6 @@ func userPayloadsHandler(w http.ResponseWriter, r *http.Request) {
 		}
 
 	} else if r.Method == "POST" {
-		// db := establish_database_connection()
-		// defer db.Close()
 
 		stmt, _ := db.Prepare(`INSERT INTO user_xss_payloads (payload, title, description, author, author_link) VALUES ($1, $2, $3, $4, $5)`)
 		_, err := stmt.Exec(r.FormValue("payload"), r.FormValue("title"), r.FormValue("description"), r.FormValue("author"), r.FormValue("author_link"))
@@ -374,8 +358,6 @@ func userPayloadImporterHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
 		return
 	}
-	// db := establish_database_connection()
-	// defer db.Close()
 
 	var user_payloads []UserXSSPayloads
 	err := json.NewDecoder(r.Body).Decode(&user_payloads)

database.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"database/sql"
 	"fmt"
 	"log"
-	"database/sql"
 )
 
 type Settings struct {
@@ -45,9 +45,6 @@ type InjectionRequests struct {
 var db *sql.DB
 
 func create_sqlite_tables() {
-	// db := establish_sqlite_database_connection()
-	// defer db.Close()
-
 	sqlStmt := `
 	CREATE TABLE IF NOT EXISTS settings (
 		id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -105,9 +102,6 @@ func create_sqlite_tables() {
 }
 
 func create_postgres_tables() {
-	// db := establish_postgres_database_connection()
-	// defer db.Close()
-
 	sqlStmt := `
 	CREATE TABLE IF NOT EXISTS settings (
 		id SERIAL PRIMARY KEY,

dbhelper.go

@@ -26,8 +26,6 @@ type ResultsObject map[string]Result
 
 //lint:ignore U1000 Ignore unused function temporarily for debugging
 func db_select(query string, args ...any) (ResultsObjectArray, error) {
-	// db := establish_database_connection()
-	// defer db.Close()
 
 	rows, err := db.Query(query, args...)
 	if err != nil {
@@ -153,8 +151,6 @@ func toBool(value interface{}) (bool, error) {
 }
 
 func db_single_item_query(query string, args ...any) SingleResult {
-	// db := establish_database_connection()
-	// defer db.Close()
 
 	var result interface{}
 	err := db.QueryRow(query, args...).Scan(&result)
@@ -169,8 +165,6 @@ func db_single_item_query(query string, args ...any) SingleResult {
 }
 
 func db_prepare_execute(query string, args ...any) (sql.Result, error) {
-	// db := establish_database_connection()
-	// defer db.Close()
 
 	stmt, _ := db.Prepare(query)
 	result, err := stmt.Exec(args...)
@@ -182,8 +176,6 @@ func db_prepare_execute(query string, args ...any) (sql.Result, error) {
 }
 
 func db_execute(query string, args ...any) (sql.Result, error) {
-	// db := establish_database_connection()
-	// defer db.Close()
 
 	result, err := db.Exec(query, args...)
 	if err != nil {

docker-compose.prod.yml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   xsshunter-go:
     build: 
@@ -11,4 +10,22 @@ services:
     ports:
       - "1449:1449"
     env_file:
-      - .env
+      - .env
+    networks:
+      - xsshunter-go
+  xsshunter-postgres:
+      image: postgres:16.2-alpine3.19
+      environment:
+          POSTGRES_USER: xsshunter
+          POSTGRES_PASSWORD: xsshunter
+          POSTGRES_DB: xsshunter
+      volumes:
+          - ./postgres_db/:/var/lib/postgresql/data/
+      ports:
+          - "5432:5432"
+      networks:
+        - xsshunter-go
+
+networks:
+  xsshunter-go:
+    driver: bridge

e2e/package.json

@@ -8,8 +8,8 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@playwright/test": "1.44.1",
-    "@types/node": "20.14.5",
+    "@playwright/test": "1.45.1",
+    "@types/node": "20.14.10",
     "dotenv": "16.4.5"
   }
 }

go.mod

@@ -8,13 +8,13 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.22
-	golang.org/x/crypto v0.24.0
+	golang.org/x/crypto v0.25.0
 )
 
 require (
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -23,13 +23,13 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
 golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=

main.go

@@ -222,7 +222,6 @@ func jscallbackHandler(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		// payload_fire_id := uuid.New().String()
 		err = r.ParseForm()
 		if err != nil {
 			log.Fatal(err)
@@ -234,7 +233,6 @@ func jscallbackHandler(w http.ResponseWriter, r *http.Request) {
 			browser_time = 0
 		}
 		payload_fire_data := PayloadFireResults{
-			// ID:                 payload_fire_id,
 			Url:                   r.FormValue("uri"),
 			Ip_address:            ip_address,
 			Referer:               r.FormValue("referrer"),
@@ -255,9 +253,6 @@ func jscallbackHandler(w http.ResponseWriter, r *http.Request) {
 		if injection_key != "" {
 			query := "SELECT id, request FROM injection_requests WHERE injection_key = $1"
 
-			// db := establish_database_connection()
-			// defer db.Close()
-
 			rows, err := db.Query(query, injection_key)
 			if err != nil {
 				fmt.Println("Error getting injection request:", err)