Gitlab Target Reconciliation

.github/workflows/test_sast.yml

@@ -1,6 +1,7 @@
 name: Test - SAST
 
 on:
+  push:
   pull_request:
 
 permissions:

.github/workflows/test_unit.yml

@@ -23,4 +23,4 @@ jobs:
       - name: Test
         run: |
           go mod download
-          go test --race --coverprofile cover.out -v ./...
+          go test --race --count=1 --coverprofile cover.out -v ./...

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
     hooks:
       - id: go-mod-tidy-repo
       - id: go-test-repo-mod
-        args: [ -race ]
+        args: [ -race, -count=1 ]
       - id: go-vet-repo-mod
       - id: go-fumpt-repo
         args: [ -l, -w ]

.vscode/settings.json

@@ -1,6 +1,7 @@
 {
     "go.testFlags": [
         "-race",
-        "-cover"
+        "-cover",
+        "-count=1"
     ]
 }

README.md

@@ -8,16 +8,17 @@
 
 - [About this component](#about-this-component)
 - [Installation](#installation)
-  - [Binary](#binary)
-  - [Container Image](#container-image)
-  - [Helm](#helm)
+    - [Binary](#binary)
+    - [Container Image](#container-image)
+    - [Helm](#helm)
 - [Usage](#usage)
-  - [Container Image](#container-image-1)
+    - [Container Image](#container-image-1)
 - [Configuration](#configuration)
-  - [Startup](#startup)
-    - [Loader](#loader)
-  - [Runtime](#runtime)
-  - [Check: Health](#check-health)
+    - [Startup](#startup)
+        - [Loader](#loader)
+    - [Runtime](#runtime)
+    - [TargetManager](#targetmanager)
+    - [Check: Health](#check-health)
     - [Health Metrics](#health-metrics)
   - [Check: Latency](#check-latency)
     - [Latency Metrics](#latency-metrics)
@@ -38,7 +39,8 @@ The `sparrow` performs several checks to monitor the health of the infrastructur
 The following checks are available:
 
 1. Health check - `health`: The `sparrow` is able to perform an HTTP-based (HTTP/1.1) health check to the provided
-   endpoints. The `sparrow` will expose its own health check endpoint as well.
+   endpoints.
+   The `sparrow` will expose its own health check endpoint as well.
 
 2. Latency check - `latency`: The `sparrow` is able to communicate with other `sparrow` instances to calculate the time
    a request takes to the target and back. The check is http (HTTP/1.1) based as well.
@@ -168,6 +170,35 @@ checks:
     enabled: true
 ```
 
+### Target Manager
+
+The `sparrow` is able to manage the targets for the checks and register the `sparrow` as target on a (remote) backend.
+This is done via a `TargetManager` interface, which can be configured on startup. The available configuration options
+are listed below and can be set in a startup YAML configuration file (per default `tmconfig.yaml` in the current
+directory).
+
+| Type                                 | Description                                                                          | Default              |
+|--------------------------------------|--------------------------------------------------------------------------------------|----------------------|
+| `targetManager.checkInterval`        | The interval in seconds to check for new targets.                                    | `300`                |
+| `targetManager.unhealthyThreshold`   | The threshold in seconds to mark a target as unhealthy and remove it from the state. | `600`                |
+| `targetManager.registrationInterval` | The interval in seconds to register the current sparrow at the targets backend.      | `300`                |
+| `targetManager.gitlab.token`         | The token to authenticate against the gitlab instance.                               | `""`                 |
+| `targetManager.gitlab.baseUrl`       | The base URL of the gitlab instance.                                                 | `https://gitlab.com` |
+| `targetManager.gitlab.projectId`     | The project ID of the gitlab project to use as a remote state backend.               | `""`                 |
+
+Currently, only one target manager exists: the Gitlab target manager. It uses a gitlab project as the remote state
+backend. The various `sparrow` instances will
+register themselves as targets in the project. The `sparrow` instances will also check the project for new targets and
+add them to the local state. The registration is done by committing a "state" file in the main branch of the repository,
+which is named after the DNS name of the `sparrow`. The state file contains the following information:
+
+```json
+{
+  "url": "https://<SPARROW_DNS_NAME>",
+  "lastSeen": "2021-09-30T12:00:00Z"
+}
+```
+
 ### Check: Health
 
 Available configuration options:

cmd/gen-docs.go

@@ -25,7 +25,7 @@ import (
 	"github.com/spf13/cobra/doc"
 )
 
-// NewCmdRun creates a new gen-docs command
+// NewCmdGenDocs creates a new gen-docs command
 func NewCmdGenDocs(rootCmd *cobra.Command) *cobra.Command {
 	var docPath string
 

cmd/run.go

@@ -48,6 +48,7 @@ func NewCmdRun() *cobra.Command {
 		LoaderHttpRetryCount: "loaderHttpRetryCount",
 		LoaderHttpRetryDelay: "loaderHttpRetryDelay",
 		LoaderFilePath:       "loaderFilePath",
+		TargetManagerConfig:  "tmconfig",
 	}
 
 	cmd := &cobra.Command{
@@ -67,6 +68,7 @@ func NewCmdRun() *cobra.Command {
 	cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryCount, defaultHttpRetryCount, "http loader: Amount of retries trying to load the configuration")
 	cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryDelay, defaultHttpRetryDelay, "http loader: The initial delay between retries in seconds")
 	cmd.PersistentFlags().String(flagMapping.LoaderFilePath, "config.yaml", "file loader: The path to the file to read the runtime config from")
+	cmd.PersistentFlags().String(flagMapping.TargetManagerConfig, "tmconfig.yaml", "target manager: The path to the file to read the target manager config from")
 
 	_ = viper.BindPFlag(flagMapping.ApiAddress, cmd.PersistentFlags().Lookup(flagMapping.ApiAddress))
 	_ = viper.BindPFlag(flagMapping.LoaderType, cmd.PersistentFlags().Lookup(flagMapping.LoaderType))
@@ -77,6 +79,7 @@ func NewCmdRun() *cobra.Command {
 	_ = viper.BindPFlag(flagMapping.LoaderHttpRetryCount, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryCount))
 	_ = viper.BindPFlag(flagMapping.LoaderHttpRetryDelay, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryDelay))
 	_ = viper.BindPFlag(flagMapping.LoaderFilePath, cmd.PersistentFlags().Lookup(flagMapping.LoaderFilePath))
+	_ = viper.BindPFlag(flagMapping.TargetManagerConfig, cmd.PersistentFlags().Lookup(flagMapping.TargetManagerConfig))
 
 	return cmd
 }
@@ -88,6 +91,7 @@ func run(fm *config.RunFlagsNameMapping) func(cmd *cobra.Command, args []string)
 		ctx := logger.IntoContext(context.Background(), log)
 
 		cfg := config.NewConfig()
+		cfg.SetTargetManagerConfig(config.NewTargetManagerConfig(viper.GetString(fm.TargetManagerConfig)))
 
 		cfg.SetApiAddress(viper.GetString(fm.ApiAddress))
 

docs/sparrow_run.md

@@ -23,6 +23,7 @@ sparrow run [flags]
       --loaderHttpUrl string       http loader: The url where to get the remote configuration
       --loaderInterval int         defines the interval the loader reloads the configuration in seconds (default 300)
   -l, --loaderType string          defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader (default "http")
+      --tmconfig string            target manager: The path to the file to read the target manager config from (default "tmconfig.yaml")
 ```
 
 ### Options inherited from parent commands

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/getkin/kin-openapi v0.120.0
 	github.com/go-chi/chi/v5 v5.0.10
+	github.com/go-test/deep v1.0.8
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/cobra v1.8.0

pkg/checks/checks.go

@@ -77,6 +77,13 @@ type Result struct {
 	Err string `json:"error"`
 }
 
+// GlobalTarget includes the basic information regarding
+// other Sparrow instances, which this Sparrow can communicate with.
+type GlobalTarget struct {
+	Url      string    `json:"url"`
+	LastSeen time.Time `json:"lastSeen"`
+}
+
 type ResultDTO struct {
 	Name   string
 	Result *Result

pkg/config/config.go

@@ -19,15 +19,32 @@
 package config
 
 import (
+	"os"
 	"time"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/caas-team/sparrow/internal/helper"
 )
 
+type GitlabTargetManagerConfig struct {
+	BaseURL   string `yaml:"baseUrl"`
+	Token     string `yaml:"token"`
+	ProjectID int    `yaml:"projectId"`
+}
+
+type TargetManagerConfig struct {
+	CheckInterval        time.Duration             `yaml:"checkInterval"`
+	RegistrationInterval time.Duration             `yaml:"registrationInterval"`
+	UnhealthyThreshold   time.Duration             `yaml:"unhealthyThreshold"`
+	Gitlab               GitlabTargetManagerConfig `yaml:"gitlab"`
+}
+
 type Config struct {
-	Checks map[string]any
-	Loader LoaderConfig
-	Api    ApiConfig
+	Checks        map[string]any
+	Loader        LoaderConfig
+	Api           ApiConfig
+	TargetManager TargetManagerConfig
 }
 
 // ApiConfig is the configuration for the data API
@@ -56,6 +73,23 @@ type FileLoaderConfig struct {
 	path string
 }
 
+// NewTargetManagerConfig creates a new TargetManagerConfig
+// from the passed file
+func NewTargetManagerConfig(path string) TargetManagerConfig {
+	var res TargetManagerConfig
+	f, err := os.ReadFile(path) //#nosec G304
+	if err != nil {
+		panic("failed to read config file " + err.Error())
+	}
+
+	err = yaml.Unmarshal(f, &res)
+	if err != nil {
+		panic("failed to parse config file: " + err.Error())
+	}
+
+	return res
+}
+
 // NewConfig creates a new Config
 func NewConfig() *Config {
 	return &Config{
@@ -108,3 +142,8 @@ func (c *Config) SetLoaderHttpRetryCount(retryCount int) {
 func (c *Config) SetLoaderHttpRetryDelay(retryDelay int) {
 	c.Loader.http.retryCfg.Delay = time.Duration(retryDelay) * time.Second
 }
+
+// SetTargetManagerConfig sets the target manager config
+func (c *Config) SetTargetManagerConfig(config TargetManagerConfig) {
+	c.TargetManager = config
+}

pkg/config/config_test.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/go-test/deep"
+)
+
+func Test_NewTargetManagerConfig_Gitlab(t *testing.T) {
+	got := NewTargetManagerConfig("testdata/tmconfig.yaml")
+	want := TargetManagerConfig{
+		CheckInterval:        300 * time.Second,
+		RegistrationInterval: 600 * time.Second,
+		UnhealthyThreshold:   900 * time.Second,
+		Gitlab: GitlabTargetManagerConfig{
+			BaseURL:   "https://gitlab.devops.telekom.de",
+			ProjectID: 666,
+			Token:     "gitlab-token",
+		},
+	}
+
+	if diff := deep.Equal(got, want); diff != nil {
+		t.Error(diff)
+	}
+}

pkg/config/flags.go

@@ -29,4 +29,6 @@ type RunFlagsNameMapping struct {
 	LoaderHttpRetryCount string
 	LoaderHttpRetryDelay string
 	LoaderFilePath       string
+
+	TargetManagerConfig string
 }

pkg/config/http.go

@@ -42,7 +42,7 @@ func NewHttpLoader(cfg *Config, cCfgChecks chan<- map[string]any) *HttpLoader {
 	}
 }
 
-// GetRuntimeConfig gets the runtime configuration
+// Run gets the runtime configuration
 // from the http remote endpoint.
 // The config is will be loaded periodically defined by the
 // loader interval configuration. A failed request will be retried defined

pkg/config/testdata/tmconfig.yaml

@@ -0,0 +1,7 @@
+checkInterval: 300s
+registrationInterval: 600s
+unhealthyThreshold: 900s
+gitlab:
+    token: gitlab-token
+    baseUrl: https://gitlab.devops.telekom.de
+    projectId: 666