From cb6e42ef02cbabdf3325aa05fdb27afdf3322a5e Mon Sep 17 00:00:00 2001 From: Haneen Hany <124837763+khalifaa55@users.noreply.github.com> Date: Tue, 15 Oct 2024 19:49:43 +0300 Subject: [PATCH] tests: Add e2e test cases for lido-exporter service integration * tests: Add e2e test cases for lido-exporter in sedge * tests: Update tests * tests: Update assertions * tests: Update tests * tests: Update assertions * tests: Add checks * tests: Update lido service test case * style: Update comments * style: Adjust comments --- cli/monitoring.go | 28 +++- e2e/sedge/checks.go | 41 +++++- e2e/sedge/monitoring_stack_test.go | 135 +++++++++++++++++- internal/monitoring/monitoring_test.go | 11 +- .../services/lido_exporter/dotenv.go | 2 +- .../services/lido_exporter/service.go | 1 + 6 files changed, 204 insertions(+), 14 deletions(-) diff --git a/cli/monitoring.go b/cli/monitoring.go index ae625e92..fddf0f9f 100644 --- a/cli/monitoring.go +++ b/cli/monitoring.go @@ -18,6 +18,7 @@ package cli import ( "errors" "fmt" + "math/big" "time" log "github.com/sirupsen/logrus" @@ -26,7 +27,7 @@ import ( "github.com/NethermindEth/sedge/internal/common" "github.com/NethermindEth/sedge/internal/monitoring" lidoExporter "github.com/NethermindEth/sedge/internal/monitoring/services/lido_exporter" - "github.com/NethermindEth/sedge/internal/ui" + "github.com/NethermindEth/sedge/internal/utils" ) func MonitoringCmd(mgr MonitoringManager) *cobra.Command { @@ -46,6 +47,11 @@ func InitSubCmd(mgr MonitoringManager) *cobra.Command { cmd := &cobra.Command{ Use: "init", Short: "Initialize the monitoring stack", + Long: `This command initializes the monitoring stack (Grafana, Prometheus, etc.) for Lido CSM or general node monitoring. + +The monitoring stack includes: +- Grafana dashboards for real-time monitoring of Lido CSM node metrics. +- Prometheus for collecting and displaying key metrics about your node operations.`, } cmd.AddCommand(DefaultSubCmd(mgr, additionalServices)) cmd.AddCommand(LidoSubCmd(mgr, additionalServices)) @@ -68,14 +74,26 @@ func LidoSubCmd(mgr MonitoringManager, additionalServices []monitoring.ServiceAP cmd := &cobra.Command{ Use: "lido", Short: "Configure Lido CSM Node monitoring", - Long: "Configure Lido CSM Node monitoring using Prometheus, Grafana, Node Exporter, and Lido Exporter", + Long: "Configure Lido CSM node monitoring (Prometheus, Grafana, Node Exporter,Lido Exporter)", Args: cobra.NoArgs, PreRunE: func(cmd *cobra.Command, args []string) error { if lido.NodeOperatorID == "" && lido.RewardAddress == "" { return errors.New("Node Operator ID or Reward Address is required") } - if err := ui.EthAddressValidator(rewardAddress, false); err != nil && len(args) != 0 { - return err + if lido.NodeOperatorID != "" { + var nodeOperatorIDBigInt *big.Int + var ok bool + nodeOperatorIDBigInt, ok = new(big.Int).SetString(lido.NodeOperatorID, 10) + if !ok { + return errors.New("Failed to convert Node Operator ID to big.Int") + } + if nodeOperatorIDBigInt.Sign() < 0 { + return errors.New("Node Operator ID cannot be negative") + } + } else { + if !utils.IsAddress(rewardAddress) { + return errors.New("Invalid reward address") + } } additionalServices = append(additionalServices, lidoExporter.NewLidoExporter(*lido)) return nil @@ -100,7 +118,7 @@ func DefaultSubCmd(mgr MonitoringManager, additionalServices []monitoring.Servic cmd := &cobra.Command{ Use: "default", Short: "Default monitoring configuration", - Long: "Default monitoring configuration using Prometheus, Grafana, and Node Exporter", + Long: "Default monitoring configuration (Prometheus, Grafana, Node Exporter)", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return InitMonitoring(true, true, mgr, nil) diff --git a/e2e/sedge/checks.go b/e2e/sedge/checks.go index 9d9be956..7cd4a8a3 100644 --- a/e2e/sedge/checks.go +++ b/e2e/sedge/checks.go @@ -66,7 +66,7 @@ func checkMonitoringStackNotInstalled(t *testing.T) { // checkMonitoringStackContainers checks that the monitoring stack containers are running func checkMonitoringStackContainers(t *testing.T, containerNames ...string) { t.Logf("Checking monitoring stack containers") - containerNames = append(containerNames, "sedge_grafana", "sedge_prometheus", "sedge_node_exporter") + containerNames = append(containerNames, "sedge_grafana", "sedge_prometheus", "sedge_node_exporter", "sedge_alertmanager") checkContainerRunning(t, containerNames...) } @@ -168,7 +168,7 @@ func checkGrafanaHealth(t *testing.T) { // checkMonitoringStackContainersNotRunning checks that the monitoring stack containers are not running func checkMonitoringStackContainersNotRunning(t *testing.T, containerNames ...string) { t.Logf("Checking monitoring stack containers are not running") - containerNames = append(containerNames, "sedge_grafana", "sedge_prometheus", "sedge_node_exporter") + containerNames = append(containerNames, "sedge_grafana", "sedge_prometheus", "sedge_node_exporter", "sedge_alertmanager") checkContainerNotExisting(t, containerNames...) } @@ -191,3 +191,40 @@ func checkContainerNotExisting(t *testing.T, containerNames ...string) { assert.Error(t, err) } } + +// checkMonitoringStackDir checks that the monitoring stack directory exists and contains the docker-compose file +func checkPrometheusDir(t *testing.T) { + t.Logf("Checking prometheus directory") + // Check monitoring folder exists + dataDir, err := dataDirPath() + if err != nil { + t.Fatal(err) + } + prometheusDir := filepath.Join(dataDir, "monitoring", "prometheus") + assert.DirExists(t, prometheusDir) + + assert.DirExists(t, filepath.Join(prometheusDir, "rules")) + // Check monitoring docker-compose file exists + assert.FileExists(t, filepath.Join(prometheusDir, "alertmanager", "alertmanager.yml")) +} + +// checkContainerRunning checks that the given containers are running +func checkContainerNotRunning(t *testing.T, containerNames ...string) { + cli, err := client.NewClientWithOpts( + client.FromEnv, + client.WithAPIVersionNegotiation(), + ) + if err != nil { + t.Fatalf("Failed to create Docker client: %v", err) + } + defer cli.Close() + + dockerServiceManager := services.NewDockerServiceManager(cli) + + for _, containerName := range containerNames { + t.Logf("Checking %s container is not running", containerName) + isRunning, err := dockerServiceManager.IsRunning(containerName) + require.NoError(t, err) + assert.False(t, isRunning, "%s container should not be running", containerName) + } +} diff --git a/e2e/sedge/monitoring_stack_test.go b/e2e/sedge/monitoring_stack_test.go index f166e166..62d0e6be 100644 --- a/e2e/sedge/monitoring_stack_test.go +++ b/e2e/sedge/monitoring_stack_test.go @@ -29,6 +29,8 @@ func skipIfNotAMD64(t *testing.T) { } } +var grafanaOnCallContainers = []string{"engine", "celery", "redis", "oncall_setup"} + // TestMonitoringStack_Init tests that the monitoring stack is not initialized if the user does not run the init-monitoring command func TestE2E_MonitoringStack_NotInitialized(t *testing.T) { skipIfNotAMD64(t) @@ -50,7 +52,7 @@ func TestE2E_MonitoringStack_NotInitialized(t *testing.T) { assert.NoError(t, runErr) checkMonitoringStackNotInstalled(t) - checkMonitoringStackContainersNotRunning(t) + checkMonitoringStackContainersNotRunning(t, grafanaOnCallContainers...) }, ) // Run test case @@ -77,7 +79,8 @@ func TestE2E_MonitoringStack_Init(t *testing.T) { func(t *testing.T, dataDirPath string) { assert.NoError(t, runErr) checkMonitoringStackDir(t) - checkMonitoringStackContainers(t) + checkPrometheusDir(t) + checkMonitoringStackContainers(t, grafanaOnCallContainers...) checkPrometheusTargetsUp(t, "sedge_node_exporter:9100") checkGrafanaHealth(t) }, @@ -124,6 +127,7 @@ func TestE2E_MonitoringStack_NotReinstalled(t *testing.T) { assert.NoError(t, runErr) checkMonitoringStackDir(t) + checkPrometheusDir(t) checkMonitoringStackContainers(t) checkGrafanaHealth(t) newGrafanaContainerID, err := getContainerIDByName("sedge_grafana") @@ -166,7 +170,7 @@ func TestE2E_MonitoringStack_Clean(t *testing.T) { assert.NoDirExists(t, dataDirPath) // Check that monitoring stack containers are removed - checkMonitoringStackContainersNotRunning(t) + checkMonitoringStackContainersNotRunning(t, grafanaOnCallContainers...) }, ) // Run test case @@ -196,14 +200,14 @@ func TestE2E_MonitoringStack_CleanNonExistent(t *testing.T) { assert.NoDirExists(t, dataDirPath) // Check that monitoring stack containers don't exist - checkMonitoringStackContainersNotRunning(t) + checkMonitoringStackContainersNotRunning(t, grafanaOnCallContainers...) }, ) // Run test case e2eTest.run() } -func TestE2E_MonitoringStack_InitLido(t *testing.T) { +func TestE2E_MonitoringStack_InitLido_ValidID(t *testing.T) { skipIfNotAMD64(t) // Test context var ( @@ -222,6 +226,7 @@ func TestE2E_MonitoringStack_InitLido(t *testing.T) { func(t *testing.T, dataDirPath string) { assert.NoError(t, runErr) checkMonitoringStackDir(t) + checkPrometheusDir(t) checkMonitoringStackContainers(t, "sedge_lido_exporter") checkPrometheusTargetsUp(t, "sedge_lido_exporter:8080", "sedge_node_exporter:9100") checkGrafanaHealth(t) @@ -262,3 +267,123 @@ func TestE2E_MonitoringStack_CleanLido(t *testing.T) { // Run test case e2eTest.run() } + +func TestE2E_MonitoringStack_InitLido_InvalidAddress(t *testing.T) { + skipIfNotAMD64(t) + // Test context + var ( + runErr error + ) + // Build test case + e2eTest := newE2ESedgeTestCase( + t, + // Arrange + func(t *testing.T, sedgePath string) error { + return base.RunCommand(t, sedgePath, "sedge", "monitoring", "clean") + }, + // Act + func(t *testing.T, binaryPath string, dataDirPath string) { + runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init", "lido", "--reward-address", "lol_what_a_reward_address") + }, + // Assert + func(t *testing.T, dataDirPath string) { + assert.Error(t, runErr) + + checkMonitoringStackNotInstalled(t) + checkMonitoringStackContainersNotRunning(t, grafanaOnCallContainers...) + }, + ) + // Run test case + e2eTest.run() +} + +func TestE2E_MonitoringStack_InitLido_OccupiedPort(t *testing.T) { + skipIfNotAMD64(t) + // Test context + var ( + runErr error + ) + // Build test case + e2eTest := newE2ESedgeTestCase( + t, + // Arrange + nil, + // Act + func(t *testing.T, binaryPath string, dataDirPath string) { + runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init", "lido", "--node-operator-id", "10", "--port", "9090") + }, + // Assert + func(t *testing.T, dataDirPath string) { + assert.Error(t, runErr) + checkContainerNotRunning(t, "sedge_lido_exporter") + }, + ) + // Run test case + e2eTest.run() +} + +func TestE2E_MonitoringStack_InitLido(t *testing.T) { + skipIfNotAMD64(t) + // Test context + var ( + runErr error + ) + // Build test case + e2eTest := newE2ESedgeTestCase( + t, + // Arrange + nil, + // Act + func(t *testing.T, binaryPath string, dataDirPath string) { + runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init", "lido", + "--rpc-endpoints", "https://endpoints.omniatech.io/v1/eth/holesky/public,https://ethereum-holesky-rpc.publicnode.com", + "--ws-endpoints", "https://ethereum-holesky-rpc.publicnode.com,wss://ethereum-holesky-rpc.publicnode.com", + "--port", "9989", + "--scrape-time", "30s", + "--network", "holesky", + "--node-operator-id", "250", + "--reward-address", "0x22bA5CaFB5E26E6Fe51f330294209034013A5A4c", + ) + }, + // Assert + func(t *testing.T, dataDirPath string) { + assert.NoError(t, runErr) + checkMonitoringStackDir(t) + checkPrometheusDir(t) + checkMonitoringStackContainers(t, "sedge_lido_exporter") + checkPrometheusTargetsUp(t, "sedge_lido_exporter:9989", "sedge_node_exporter:9100") + checkGrafanaHealth(t) + }, + ) + // Run test case + e2eTest.run() +} + +func TestE2E_MonitoringStack_InitLido_InvalidNodeID(t *testing.T) { + skipIfNotAMD64(t) + // Test context + var ( + runErr error + ) + // Build test case + e2eTest := newE2ESedgeTestCase( + t, + // Arrange + func(t *testing.T, sedgePath string) error { + return base.RunCommand(t, sedgePath, "sedge", "monitoring", "clean") + }, + // Act + func(t *testing.T, binaryPath string, dataDirPath string) { + runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init", "lido", "--node-operator-id", "-1") + }, + // Assert + func(t *testing.T, dataDirPath string) { + assert.Error(t, runErr) + + checkMonitoringStackNotInstalled(t) + checkMonitoringStackContainersNotRunning(t, grafanaOnCallContainers...) + }, + ) + // Run test case + e2eTest.run() +} diff --git a/internal/monitoring/monitoring_test.go b/internal/monitoring/monitoring_test.go index bd0d8935..ccc1a3b4 100644 --- a/internal/monitoring/monitoring_test.go +++ b/internal/monitoring/monitoring_test.go @@ -24,6 +24,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sort" "strconv" "strings" "testing" @@ -1974,7 +1975,15 @@ func TestUpdateEnvFile(t *testing.T) { assert.NoError(t, err) content, err := afero.ReadFile(fs, filepath.Join(manager.stack.Path(), ".env")) assert.NoError(t, err) - assert.Equal(t, tt.expectedEnv, string(content)) + + // Normalize and sort the lines for comparison + expectedLines := strings.Split(strings.TrimSpace(tt.expectedEnv), "\n") + actualLines := strings.Split(strings.TrimSpace(string(content)), "\n") + + sort.Strings(expectedLines) + sort.Strings(actualLines) + + assert.Equal(t, expectedLines, actualLines) } }) } diff --git a/internal/monitoring/services/lido_exporter/dotenv.go b/internal/monitoring/services/lido_exporter/dotenv.go index 3796911f..2455bd80 100644 --- a/internal/monitoring/services/lido_exporter/dotenv.go +++ b/internal/monitoring/services/lido_exporter/dotenv.go @@ -17,7 +17,7 @@ package lido_exporter var dotEnv map[string]string = map[string]string{ "LIDO_EXPORTER_IMAGE": "nethermindeth/lido-exporter:v1.0.1", - "LIDO_EXPORTER_PORT": "8080", + "LIDO_EXPORTER_PORT": "", "LIDO_EXPORTER_NODE_OPERATOR_ID": "", "LIDO_EXPORTER_REWARD_ADDRESS": "", "LIDO_EXPORTER_NETWORK": "", diff --git a/internal/monitoring/services/lido_exporter/service.go b/internal/monitoring/services/lido_exporter/service.go index 76cf2600..115e328f 100644 --- a/internal/monitoring/services/lido_exporter/service.go +++ b/internal/monitoring/services/lido_exporter/service.go @@ -53,6 +53,7 @@ func NewLidoExporter(params LidoExporterParams) *LidoExporterService { dotEnv["LIDO_EXPORTER_WS_ENDPOINTS"] = strings.Join(params.WSEndpoints, ",") dotEnv["LIDO_EXPORTER_SCRAPE_TIME"] = params.ScrapeTime.String() dotEnv["LIDO_EXPORTER_LOG_LEVEL"] = params.LogLevel + dotEnv["LIDO_EXPORTER_PORT"] = strconv.Itoa(int(params.Port)) return &LidoExporterService{ params: params,