Add kustomization to e2e suite

Add a dedicated kustomization overlay (config/test-e2e) that includes
Prometheus, allowing e2e tests to use 'make deploy-e2e' without
modifying config/default/kustomization.yaml.

Signed-off-by: Pujol <enric.pujol@sap.com>

Author: nikatza

Makefile

@@ -125,6 +125,11 @@ deploy: FORCE generate install-kustomize
 	@printf "\e[1;36m>> kustomize build config/default | kubectl apply -f -\e[0m\n"
 	@kustomize build config/default | kubectl apply -f -
 
+# Deploy controller for e2e tests (includes Prometheus)
+deploy-e2e: FORCE generate install-kustomize
+	@printf "\e[1;36m>> kustomize build config/test-e2e | kubectl apply -f -\e[0m\n"
+	@kustomize build config/test-e2e | kubectl apply -f -
+
 # Undeploy controller from the k8s cluster
 undeploy: FORCE install-kustomize
 	@printf "\e[1;36m>> kustomize build config/default | kubectl delete -f -\e[0m\n"

config/test-e2e/kustomization.yaml

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: network-operator-system
+
+resources:
+- ../default
+- ../prometheus

test/e2e/e2e_suite_test.go

@@ -30,10 +30,6 @@ var (
 // with the code source changes to be tested.
 const image = "ghcr.io/ironcore-dev/network-operator:latest"
 
-// serverImage is the name of the image which will be built and loaded
-// with the gNMI test server.
-const serverImage = "ghcr.io/ironcore-dev/gnmi-test-server:latest"
-
 // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
 // temporary environment to validate project changes with the purposed to be used in CI jobs.
 // The default setup requires Kind, builds/loads the Manager Docker image locally, and installs
@@ -45,31 +41,15 @@ func TestE2E(t *testing.T) {
 }
 
 var _ = BeforeSuite(func(ctx SpecContext) {
-	By("Ensure that Prometheus is enabled")
-	cwd, err := GetProjectDir()
-	Expect(err).NotTo(HaveOccurred(), "Failed to get project directory")
-
-	err = UncommentCode(cwd+"/config/default/kustomization.yaml", "#- ../prometheus", "#")
-	Expect(err).NotTo(HaveOccurred(), "Failed to enable Prometheus")
-
 	By("building the manager(Operator) image")
 	cmd := exec.CommandContext(ctx, "make", "docker-build", "IMG="+image)
-	_, err = Run(cmd)
+	_, err := Run(cmd)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")
 
 	By("loading the manager(Operator) image on Kind")
 	err = LoadImageToKindClusterWithName(ctx, image)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")
 
-	By("building the gnmi-test-server image")
-	cmd = exec.CommandContext(ctx, "make", "docker-build-test-gnmi-server", "TEST_SERVER_IMG="+serverImage)
-	_, err = Run(cmd)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the gnmi-test-server image")
-
-	By("loading the gnmi-test-server image on Kind")
-	err = LoadImageToKindClusterWithName(ctx, serverImage)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the gnmi-test-server image into Kind")
-
 	// The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
 	// To prevent errors when tests run in environments with Prometheus or CertManager already installed,
 	// we check for their presence before execution.

test/e2e/e2e_test.go

@@ -10,13 +10,11 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"slices"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"golang.org/x/tools/txtar"
 )
 
 // namespace where the project is deployed in
@@ -33,45 +31,17 @@ const metricsRoleBindingName = "network-operator-metrics-binding"
 
 var _ = Describe("Manager", Ordered, func() {
 	var controllerPodName string
-	var gnmiServerIPAddr string
 
 	// Before running the tests, set up the environment by creating the namespace,
 	// enforce the restricted security policy to the namespace, installing CRDs,
 	// and deploying the controller.
 	BeforeAll(func(ctx SpecContext) {
-		By("deploying the gnmi-test-server")
-		cmd := exec.CommandContext(
-			ctx, "kubectl", "run", "gnmi-test-server",
-			"--image", serverImage,
-			"--image-pull-policy", "Never",
-			"--namespace", "default",
-			"--restart", "Never",
-			"--port", "8000",
-			"--port", "9339",
-		)
-		_, err := Run(cmd)
-		Expect(err).NotTo(HaveOccurred(), "Failed to deploy the gnmi-test-server")
-
-		cmd = exec.CommandContext(
-			ctx, "kubectl", "wait", "pods/gnmi-test-server",
-			"--for", "condition=Ready",
-			"--namespace", "default",
-			"--timeout", "1m",
-		)
-		_, err = Run(cmd)
-		Expect(err).NotTo(HaveOccurred())
-
-		cmd = exec.CommandContext(
-			ctx, "kubectl", "get", "pod", "gnmi-test-server",
-			"--output", "jsonpath='{.status.podIP}'",
-			"--namespace", "default",
-		)
+		By("creating manager namespace")
+		cmd := exec.CommandContext(ctx, "kubectl", "create", "ns", namespace, "--dry-run=client", "-o", "yaml")
 		out, err := Run(cmd)
 		Expect(err).NotTo(HaveOccurred())
-		gnmiServerIPAddr = strings.ReplaceAll(strings.TrimSpace(out), "'", "")
-
-		By("creating manager namespace")
-		cmd = exec.CommandContext(ctx, "kubectl", "create", "ns", namespace)
+		cmd = exec.CommandContext(ctx, "kubectl", "apply", "-f", "-")
+		cmd.Stdin = strings.NewReader(out)
 		_, err = Run(cmd)
 		Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
 
@@ -86,21 +56,25 @@ var _ = Describe("Manager", Ordered, func() {
 		Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
 
 		By("deploying the controller-manager")
-		cmd = exec.CommandContext(ctx, "make", "deploy")
-		_, err = Run(cmd)
-		Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
+		// Retry deploy because cert-manager webhook may not be ready immediately
+		// after its deployment is Available (TLS certificate propagation delay).
+		Eventually(func() error {
+			cmd = exec.CommandContext(ctx, "make", "deploy-e2e")
+			_, err = Run(cmd)
+			return err
+		}).WithTimeout(2*time.Minute).WithPolling(10*time.Second).Should(Succeed(), "Failed to deploy the controller-manager")
 	})
 
 	// After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs,
 	// and deleting the namespace.
 	AfterAll(func(ctx SpecContext) {
 		By("cleaning up the ClusterRoleBinding of the service account to allow access to metrics")
-		cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterrolebinding", metricsRoleBindingName)
+		cmd := exec.CommandContext(ctx, "kubectl", "delete", "clusterrolebinding", metricsRoleBindingName, "--ignore-not-found")
 		_, err := Run(cmd)
 		Expect(err).NotTo(HaveOccurred(), "Failed to delete ClusterRoleBinding")
 
 		By("cleaning up the curl pod for metrics")
-		cmd = exec.CommandContext(ctx, "kubectl", "delete", "pod", "curl-metrics", "-n", namespace)
+		cmd = exec.CommandContext(ctx, "kubectl", "delete", "pod", "curl-metrics", "-n", namespace, "--ignore-not-found")
 		_, err = Run(cmd)
 		Expect(err).NotTo(HaveOccurred(), "Failed to delete curl-metrics pod")
 
@@ -118,11 +92,6 @@ var _ = Describe("Manager", Ordered, func() {
 		cmd = exec.CommandContext(ctx, "kubectl", "delete", "ns", namespace, "--ignore-not-found")
 		_, err = Run(cmd)
 		Expect(err).NotTo(HaveOccurred(), "Failed to delete namespace")
-
-		By("cleaning up the gnmi-test-server pod")
-		cmd = exec.CommandContext(ctx, "kubectl", "delete", "pod", "gnmi-test-server", "-n", "default")
-		_, err = Run(cmd)
-		Expect(err).NotTo(HaveOccurred(), "Failed to delete gnmi-test-server pod")
 	})
 
 	// After each test, check for failures and collect logs, events,
@@ -311,113 +280,6 @@ var _ = Describe("Manager", Ordered, func() {
 		})
 
 		// +kubebuilder:scaffold:e2e-webhooks-checks
-
-		// TODO: Customize the e2e test suite with scenarios specific to your project.
-		// Consider applying sample/CR(s) and check their status and/or verifying
-		// the reconciliation by using the metrics, i.e.:
-		// metricsOutput := getMetricsOutput()
-		// Expect(metricsOutput).To(ContainSubstring(
-		//    fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
-		//    strings.ToLower(<Kind>),
-		// ))
-
-		DescribeTable(
-			"Should reconcile the api objects",
-			func(ctx SpecContext, file string, numFiles int) {
-				device := `
-apiVersion: networking.metal.ironcore.dev/v1alpha1
-kind: Device
-metadata:
-  name: device
-  namespace: default
-spec:
-  endpoint:
-    address: "%s"`
-				err := Apply(ctx, fmt.Sprintf(device, gnmiServerIPAddr+":9339"))
-				Expect(err).NotTo(HaveOccurred(), "Failed to apply Device")
-
-				dir, err := GetProjectDir()
-				Expect(err).NotTo(HaveOccurred(), "Failed to get project directory")
-
-				a, err := txtar.ParseFile(filepath.Join(dir, "test", "e2e", "testdata", file))
-				Expect(err).NotTo(HaveOccurred(), "Failed to parse test file")
-				Expect(a.Files).To(HaveLen(numFiles), "Unexpected number of files in the test archive")
-
-				// All sections except the last are resource manifests; last is expected state.
-				resources := a.Files[:len(a.Files)-1]
-				stateFile := a.Files[len(a.Files)-1]
-
-				for _, res := range resources {
-					err = Apply(ctx, string(res.Data))
-					Expect(err).NotTo(HaveOccurred(), "Failed to apply resource %s", res.Name)
-
-					// Determine wait condition from resource type prefix.
-					// vlans/ have no provider in openconfig — skip wait.
-					var condition string
-					switch {
-					case strings.HasPrefix(res.Name, "banners/"):
-						condition = "Ready"
-					case strings.HasPrefix(res.Name, "vlans/"):
-						continue
-					default:
-						condition = "Configured"
-					}
-
-					// #nosec G204
-					cmd := exec.CommandContext(
-						ctx, "kubectl", "wait", res.Name,
-						"--for", "condition="+condition,
-						"--namespace", "default",
-						"--timeout", "5m",
-					)
-					_, err = Run(cmd)
-					Expect(err).NotTo(HaveOccurred())
-				}
-
-				cmd := exec.CommandContext(
-					ctx, "kubectl", "exec", "gnmi-test-server",
-					"--namespace", "default",
-					"--",
-					"wget", "-qO-", "http://localhost:8000/v1/state",
-				)
-				got, err := Run(cmd)
-				Expect(err).NotTo(HaveOccurred(), "Failed to execute command on gnmi-test-server")
-
-				err = CompareJSON(got, string(stateFile.Data))
-				Expect(err).NotTo(HaveOccurred(), "State output does not match expected JSON")
-
-				// Delete resources in reverse order.
-				for _, res := range slices.Backward(resources) {
-					// #nosec G204
-					cmd = exec.CommandContext(ctx, "kubectl", "delete", res.Name)
-					_, err = Run(cmd)
-					Expect(err).NotTo(HaveOccurred(), "Failed to delete object")
-				}
-
-				cmd = exec.CommandContext(ctx, "kubectl", "delete", "devices/device", "--cascade=foreground")
-				_, err = Run(cmd)
-				Expect(err).NotTo(HaveOccurred(), "Failed to delete object")
-
-				cmd = exec.CommandContext(
-					ctx, "kubectl", "exec", "gnmi-test-server",
-					"--namespace", "default",
-					"--",
-					"wget", "-qO-", "--header", "X-HTTP-Method-Override: DELETE", "http://localhost:8000/v1/state",
-				)
-				_, err = Run(cmd)
-				Expect(err).NotTo(HaveOccurred(), "Failed to execute command on gnmi-test-server")
-			},
-			Entry("Loopback Interface", "interface.txt", 2),
-			Entry("Loopback Multi-Address", "interface_loopback_multi_addr.txt", 2),
-			Entry("Physical IPv4 Address", "interface_physical_ipv4.txt", 2),
-			Entry("Physical Unnumbered", "interface_physical_unnumbered.txt", 3),
-			Entry("Physical Switchport Access", "interface_physical_switchport_access.txt", 2),
-			Entry("Physical Switchport Trunk", "interface_physical_switchport_trunk.txt", 2),
-			Entry("Aggregate L2 Trunk", "interface_aggregate_l2_trunk.txt", 3),
-			Entry("Aggregate L3 Address", "interface_aggregate_l3.txt", 3),
-			Entry("Routed VLAN", "interface_routed_vlan.txt", 3),
-			Entry("Banner PreLogin", "banner.txt", 2),
-		)
 	})
 })
 

test/e2e/util_test.go

@@ -4,7 +4,6 @@
 package e2e
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -164,17 +163,19 @@ func InstallCertManager(ctx context.Context) error {
 	if _, err := Run(cmd); err != nil {
 		return err
 	}
-	// Wait for cert-manager-webhook to be ready, which can take time if cert-manager
-	// was re-installed after uninstalling on a cluster.
-	cmd = exec.CommandContext(
-		ctx, "kubectl", "wait", "deployment.apps/cert-manager-webhook",
-		"--for", "condition=Available",
-		"--namespace", "cert-manager",
-		"--timeout", "5m",
-	)
-
-	_, err := Run(cmd)
-	return err
+	// Wait for all cert-manager deployments to be ready.
+	for _, deploy := range []string{"cert-manager", "cert-manager-cainjector", "cert-manager-webhook"} {
+		cmd = exec.CommandContext(
+			ctx, "kubectl", "wait", "deployment.apps/"+deploy,
+			"--for", "condition=Available",
+			"--namespace", "cert-manager",
+			"--timeout", "5m",
+		)
+		if _, err := Run(cmd); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // UninstallCertManager uninstalls the cert manager
@@ -276,50 +277,3 @@ func GetProjectDir() (string, error) {
 	wd = strings.ReplaceAll(wd, "/test/e2e", "")
 	return wd, nil
 }
-
-// UncommentCode searches for target in the file and remove the comment prefix
-// of the target content. The target content may span multiple lines.
-func UncommentCode(filename, target, prefix string) error {
-	content, err := os.ReadFile(filename)
-	if err != nil {
-		return err
-	}
-
-	before, after, ok := bytes.Cut(content, []byte(target))
-	if !ok {
-		if bytes.Contains(content, []byte(target)[len(prefix):]) {
-			return nil // already uncommented
-		}
-
-		return fmt.Errorf("unable to find the code %s to be uncomment", target)
-	}
-
-	out := new(bytes.Buffer)
-	if _, err = out.Write(before); err != nil {
-		return err
-	}
-
-	scanner := bufio.NewScanner(bytes.NewBufferString(target))
-	if !scanner.Scan() {
-		return nil
-	}
-	for {
-		_, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix))
-		if err != nil {
-			return err
-		}
-		// Avoid writing a newline in case the previous line was the last in target.
-		if !scanner.Scan() {
-			break
-		}
-		if _, err = out.WriteString("\n"); err != nil {
-			return err
-		}
-	}
-
-	if _, err = out.Write(after); err != nil {
-		return err
-	}
-
-	return os.WriteFile(filename, out.Bytes(), 0o644)
-}