Standalone test for LSP preservation for IPAM-less localnet pods after ovnkube-node restart

go.mod

module lsp-preservation-test

go 1.22.0

toolchain go1.24.5

require (
	k8s.io/api v0.31.0
	k8s.io/apimachinery v0.31.0
	k8s.io/client-go v0.31.0
)

require (
	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
	github.com/go-logr/logr v1.4.2 // indirect
	github.com/go-openapi/jsonpointer v0.19.6 // indirect
	github.com/go-openapi/jsonreference v0.20.2 // indirect
	github.com/go-openapi/swag v0.22.4 // indirect
	github.com/gogo/protobuf v1.3.2 // indirect
	github.com/golang/protobuf v1.5.4 // indirect
	github.com/google/gnostic-models v0.6.8 // indirect
	github.com/google/go-cmp v0.6.0 // indirect
	github.com/google/gofuzz v1.2.0 // indirect
	github.com/google/uuid v1.6.0 // indirect
	github.com/gorilla/websocket v1.5.0 // indirect
	github.com/imdario/mergo v0.3.6 // indirect
	github.com/josharian/intern v1.0.0 // indirect
	github.com/json-iterator/go v1.1.12 // indirect
	github.com/mailru/easyjson v0.7.7 // indirect
	github.com/moby/spdystream v0.4.0 // indirect
	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
	github.com/modern-go/reflect2 v1.0.2 // indirect
	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
	github.com/spf13/pflag v1.0.5 // indirect
	github.com/x448/float16 v0.8.4 // indirect
	golang.org/x/net v0.26.0 // indirect
	golang.org/x/oauth2 v0.21.0 // indirect
	golang.org/x/sys v0.21.0 // indirect
	golang.org/x/term v0.21.0 // indirect
	golang.org/x/text v0.16.0 // indirect
	golang.org/x/time v0.3.0 // indirect
	google.golang.org/protobuf v1.34.2 // indirect
	gopkg.in/inf.v0 v0.9.1 // indirect
	gopkg.in/yaml.v2 v2.4.0 // indirect
	gopkg.in/yaml.v3 v3.0.1 // indirect
	k8s.io/klog/v2 v2.130.1 // indirect
	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
	sigs.k8s.io/yaml v1.4.0 // indirect
)

ipamless-localnet-nad.yaml

---
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  name: ipamless-localnet
  namespace: default
spec:
  config: |
    {
      "cniVersion": "0.3.1",
      "name": "ipamless-localnet",
      "type": "ovn-k8s-cni-overlay",
      "topology": "localnet",
      "physicalNetworkName": "physnet",
      "netAttachDefName": "default/ipamless-localnet",
      "role": "secondary"
    }

lsp_preservation_test.go

package main

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/remotecommand"
)

const (
	// OpenShift OVN namespace
	ovnNamespace = "openshift-ovn-kubernetes"
	// Container name for nbdb in ovnkube-node pod on OpenShift
	nbdbContainerName = "nbdb"
	// Physical network name (must match your bridge-mapping)
	physicalNetworkName = "physnet"
	// Test namespace
	testNamespace = "default"
	// NAD name
	nadName = "ipamless-localnet"
)

func getKubeConfig() (*rest.Config, error) {
	kubeconfig := os.Getenv("KUBECONFIG")
	if kubeconfig == "" {
		kubeconfig = os.Getenv("HOME") + "/.kube/config"
	}
	return clientcmd.BuildConfigFromFlags("", kubeconfig)
}

func execInPod(config *rest.Config, clientset *kubernetes.Clientset, namespace, podName, containerName string, command []string) (string, string, error) {
	req := clientset.CoreV1().RESTClient().Post().
		Resource("pods").
		Name(podName).
		Namespace(namespace).
		SubResource("exec").
		VersionedParams(&corev1.PodExecOptions{
			Container: containerName,
			Command:   command,
			Stdin:     false,
			Stdout:    true,
			Stderr:    true,
			TTY:       false,
		}, scheme.ParameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
	if err != nil {
		return "", "", fmt.Errorf("failed to create executor: %w", err)
	}

	var stdout, stderr bytes.Buffer
	err = exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{
		Stdout: &stdout,
		Stderr: &stderr,
	})

	return stdout.String(), stderr.String(), err
}

func waitForPodWithAnnotation(ctx context.Context, clientset *kubernetes.Clientset, namespace, podName, annotation string, timeout time.Duration) error {
	return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
		pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
		if err != nil {
			return false, nil
		}
		_, hasAnnotation := pod.Annotations[annotation]
		return pod.Status.Phase == corev1.PodRunning && hasAnnotation, nil
	})
}

func deletePodAndWait(ctx context.Context, clientset *kubernetes.Clientset, namespace, podName string, timeout time.Duration) error {
	err := clientset.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
	if err != nil {
		return err
	}

	return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
		_, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
		if err != nil {
			return true, nil // Pod is gone
		}
		return false, nil
	})
}

func restartOVNKubeNodePod(ctx context.Context, clientset *kubernetes.Clientset, nodeName string) error {
	// Find ovnkube-node pod on the node
	pods, err := clientset.CoreV1().Pods(ovnNamespace).List(ctx, metav1.ListOptions{
		LabelSelector: "app=ovnkube-node",
		FieldSelector: "spec.nodeName=" + nodeName,
	})
	if err != nil {
		return fmt.Errorf("failed to list ovnkube-node pods: %w", err)
	}
	if len(pods.Items) == 0 {
		return fmt.Errorf("no ovnkube-node pod found on node %s", nodeName)
	}

	oldPodName := pods.Items[0].Name
	fmt.Printf("Deleting ovnkube-node pod %s on node %s\n", oldPodName, nodeName)

	// Delete the pod
	if err := deletePodAndWait(ctx, clientset, ovnNamespace, oldPodName, 2*time.Minute); err != nil {
		return fmt.Errorf("failed to delete ovnkube-node pod: %w", err)
	}

	// Wait for new pod to be ready
	fmt.Printf("Waiting for new ovnkube-node pod on node %s\n", nodeName)
	return wait.PollUntilContextTimeout(ctx, 2*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) {
		pods, err := clientset.CoreV1().Pods(ovnNamespace).List(ctx, metav1.ListOptions{
			LabelSelector: "app=ovnkube-node",
			FieldSelector: "spec.nodeName=" + nodeName,
		})
		if err != nil || len(pods.Items) == 0 {
			return false, nil
		}

		pod := pods.Items[0]
		if pod.Status.Phase != corev1.PodRunning {
			return false, nil
		}

		for _, cs := range pod.Status.ContainerStatuses {
			if !cs.Ready {
				return false, nil
			}
		}
		return true, nil
	})
}

func TestLSPPreservationForIPAMLessLocalnet(t *testing.T) {
	ctx := context.Background()

	// Setup kubernetes client
	config, err := getKubeConfig()
	if err != nil {
		t.Fatalf("Failed to get kubeconfig: %v", err)
	}

	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		t.Fatalf("Failed to create clientset: %v", err)
	}

	// Create test pod
	podName := "test-ipamless-lsp-pod"
	networkAnnotation := fmt.Sprintf("%s/%s", testNamespace, nadName)

	testPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      podName,
			Namespace: testNamespace,
			Annotations: map[string]string{
				"k8s.v1.cni.cncf.io/networks": networkAnnotation,
			},
			Labels: map[string]string{
				"app": "test-ipamless-lsp",
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{
				{
					Name:    "test",
					Image:   "registry.access.redhat.com/ubi8/ubi-minimal:latest",
					Command: []string{"sleep", "infinity"},
				},
			},
			RestartPolicy: corev1.RestartPolicyAlways,
		},
	}

	t.Log("Creating test pod")
	_, err = clientset.CoreV1().Pods(testNamespace).Create(ctx, testPod, metav1.CreateOptions{})
	if err != nil {
		t.Fatalf("Failed to create test pod: %v", err)
	}

	// Cleanup pod at the end
	defer func() {
		t.Log("Cleaning up test pod")
		clientset.CoreV1().Pods(testNamespace).Delete(ctx, podName, metav1.DeleteOptions{})
	}()

	// Wait for pod to be running and annotated
	t.Log("Waiting for pod to be running and annotated")
	err = waitForPodWithAnnotation(ctx, clientset, testNamespace, podName, "k8s.ovn.org/pod-networks", 2*time.Minute)
	if err != nil {
		t.Fatalf("Pod did not become ready with annotation: %v", err)
	}

	// Get node where pod is running
	pod, err := clientset.CoreV1().Pods(testNamespace).Get(ctx, podName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get pod: %v", err)
	}
	nodeName := pod.Spec.NodeName
	t.Logf("Pod is running on node: %s", nodeName)

	// Find ovnkube-node pod on that node
	ovnkubeNodePods, err := clientset.CoreV1().Pods(ovnNamespace).List(ctx, metav1.ListOptions{
		LabelSelector: "app=ovnkube-node",
		FieldSelector: "spec.nodeName=" + nodeName,
	})
	if err != nil {
		t.Fatalf("Failed to list ovnkube-node pods: %v", err)
	}
	if len(ovnkubeNodePods.Items) == 0 {
		t.Fatalf("No ovnkube-node pod found on node %s", nodeName)
	}
	ovnkubeNodePod := ovnkubeNodePods.Items[0]
	t.Logf("Found ovnkube-node pod: %s", ovnkubeNodePod.Name)

	// Calculate expected LSP name
	fullNadName := fmt.Sprintf("%s/%s", testNamespace, nadName)
	nadNameWithDots := strings.ReplaceAll(strings.ReplaceAll(fullNadName, "-", "."), "/", ".")
	expectedLSPName := fmt.Sprintf("%s_%s_%s", nadNameWithDots, testNamespace, podName)
	t.Logf("Expected LSP name: %s", expectedLSPName)

	// Find LSP in OVN NB database
	findLSP := func() (string, error) {
		stdout, stderr, err := execInPod(config, clientset, ovnNamespace, ovnkubeNodePod.Name, nbdbContainerName,
			[]string{"ovn-nbctl", "get", "logical-switch-port", expectedLSPName, "_uuid"})
		if err != nil {
			return "", fmt.Errorf("failed to find LSP: %v, stderr: %s", err, stderr)
		}
		return strings.TrimSpace(stdout), nil
	}

	t.Log("Verifying LSP exists in OVN NB database")
	originalLSPUUID, err := findLSP()
	if err != nil {
		t.Fatalf("LSP should exist before restart: %v", err)
	}
	if originalLSPUUID == "" {
		t.Fatal("LSP UUID should not be empty")
	}
	t.Logf("Found LSP with UUID: %s", originalLSPUUID)

	// Restart ovnkube-node pod
	t.Log("Restarting ovnkube-node pod")
	err = restartOVNKubeNodePod(ctx, clientset, nodeName)
	if err != nil {
		t.Fatalf("Failed to restart ovnkube-node pod: %v", err)
	}

	// Wait for sync
	t.Log("Waiting for ovnkube-node to complete sync (30s)")
	time.Sleep(30 * time.Second)

	// Find new ovnkube-node pod
	ovnkubeNodePods, err = clientset.CoreV1().Pods(ovnNamespace).List(ctx, metav1.ListOptions{
		LabelSelector: "app=ovnkube-node",
		FieldSelector: "spec.nodeName=" + nodeName,
	})
	if err != nil {
		t.Fatalf("Failed to list ovnkube-node pods after restart: %v", err)
	}
	if len(ovnkubeNodePods.Items) == 0 {
		t.Fatalf("No ovnkube-node pod found after restart")
	}
	ovnkubeNodePod = ovnkubeNodePods.Items[0]
	t.Logf("Found new ovnkube-node pod: %s", ovnkubeNodePod.Name)

	// Verify LSP still exists with same UUID
	t.Log("Verifying LSP still exists after ovnkube-node restart")

	// Check multiple times to ensure consistency
	for i := 0; i < 5; i++ {
		currentLSPUUID, err := findLSP()
		if err != nil {
			t.Fatalf("LSP should still exist after restart (attempt %d): %v", i+1, err)
		}
		if currentLSPUUID != originalLSPUUID {
			t.Fatalf("LSP was recreated! Original UUID: %s, Current UUID: %s", originalLSPUUID, currentLSPUUID)
		}
		t.Logf("Attempt %d: LSP UUID matches: %s", i+1, currentLSPUUID)
		time.Sleep(2 * time.Second)
	}

	// Verify pod still has network annotation
	pod, err = clientset.CoreV1().Pods(testNamespace).Get(ctx, podName, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get pod after restart: %v", err)
	}
	if _, hasAnnotation := pod.Annotations["k8s.ovn.org/pod-networks"]; !hasAnnotation {
		t.Fatal("Pod should still have network annotation")
	}

	t.Log("SUCCESS: LSP was preserved after ovnkube-node restart")
}