From 09cf3d39dd708b27c3bbdc3b031a28fdc6c5d93a Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Wed, 1 Nov 2023 17:06:05 +0000 Subject: [PATCH] temp: seems broken Signed-off-by: Maryam Tahhan --- Makefile | 4 ++ examples/cndp-1-0.yaml | 24 ++++++++ internal/bpfd/client.go | 91 ++++++++++++++-------------- internal/deviceplugin/poolManager.go | 2 +- 4 files changed, 76 insertions(+), 45 deletions(-) create mode 100644 examples/cndp-1-0.yaml diff --git a/Makefile b/Makefile index 2616be0..f0f900e 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,10 @@ kind-label-nodes: ## Label the kind worker nodes with cndp="true" kubectl label node ${KIND_CLUSTER_NAME}-worker cndp="true" kubectl label node ${KIND_CLUSTER_NAME}-worker2 cndp="true" +.PHONY: kind-label-bpfd-cp +kind-label-bpfd-cp: ## Label the kind worker nodes with cndp="true" + kubectl label node ${KIND_CLUSTER_NAME}-control-plane cndp="true" + .PHONY: kind-load-images kind-load-images: ## Load the image on the kind cluster @echo "****** Loading AF_XDP DEVICE PLUGIN image ******" diff --git a/examples/cndp-1-0.yaml b/examples/cndp-1-0.yaml new file mode 100644 index 0000000..c8f6108 --- /dev/null +++ b/examples/cndp-1-0.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cndp-0-0 + annotations: + k8s.v1.cni.cncf.io/networks: afxdp-network +spec: + containers: + - name: cndp-0 + command: ["/bin/bash"] + args: ["-c", "./jsonc_gen.sh -kp ; cndpfwd -c config.jsonc lb;"] + image: quay.io/mtahhan/cndp-map-pinning:latest + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + add: + - NET_RAW + - IPC_LOCK + - BPF + resources: + requests: + afxdp/myPool: '1' + limits: + afxdp/myPool: '1' diff --git a/internal/bpfd/client.go b/internal/bpfd/client.go index 01d6f41..f6c2649 100644 --- a/internal/bpfd/client.go +++ b/internal/bpfd/client.go @@ -77,7 +77,8 @@ func (b *BpfdClient) GetXdpProgs() (*bpfdiov1alpha1.XdpProgramList, error) { } func (b *BpfdClient) GetBPFProgs() (*bpfdiov1alpha1.BpfProgramList, error) { - // Get the xdp program resources + logging.Info("Get the bpf program resources") + // Get the bpf program resources bpfProgramList, err := b.bpfProgramsClient.List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, errors.Wrapf(err, "Failed to retrieve the xdp program resources: %v", err.Error()) @@ -91,29 +92,30 @@ func (b *BpfdClient) GetBPFProgs() (*bpfdiov1alpha1.BpfProgramList, error) { return bpfProgramList, nil } -func (b *BpfdClient) SubmitXdpProg(iface, node, pm, image, sec string) (error, string) { +func (b *BpfdClient) SubmitXdpProg(iface, node, pm, image, bfunc string) (string, error) { var ( - name = node + "-" + pm + "-" + iface + name string + xskmap string interfaces []string ) + + name = node + "-" + pm + "-" + iface interfaces = append(interfaces, iface) - if len(image) == 0 || len(sec) == 0 { - return errors.New("BPF image or function is empty"), "" + if len(image) == 0 || len(bfunc) == 0 { + return "", errors.New("BPF image or function is empty") } - logging.Infof("IMAGE %s FUNCTION %s", image, sec) - // Create an XdpProgram resource xdpProgram := &bpfdiov1alpha1.XdpProgram{ ObjectMeta: metav1.ObjectMeta{ Name: name, // Namespace: "bpfd", //TODO decide on namespace - Labels: map[string]string{"afxdp.io/xdpprog": name}, + Labels: map[string]string{"afxdp/xdpprog": name}, }, Spec: bpfdiov1alpha1.XdpProgramSpec{ BpfProgramCommon: bpfdiov1alpha1.BpfProgramCommon{ - BpfFunctionName: sec, + BpfFunctionName: bfunc, NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "kubernetes.io/hostname": node, @@ -136,43 +138,45 @@ func (b *BpfdClient) SubmitXdpProg(iface, node, pm, image, sec string) (error, s logging.Infof("Submitting this xdp program %v \n", xdpProgram) // Submit the XdpProgram resource to the API - _, err := b.xdpProgramsClient.Create(context.TODO(), xdpProgram, metav1.CreateOptions{}) - if err != nil { - return errors.Wrapf(err, "Failed to create XdpProgram resource: %v", err), "" + x, err := b.xdpProgramsClient.Create(context.TODO(), xdpProgram, metav1.CreateOptions{}) + if err != nil || x == nil { + return "", errors.Wrapf(err, "Failed to create XdpProgram resource: %v", err) } time.Sleep(time.Second) err = b.checkProgStatus(name) if err != nil { - return errors.Wrapf(err, "Failed to create XdpProgram resource: %v", err), "" + return "", errors.Wrapf(err, "Failed to create XdpProgram resource: %v", err) } bpfProgramList, err := b.GetBPFProgs() - if err != nil { - return errors.Wrapf(err, "Failed to get bpf prog resources: %v", err), "" + if err != nil || bpfProgramList == nil { + return "", errors.Wrapf(err, "Failed to get bpf prog resources: %v", err) } - var xskmap string - - // Try to get the xspmap path from the bpf program resource - if bpfProgramList != nil { - bpfProgName := node + "-" + pm + "-" + iface + "-" + node - for _, bpfProgram := range bpfProgramList.Items { - if bpfProgram.ObjectMeta.Name == bpfProgName { - if len(bpfProgram.Spec.Maps) == 0 { - logging.Errorf("NO MAPS FOUND for %s", bpfProgName) - return errors.New("Failed to find a map for the loaded bpf program"), "" - } - for m, path := range bpfProgram.Spec.Maps { - logging.Infof("map: %v", m) - xskmap = path - } - + logging.Info("Try to get the xskmap path from the bpf program resource") + + // Try to get the xskmap path from the bpf program resource + bpfProgName := node + "-" + pm + "-" + iface + "-" + node + for _, bpfProgram := range bpfProgramList.Items { + logging.Infof("\n\nbpfProgram %v", bpfProgram) + if bpfProgram.ObjectMeta.Name == bpfProgName { + logging.Infof("FOUND %v", bpfProgram) + logging.Infof("\nbpfProgram.Spec.Maps %v", bpfProgram.Spec.Maps) + if len(bpfProgram.Spec.Maps) == 0 { + logging.Errorf("NO MAPS FOUND for %s", bpfProgName) + return "", errors.New("Failed to find a map for the loaded bpf program") } + for m, path := range bpfProgram.Spec.Maps { + logging.Infof("map: %v", m) + xskmap = path + } + } + logging.Info("\n\n") } - return nil, xskmap + return xskmap, nil } // Delete XdpProgram @@ -233,41 +237,40 @@ func (b *BpfdClient) checkProgStatus(name string) error { // Try to check status for 10 seconds for i := 0; i < 10; i++ { - logging.Info("CHECKING STATUS OF XDP PROG") + logging.Debug("CHECKING STATUS OF XDP PROG") event, ok := <-xdpWatcher.ResultChan() if !ok { // channel closed - logging.Errorf("Channel closed") + logging.Debug("Channel closed") return errors.New("Channel Closed") } prog, ok := event.Object.(*bpfdiov1alpha1.XdpProgram) if !ok { - logging.Errorf("couldn't get xdp prog object") + logging.Debug("couldn't get xdp prog object") return errors.New("Failed to get xdp prog object") } - logging.Infof("\n%v", prog.Status) + logging.Debug("GOT xdp prog object checking condition") // Get most recent condition recentIdx := len(prog.Status.Conditions) - 1 condition := prog.Status.Conditions[recentIdx] - switch condition.Type { - case ProgramReconcileError: + if condition.Type == BpfProgCondLoaded || condition.Type == ProgramReconcileSuccess { + logging.Debugf("Bpf program Loaded %v", ProgramReconcileSuccess) + return nil + } else if condition.Type == ProgramReconcileError { + logging.Debugf("ProgramReconcileError %v", ProgramReconcileError) if i == 9 { logging.Errorf("Failed to load xdp program") return errors.New("Failed to load xdp program") } - case BpfProgCondLoaded: - case ProgramReconcileSuccess: - logging.Infof("Bpf program Loaded %v", ProgramReconcileSuccess) - return nil - default: - logging.Infof("Bpf program status %v", condition.Type) } + time.Sleep(1 * time.Second) + } return errors.New("Failed to load xdp program") diff --git a/internal/deviceplugin/poolManager.go b/internal/deviceplugin/poolManager.go index 1c2b0bd..9c29442 100644 --- a/internal/deviceplugin/poolManager.go +++ b/internal/deviceplugin/poolManager.go @@ -285,7 +285,7 @@ func (pm *PoolManager) Allocate(ctx context.Context, } if pm.BpfdClientEnable { - err, xskmap := pm.BpfdClient.SubmitXdpProg(device.Name(), pm.Node, pm.DevicePrefix, pm.BpfProg, pm.BpfSec) + xskmap, err := pm.BpfdClient.SubmitXdpProg(device.Name(), pm.Node, pm.DevicePrefix, pm.BpfProg, pm.BpfSec) if err != nil { logging.Errorf("Error SubmitXdpProg to bpfd %v", err) return &response, err