package netlink import ( "bytes" "encoding/binary" "fmt" "net" "github.com/vishvananda/netlink/nl" "golang.org/x/sys/unix" ) // LinkAttrs represents data shared by most link types type RdmaLinkAttrs struct { Index uint32 Name string FirmwareVersion string NodeGuid string SysImageGuid string } // Link represents a rdma device from netlink. type RdmaLink struct { Attrs RdmaLinkAttrs } func getProtoField(clientType int, op int) int { return ((clientType << nl.RDMA_NL_GET_CLIENT_SHIFT) | op) } func uint64ToGuidString(guid uint64) string { //Convert to byte array sysGuidBytes := new(bytes.Buffer) binary.Write(sysGuidBytes, binary.LittleEndian, guid) //Convert to HardwareAddr sysGuidNet := net.HardwareAddr(sysGuidBytes.Bytes()) //Get the String return sysGuidNet.String() } func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) { link := RdmaLink{} reader := bytes.NewReader(data) for reader.Len() >= 4 { _, attrType, len, value := parseNfAttrTLV(reader) switch attrType { case nl.RDMA_NLDEV_ATTR_DEV_INDEX: var Index uint32 r := bytes.NewReader(value) binary.Read(r, nl.NativeEndian(), &Index) link.Attrs.Index = Index case nl.RDMA_NLDEV_ATTR_DEV_NAME: link.Attrs.Name = string(value[0 : len-1]) case nl.RDMA_NLDEV_ATTR_FW_VERSION: link.Attrs.FirmwareVersion = string(value[0 : len-1]) case nl.RDMA_NLDEV_ATTR_NODE_GUID: var guid uint64 r := bytes.NewReader(value) binary.Read(r, nl.NativeEndian(), &guid) link.Attrs.NodeGuid = uint64ToGuidString(guid) case nl.RDMA_NLDEV_ATTR_SYS_IMAGE_GUID: var sysGuid uint64 r := bytes.NewReader(value) binary.Read(r, nl.NativeEndian(), &sysGuid) link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid) } if (len % 4) != 0 { // Skip pad bytes reader.Seek(int64(4-(len%4)), seekCurrent) } } return &link, nil } func execRdmaSetLink(req *nl.NetlinkRequest) error { _, err := req.Execute(unix.NETLINK_RDMA, 0) return err } // RdmaLinkList gets a list of RDMA link devices. // Equivalent to: `rdma dev show` func RdmaLinkList() ([]*RdmaLink, error) { return pkgHandle.RdmaLinkList() } // RdmaLinkList gets a list of RDMA link devices. // Equivalent to: `rdma dev show` func (h *Handle) RdmaLinkList() ([]*RdmaLink, error) { proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_GET) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP) msgs, err := req.Execute(unix.NETLINK_RDMA, 0) if err != nil { return nil, err } var res []*RdmaLink for _, m := range msgs { link, err := executeOneGetRdmaLink(m) if err != nil { return nil, err } res = append(res, link) } return res, nil } // RdmaLinkByName finds a link by name and returns a pointer to the object if // found and nil error, otherwise returns error code. func RdmaLinkByName(name string) (*RdmaLink, error) { return pkgHandle.RdmaLinkByName(name) } // RdmaLinkByName finds a link by name and returns a pointer to the object if // found and nil error, otherwise returns error code. func (h *Handle) RdmaLinkByName(name string) (*RdmaLink, error) { links, err := h.RdmaLinkList() if err != nil { return nil, err } for _, link := range links { if link.Attrs.Name == name { return link, nil } } return nil, fmt.Errorf("Rdma device %v not found", name) } // RdmaLinkSetName sets the name of the rdma link device. Return nil on success // or error otherwise. // Equivalent to: `rdma dev set $old_devname name $name` func RdmaLinkSetName(link *RdmaLink, name string) error { return pkgHandle.RdmaLinkSetName(link, name) } // RdmaLinkSetName sets the name of the rdma link device. Return nil on success // or error otherwise. // Equivalent to: `rdma dev set $old_devname name $name` func (h *Handle) RdmaLinkSetName(link *RdmaLink, name string) error { proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SET) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) b := make([]byte, 4) native.PutUint32(b, uint32(link.Attrs.Index)) data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b) req.AddData(data) b = make([]byte, len(name)+1) copy(b, name) data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_NAME, b) req.AddData(data) return execRdmaSetLink(req) } func netnsModeToString(mode uint8) string { switch mode { case 0: return "exclusive" case 1: return "shared" default: return "unknown" } } func executeOneGetRdmaNetnsMode(data []byte) (string, error) { reader := bytes.NewReader(data) for reader.Len() >= 4 { _, attrType, len, value := parseNfAttrTLV(reader) switch attrType { case nl.RDMA_NLDEV_SYS_ATTR_NETNS_MODE: var mode uint8 r := bytes.NewReader(value) binary.Read(r, nl.NativeEndian(), &mode) return netnsModeToString(mode), nil } if (len % 4) != 0 { // Skip pad bytes reader.Seek(int64(4-(len%4)), seekCurrent) } } return "", fmt.Errorf("Invalid netns mode") } // RdmaSystemGetNetnsMode gets the net namespace mode for RDMA subsystem // Returns mode string and error status as nil on success or returns error // otherwise. // Equivalent to: `rdma system show netns' func RdmaSystemGetNetnsMode() (string, error) { return pkgHandle.RdmaSystemGetNetnsMode() } // RdmaSystemGetNetnsMode gets the net namespace mode for RDMA subsystem // Returns mode string and error status as nil on success or returns error // otherwise. // Equivalent to: `rdma system show netns' func (h *Handle) RdmaSystemGetNetnsMode() (string, error) { proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SYS_GET) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) msgs, err := req.Execute(unix.NETLINK_RDMA, 0) if err != nil { return "", err } if len(msgs) == 0 { return "", fmt.Errorf("No valid response from kernel") } return executeOneGetRdmaNetnsMode(msgs[0]) } func netnsModeStringToUint8(mode string) (uint8, error) { switch mode { case "exclusive": return 0, nil case "shared": return 1, nil default: return 0, fmt.Errorf("Invalid mode; %q", mode) } } // RdmaSystemSetNetnsMode sets the net namespace mode for RDMA subsystem // Returns nil on success or appropriate error code. // Equivalent to: `rdma system set netns { shared | exclusive }' func RdmaSystemSetNetnsMode(NewMode string) error { return pkgHandle.RdmaSystemSetNetnsMode(NewMode) } // RdmaSystemSetNetnsMode sets the net namespace mode for RDMA subsystem // Returns nil on success or appropriate error code. // Equivalent to: `rdma system set netns { shared | exclusive }' func (h *Handle) RdmaSystemSetNetnsMode(NewMode string) error { value, err := netnsModeStringToUint8(NewMode) if err != nil { return err } proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SYS_SET) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) data := nl.NewRtAttr(nl.RDMA_NLDEV_SYS_ATTR_NETNS_MODE, []byte{value}) req.AddData(data) _, err = req.Execute(unix.NETLINK_RDMA, 0) return err } // RdmaLinkSetNsFd puts the RDMA device into a new network namespace. The // fd must be an open file descriptor to a network namespace. // Similar to: `rdma dev set $dev netns $ns` func RdmaLinkSetNsFd(link *RdmaLink, fd uint32) error { return pkgHandle.RdmaLinkSetNsFd(link, fd) } // RdmaLinkSetNsFd puts the RDMA device into a new network namespace. The // fd must be an open file descriptor to a network namespace. // Similar to: `rdma dev set $dev netns $ns` func (h *Handle) RdmaLinkSetNsFd(link *RdmaLink, fd uint32) error { proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_SET) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, nl.Uint32Attr(link.Attrs.Index)) req.AddData(data) data = nl.NewRtAttr(nl.RDMA_NLDEV_NET_NS_FD, nl.Uint32Attr(fd)) req.AddData(data) return execRdmaSetLink(req) } // RdmaLinkDel deletes an rdma link // // Similar to: rdma link delete NAME // REF: https://man7.org/linux/man-pages/man8/rdma-link.8.html func RdmaLinkDel(name string) error { return pkgHandle.RdmaLinkDel(name) } // RdmaLinkDel deletes an rdma link. func (h *Handle) RdmaLinkDel(name string) error { link, err := h.RdmaLinkByName(name) if err != nil { return err } proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_DELLINK) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) b := make([]byte, 4) native.PutUint32(b, link.Attrs.Index) req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b)) _, err = req.Execute(unix.NETLINK_RDMA, 0) return err } // RdmaLinkAdd adds an rdma link for the specified type to the network device. // Similar to: rdma link add NAME type TYPE netdev NETDEV // NAME - specifies the new name of the rdma link to add // TYPE - specifies which rdma type to use. Link types: // rxe - Soft RoCE driver // siw - Soft iWARP driver // NETDEV - specifies the network device to which the link is bound // // REF: https://man7.org/linux/man-pages/man8/rdma-link.8.html func RdmaLinkAdd(linkName, linkType, netdev string) error { return pkgHandle.RdmaLinkAdd(linkName, linkType, netdev) } // RdmaLinkAdd adds an rdma link for the specified type to the network device. func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) error { proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_NEWLINK) req := h.newNetlinkRequest(proto, unix.NLM_F_ACK) req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_NAME, nl.ZeroTerminated(linkName))) req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_LINK_TYPE, nl.ZeroTerminated(linkType))) req.AddData(nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_NDEV_NAME, nl.ZeroTerminated(netdev))) _, err := req.Execute(unix.NETLINK_RDMA, 0) return err }