Add RDMA netlink socket for RDMA device information

This patch adds very basic support for getting information about RDMA
networking device; starting with device index, name, firmware version,
node GUID and system image GUID.
This is done through RDMA netlink socket.

RDMA devices are some what similar to Ethernet devices.
However there are few major differences between them.
RDMA devices usually have one or two ports, unlike Ethernet devices.
Each port has its own attributes, state and network addresses which are
different than Ethernet devices (Link and LinkAttrs). They almost don't
overlap with Link and LinkAttrs.

Therefore it doesn't derive Link and LinkAttrs structure; instead they
are represented using RdmaLink and RdmaLinkAttrs.

RdmaLink represents a RDMA device containing its attributes.
All Rdma device communication occurs through rdma subsystem's netlink
socket.

Signed-off-by: Parav Pandit parav@mellanox.com

nl/rdma_link_linux.go

@@ -0,0 +1,30 @@
+package nl
+
+const (
+	RDMA_NL_GET_CLIENT_SHIFT = 10
+)
+
+const (
+	RDMA_NL_NLDEV = 5
+)
+
+const (
+	RDMA_NLDEV_CMD_GET = 1
+)
+
+const (
+	RDMA_NLDEV_ATTR_DEV_INDEX       = 1
+	RDMA_NLDEV_ATTR_DEV_NAME        = 2
+	RDMA_NLDEV_ATTR_PORT_INDEX      = 3
+	RDMA_NLDEV_ATTR_CAP_FLAGS       = 4
+	RDMA_NLDEV_ATTR_FW_VERSION      = 5
+	RDMA_NLDEV_ATTR_NODE_GUID       = 6
+	RDMA_NLDEV_ATTR_SYS_IMAGE_GUID  = 7
+	RDMA_NLDEV_ATTR_SUBNET_PREFIX   = 8
+	RDMA_NLDEV_ATTR_LID             = 9
+	RDMA_NLDEV_ATTR_SM_LID          = 10
+	RDMA_NLDEV_ATTR_LMC             = 11
+	RDMA_NLDEV_ATTR_PORT_STATE      = 12
+	RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
+	RDMA_NLDEV_ATTR_DEV_NODE_TYPE   = 14
+)

rdma_link_linux.go

@@ -0,0 +1,112 @@
+package netlink
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"net"
+
+	"github.com/vishvananda/netlink/nl"
+	"golang.org/x/sys/unix"
+)
+
+// LinkAttrs represents data shared by most link types
+type RdmaLinkAttrs struct {
+	Index           uint32
+	Name            string
+	FirmwareVersion string
+	NodeGuid        string
+	SysImageGuid    string
+}
+
+// Link represents a rdma device from netlink.
+type RdmaLink struct {
+	Attrs RdmaLinkAttrs
+}
+
+func getProtoField(clientType int, op int) int {
+	return ((clientType << nl.RDMA_NL_GET_CLIENT_SHIFT) | op)
+}
+
+func uint64ToGuidString(guid uint64) string {
+	//Convert to byte array
+	sysGuidBytes := new(bytes.Buffer)
+	binary.Write(sysGuidBytes, binary.LittleEndian, guid)
+
+	//Convert to HardwareAddr
+	sysGuidNet := net.HardwareAddr(sysGuidBytes.Bytes())
+
+	//Get the String
+	return sysGuidNet.String()
+}
+
+func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) {
+
+	link := RdmaLink{}
+
+	reader := bytes.NewReader(data)
+	for reader.Len() >= 4 {
+		_, attrType, len, value := parseNfAttrTLV(reader)
+
+		switch attrType {
+		case nl.RDMA_NLDEV_ATTR_DEV_INDEX:
+			var Index uint32
+			r := bytes.NewReader(value)
+			binary.Read(r, nl.NativeEndian(), &Index)
+			link.Attrs.Index = Index
+		case nl.RDMA_NLDEV_ATTR_DEV_NAME:
+			link.Attrs.Name = string(value[0 : len-1])
+		case nl.RDMA_NLDEV_ATTR_FW_VERSION:
+			link.Attrs.FirmwareVersion = string(value[0 : len-1])
+		case nl.RDMA_NLDEV_ATTR_NODE_GUID:
+			var guid uint64
+			r := bytes.NewReader(value)
+			binary.Read(r, nl.NativeEndian(), &guid)
+			link.Attrs.NodeGuid = uint64ToGuidString(guid)
+		case nl.RDMA_NLDEV_ATTR_SYS_IMAGE_GUID:
+			var sysGuid uint64
+			r := bytes.NewReader(value)
+			binary.Read(r, nl.NativeEndian(), &sysGuid)
+			link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid)
+		}
+		if (len % 4) != 0 {
+			// Skip pad bytes
+			reader.Seek(int64(4-(len%4)), seekCurrent)
+		}
+	}
+	return &link, nil
+}
+
+func execRdmaGetLink(req *nl.NetlinkRequest, name string) (*RdmaLink, error) {
+
+	msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
+	if err != nil {
+		return nil, err
+	}
+	for _, m := range msgs {
+		link, err := executeOneGetRdmaLink(m)
+		if err != nil {
+			return nil, err
+		}
+		if link.Attrs.Name == name {
+			return link, nil
+		}
+	}
+	return nil, fmt.Errorf("Rdma device %v not found", name)
+}
+
+// RdmaLinkByName finds a link by name and returns a pointer to the object if
+// found and nil error, otherwise returns error code.
+func RdmaLinkByName(name string) (*RdmaLink, error) {
+	return pkgHandle.RdmaLinkByName(name)
+}
+
+// RdmaLinkByName finds a link by name and returns a pointer to the object if
+// found and nil error, otherwise returns error code.
+func (h *Handle) RdmaLinkByName(name string) (*RdmaLink, error) {
+
+	proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_GET)
+	req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP)
+
+	return execRdmaGetLink(req, name)
+}

rdma_link_test.go

@@ -0,0 +1,34 @@
+// +build linux
+
+package netlink
+
+import (
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func setupRdmaKModule(t *testing.T, name string) {
+	skipUnlessRoot(t)
+	file, err := ioutil.ReadFile("/proc/modules")
+	if err != nil {
+		t.Fatal("Failed to open /proc/modules", err)
+	}
+	for _, line := range strings.Split(string(file), "\n") {
+		n := strings.Split(line, " ")[0]
+		if n == name {
+			return
+		}
+
+	}
+	t.Skipf("Test requires kmodule %q.", name)
+}
+
+func TestRdmaGetRdmaLink(t *testing.T) {
+	minKernelRequired(t, 4, 16)
+	setupRdmaKModule(t, "ib_core")
+	_, err := RdmaLinkByName("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+}