Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rdma: support rdma resource and statistic for monitoring #970

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

rdma: support rdma resource and statistic for monitoring #970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 34 additions & 24 deletions nl/rdma_link_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,41 @@ const (
)

const (
RDMA_NLDEV_CMD_GET = 1
RDMA_NLDEV_CMD_SET = 2
RDMA_NLDEV_CMD_NEWLINK = 3
RDMA_NLDEV_CMD_DELLINK = 4
RDMA_NLDEV_CMD_SYS_GET = 6
RDMA_NLDEV_CMD_SYS_SET = 7
RDMA_NLDEV_CMD_GET = 1
RDMA_NLDEV_CMD_SET = 2
RDMA_NLDEV_CMD_NEWLINK = 3
RDMA_NLDEV_CMD_DELLINK = 4
RDMA_NLDEV_CMD_SYS_GET = 6
RDMA_NLDEV_CMD_SYS_SET = 7
RDMA_NLDEV_CMD_RES_GET = 9
RDMA_NLDEV_CMD_STAT_GET = 17
)

const (
RDMA_NLDEV_ATTR_DEV_INDEX = 1
RDMA_NLDEV_ATTR_DEV_NAME = 2
RDMA_NLDEV_ATTR_PORT_INDEX = 3
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
RDMA_NLDEV_ATTR_FW_VERSION = 5
RDMA_NLDEV_ATTR_NODE_GUID = 6
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
RDMA_NLDEV_ATTR_LID = 9
RDMA_NLDEV_ATTR_SM_LID = 10
RDMA_NLDEV_ATTR_LMC = 11
RDMA_NLDEV_ATTR_PORT_STATE = 12
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
RDMA_NLDEV_ATTR_NDEV_NAME = 51
RDMA_NLDEV_ATTR_LINK_TYPE = 65
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
RDMA_NLDEV_NET_NS_FD = 68
RDMA_NLDEV_ATTR_DEV_INDEX = 1
RDMA_NLDEV_ATTR_DEV_NAME = 2
RDMA_NLDEV_ATTR_PORT_INDEX = 3
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
RDMA_NLDEV_ATTR_FW_VERSION = 5
RDMA_NLDEV_ATTR_NODE_GUID = 6
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
RDMA_NLDEV_ATTR_LID = 9
RDMA_NLDEV_ATTR_SM_LID = 10
RDMA_NLDEV_ATTR_LMC = 11
RDMA_NLDEV_ATTR_PORT_STATE = 12
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
RDMA_NLDEV_ATTR_RES_SUMMARY = 15
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY = 16
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME = 17
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR = 18
RDMA_NLDEV_ATTR_NDEV_NAME = 51
RDMA_NLDEV_ATTR_LINK_TYPE = 65
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
RDMA_NLDEV_NET_NS_FD = 68
RDMA_NLDEV_ATTR_STAT_HWCOUNTERS = 80
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY = 81
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME = 82
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE = 83
)
215 changes: 215 additions & 0 deletions rdma_link_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type RdmaLinkAttrs struct {
FirmwareVersion string
NodeGuid string
SysImageGuid string
NumPorts uint32
}

// Link represents a rdma device from netlink.
Expand Down Expand Up @@ -68,6 +69,11 @@ func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) {
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &sysGuid)
link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid)
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
var availablePort uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &availablePort)
link.Attrs.NumPorts = availablePort
}
if (len % 4) != 0 {
// Skip pad bytes
Expand Down Expand Up @@ -329,3 +335,212 @@ func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) er
_, err := req.Execute(unix.NETLINK_RDMA, 0)
return err
}

// RdmaResource represents a rdma device resource tracking summaries
type RdmaResource struct {
Index uint32
Name string
RdmaResourceSummaryEntries map[string]uint64
}

// RdmaResourceList list rdma resource tracking information
// Returns all rdma devices resource tracking summary on success or returns error
// otherwise.
// Equivalent to: `rdma resource'
func RdmaResourceList() ([]*RdmaResource, error) {
return pkgHandle.RdmaResourceList()
}

// RdmaResourceList list rdma resource tracking information
// Returns all rdma devices resource tracking summary on success or returns error
// otherwise.
// Equivalent to: `rdma resource'
func (h *Handle) RdmaResourceList() ([]*RdmaResource, error) {
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_RES_GET)
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP)

msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
if err != nil {
return nil, err
}
if len(msgs) == 0 {
return nil, fmt.Errorf("No valid response from kernel")
}
var rdmaResources []*RdmaResource
for _, msg := range msgs {
res, err := executeOneGetRdmaResourceList(msg)
if err != nil {
return nil, err
}
rdmaResources = append(rdmaResources, res)
}
return rdmaResources, nil
}

func parseRdmaCounters(counterType uint16, data []byte) (map[string]uint64, error) {
var counterKeyType, counterValueType uint16
switch counterType {
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY:
counterKeyType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME
counterValueType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY:
counterKeyType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME
counterValueType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE
default:
return nil, fmt.Errorf("Invalid counter type: %d", counterType)
}
counters := make(map[string]uint64)
reader := bytes.NewReader(data)

for reader.Len() >= 4 {
_, attrType, _, value := parseNfAttrTLV(reader)
if attrType != counterType {
return nil, fmt.Errorf("Invalid resource summary entry type; %d", attrType)
}

summaryReader := bytes.NewReader(value)
for summaryReader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(summaryReader)
if attrType != counterKeyType {
return nil, fmt.Errorf("Invalid resource summary entry name type; %d", attrType)
}
name := string(value[0 : len-1])
// Skip pad bytes
if (len % 4) != 0 {
summaryReader.Seek(int64(4-(len%4)), seekCurrent)
}
_, attrType, len, value = parseNfAttrTLV(summaryReader)
if attrType != counterValueType {
return nil, fmt.Errorf("Invalid resource summary entry value type; %d", attrType)
}
counters[name] = native.Uint64(value)
}
}
return counters, nil
}

func executeOneGetRdmaResourceList(data []byte) (*RdmaResource, error) {
var res RdmaResource
reader := bytes.NewReader(data)
for reader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(reader)

switch attrType {
case nl.RDMA_NLDEV_ATTR_DEV_INDEX:
var Index uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &Index)
res.Index = Index
case nl.RDMA_NLDEV_ATTR_DEV_NAME:
res.Name = string(value[0 : len-1])
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY:
var err error
res.RdmaResourceSummaryEntries, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY, value)
if err != nil {
return nil, err
}
}
if (len % 4) != 0 {
// Skip pad bytes
reader.Seek(int64(4-(len%4)), seekCurrent)
}
}
return &res, nil
}

// RdmaPortStatistic represents a rdma port statistic counter
type RdmaPortStatistic struct {
PortIndex uint32
Statistics map[string]uint64
}

// RdmaDeviceStatistic represents a rdma device statistic counter
type RdmaDeviceStatistic struct {
RdmaPortStatistics []*RdmaPortStatistic
}

// RdmaStatistic get rdma device statistic counters
// Returns rdma device statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV]'
func RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
return pkgHandle.RdmaStatistic(link)
}

// RdmaStatistic get rdma device statistic counters
// Returns rdma device statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV]'
func (h *Handle) RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
rdmaLinkStatistic := make([]*RdmaPortStatistic, 0)
for portIndex := uint32(1); portIndex <= link.Attrs.NumPorts; portIndex++ {
portStatistic, err := h.RdmaPortStatisticList(link, portIndex)
if err != nil {
return nil, err
}
rdmaLinkStatistic = append(rdmaLinkStatistic, portStatistic)
}
return &RdmaDeviceStatistic{RdmaPortStatistics: rdmaLinkStatistic}, nil
}

// RdmaPortStatisticList get rdma device port statistic counters
// Returns rdma device port statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV/PORT]'
func RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
return pkgHandle.RdmaPortStatisticList(link, port)
}

// RdmaPortStatisticList get rdma device port statistic counters
// Returns rdma device port statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV/PORT]'
func (h *Handle) RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_STAT_GET)
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_REQUEST)
b := make([]byte, 4)
native.PutUint32(b, link.Attrs.Index)
data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b)
req.AddData(data)

b = make([]byte, 4)
native.PutUint32(b, port)
data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_PORT_INDEX, b)
req.AddData(data)

msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
if err != nil {
return nil, err
}
if len(msgs) != 1 {
return nil, fmt.Errorf("No valid response from kernel")
}
return executeOneGetRdmaPortStatistics(msgs[0])
}

func executeOneGetRdmaPortStatistics(data []byte) (*RdmaPortStatistic, error) {
var stat RdmaPortStatistic
reader := bytes.NewReader(data)
for reader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(reader)

switch attrType {
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
var Index uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &Index)
stat.PortIndex = Index
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTERS:
var err error
stat.Statistics, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY, value)
if err != nil {
return nil, err
}
}
if (len % 4) != 0 {
// Skip pad bytes
reader.Seek(int64(4-(len%4)), seekCurrent)
}
}
return &stat, nil
}
27 changes: 27 additions & 0 deletions rdma_link_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,30 @@ func TestRdmaLinkAddAndDel(t *testing.T) {

checkPresence(linkName, false)
}

func TestRdmaLinkMetrics(t *testing.T) {
minKernelRequired(t, 5, 1)
setupRdmaKModule(t, "rdma_rxe")
if err := RdmaLinkAdd(t.Name(), "rxe", "lo"); err != nil {
t.Fatal(err)
}
link, err := RdmaLinkByName(t.Name())
if err != nil {
t.Fatal(err)
}
defer RdmaLinkDel(t.Name())
resources, err := RdmaResourceList()
if err != nil {
t.Fatal(err)
}
for _, resource := range resources {
t.Logf("resource: %+v", resource)
}
stats, err := RdmaStatistic(link)
if err != nil {
t.Fatal(err)
}
for _, stat := range stats.RdmaPortStatistics {
t.Logf("stat: %+v", stat)
}
}
Loading