From 4bd951002cdec2296bc8939bf63fc1bc863f7b2e Mon Sep 17 00:00:00 2001 From: Atul Patel Date: Mon, 26 Aug 2024 05:45:13 +0100 Subject: [PATCH] fix(evpn_bridge): feature ecmp Signed-off-by: atulpatel261194 --- config.yaml | 10 ++ pkg/config/config.go | 39 ++++---- pkg/frr/frr.go | 4 +- pkg/netlink/common.go | 15 ++- pkg/netlink/neighbor.go | 1 + pkg/netlink/netlink_watcher.go | 32 ++++--- pkg/netlink/nexthop.go | 165 ++++++++++++++++++++++----------- pkg/netlink/route.go | 35 +++++-- 8 files changed, 207 insertions(+), 94 deletions(-) diff --git a/config.yaml b/config.yaml index bea7a5cc..205c12a8 100644 --- a/config.yaml +++ b/config.yaml @@ -21,6 +21,16 @@ grpc: server_port: 51703 num_threads: 10 static_external_macs: [] +interfaces: + phyports: + - rep: "enp0s1f0d1" + vsi: 0 + - rep: "enp0s1f0d2" + vsi: 1 + grpcacc: "host" + grpchost: "00:20:00:00:14:48" + vrfmux: "enp0s1f0d4" + portmux: "enp0s1f0d5" linuxfrr: enabled: true defaultvtep: "vxlan-vtep" diff --git a/pkg/config/config.go b/pkg/config/config.go index 7cae4e6d..e1f7fa76 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -30,19 +30,14 @@ type P4FilesConfig struct { // RepresentorsConfig Representors config structure type RepresentorsConfig struct { - PortMux string `yaml:"port-mux"` - VrfMux string `yaml:"vrf_mux"` - GrpcAcc string `yaml:"host"` - GrpcHost string `yaml:"grpc_host"` - Phy0Rep string `yaml:"phy0_rep"` - Phy1Rep string `yaml:"phy1_rep"` + Phy0Rep string `yaml:"phy0_rep"` + Phy1Rep string `yaml:"phy1_rep"` } // P4Config p4 config structure type P4Config struct { - Enabled bool `yaml:"enabled"` - Representors map[string]interface{} `yaml:"representors"` - Config P4FilesConfig `yaml:"config"` + Enabled bool `yaml:"enabled"` + Config P4FilesConfig `yaml:"config"` } // loglevelConfig log level config structure @@ -58,20 +53,29 @@ type loglevelConfig struct { type LinuxFrrConfig struct { Enabled bool `yaml:"enabled"` DefaultVtep string `yaml:"defaultvtep"` - PortMux string `yaml:"portmux"` - VrfMux string `yaml:"vrfmux"` IPMtu int `yaml:"ipmtu"` LocalAs int `yaml:"localas"` } +// InterfaceConfig linux frr config structure +type InterfaceConfig struct { + PhyPorts []struct { + Rep string `yaml:"rep"` + Vsi int `yaml:"vsi"` + } `yaml:"phyports"` + Other map[string]interface{} `yaml:",inline"` + GrpcAcc string `yaml:"grpcacc"` + GrpcHost string `yaml:"grpchost"` + VrfMux string `yaml:"vrfmux"` + PortMux string `yaml:"port-mux"` +} + // NetlinkConfig netlink config structure type NetlinkConfig struct { - Enabled bool `yaml:"enabled"` - PollInterval int `yaml:"pollinterval"` - PhyPorts []struct { - Name string `yaml:"name"` - Vsi int `yaml:"vsi"` - } `yaml:"phyports"` + Enabled bool `yaml:"enabled"` + PollInterval int `yaml:"pollinterval"` + GrdDefaultRoute bool `yaml:"grddefaultroute"` + EnableEcmp bool `yaml:"enableecmp"` } // Config global config structure @@ -85,6 +89,7 @@ type Config struct { Buildenv string `yaml:"buildenv"` Tracer bool `yaml:"tracer"` Subscribers []SubscriberConfig `yaml:"subscribers"` + Interfaces InterfaceConfig `yaml:"interfaces"` LinuxFrr LinuxFrrConfig `yaml:"linuxfrr"` Netlink NetlinkConfig `yaml:"netlink"` P4 P4Config `yaml:"p4"` diff --git a/pkg/frr/frr.go b/pkg/frr/frr.go index 70fe534d..9b8eafde 100644 --- a/pkg/frr/frr.go +++ b/pkg/frr/frr.go @@ -276,8 +276,8 @@ func Initialize() { } defaultVtep = config.GlobalConfig.LinuxFrr.DefaultVtep localas = config.GlobalConfig.LinuxFrr.LocalAs - portMux = config.GlobalConfig.LinuxFrr.PortMux - vrfMux = config.GlobalConfig.LinuxFrr.VrfMux + portMux = config.GlobalConfig.Interfaces.PortMux + vrfMux = config.GlobalConfig.Interfaces.VrfMux log.Printf(" frr vtep: %+v port-mux %+v vrf-mux: +%v", defaultVtep, portMux, vrfMux) // Subscribe to InfraDB notifications subscribeInfradb(&config.GlobalConfig) diff --git a/pkg/netlink/common.go b/pkg/netlink/common.go index 709ba4f1..3e5f71eb 100644 --- a/pkg/netlink/common.go +++ b/pkg/netlink/common.go @@ -26,7 +26,13 @@ var EventBus = eb.NewEventBus() // pollInterval variable var pollInterval int -// dbphyPortslock variable +// grd default route bool variable +var grdDefaultRoute bool + +// enable ecmp bool variable +var enableEcmp bool + +// phyPorts variable var phyPorts = make(map[string]int) // stopMonitoring variable @@ -236,7 +242,12 @@ func notify_changes(new_db map[interface{}]interface{}, old_db map[interface{}]i continue } } - notifyAddDel(v1, event.Operation.Update) + if event.EventType == ROUTE { + notifyAddDel(v2, event.Operation.Delete) + notifyAddDel(v1, event.Operation.Add) + } else { + notifyAddDel(v1, event.Operation.Update) + } } delete(new_db, k1) delete(old_db, k1) diff --git a/pkg/netlink/neighbor.go b/pkg/netlink/neighbor.go index 1e404df8..fe5a51ab 100644 --- a/pkg/netlink/neighbor.go +++ b/pkg/netlink/neighbor.go @@ -84,6 +84,7 @@ func parseNeigh(nm []NeighIPStruct, v string) NeighList { for _, nd := range nm { var ns NeighStruct ns.Neigh0.Type = OTHER + ns.Type = OTHER ns.VrfName = v if nd.Dev != "" { vrf, _ := vn.LinkByName(nd.Dev) diff --git a/pkg/netlink/netlink_watcher.go b/pkg/netlink/netlink_watcher.go index c577dd39..9e980bd4 100644 --- a/pkg/netlink/netlink_watcher.go +++ b/pkg/netlink/netlink_watcher.go @@ -100,14 +100,21 @@ func annotateDBEntries() { // readLatestNetlinkState reads the latest netlink state func readLatestNetlinkState() { - vrfs, _ := infradb.GetAllVrfs() - for _, v := range vrfs { - readNeighbors(v) // viswanantha library - readRoutes(v) // Viswantha library - } - m := readFDB() - for i := 0; i < len(m); i++ { - m[i].addFdbEntry() + grdVrf, err := infradb.GetVrf("//network.opiproject.org/vrfs/GRD") + if err == nil { + readNeighbors(grdVrf) + readRoutes(grdVrf) + vrfs, _ := infradb.GetAllVrfs() + for _, v := range vrfs { + if v.Name != grdVrf.Name { + readNeighbors(v) // viswanantha library + readRoutes(v) // Viswantha library + } + } + m := readFDB() + for i := 0; i < len(m); i++ { + m[i].addFdbEntry() + } } dumpDBs() } @@ -186,17 +193,20 @@ func Initialize() { pollInterval = config.GlobalConfig.Netlink.PollInterval log.Printf("netlink: poll interval: %v", pollInterval) nlEnabled := config.GlobalConfig.Netlink.Enabled + + grdDefaultRoute = config.GlobalConfig.Netlink.GrdDefaultRoute + enableEcmp = config.GlobalConfig.Netlink.EnableEcmp + if !nlEnabled { log.Printf("netlink: netlink_monitor disabled") return } - for i := 0; i < len(config.GlobalConfig.Netlink.PhyPorts); i++ { - phyPorts[config.GlobalConfig.Netlink.PhyPorts[i].Name] = config.GlobalConfig.Netlink.PhyPorts[i].Vsi + for i := 0; i < len(config.GlobalConfig.Interfaces.PhyPorts); i++ { + phyPorts[config.GlobalConfig.Interfaces.PhyPorts[i].Rep] = config.GlobalConfig.Interfaces.PhyPorts[i].Vsi } getlink() ctx = context.Background() nlink = utils.NewNetlinkWrapperWithArgs(config.GlobalConfig.Tracer) - // stopMonitoring = false stopMonitoring.Store(false) go monitorNetlink() // monitor Thread started } diff --git a/pkg/netlink/nexthop.go b/pkg/netlink/nexthop.go index e4e2993f..fd9674a9 100644 --- a/pkg/netlink/nexthop.go +++ b/pkg/netlink/nexthop.go @@ -20,6 +20,7 @@ type NexthopKey struct { Dst string Dev int Local bool + Weight int } // NexthopStruct contains nexthop structure @@ -38,6 +39,10 @@ type NexthopStruct struct { Neighbor *NeighStruct NhType int Metadata map[interface{}]interface{} + Dir int + Divisor int + Value float64 + Hashes []int } // nexthopOperations add, update, delete @@ -65,12 +70,14 @@ const ( // Nexthop type const ( // NexthopStruct TYPE & L2NEXTHOP TYPE & FDBentry PHY = iota + VRFNEIGHBOR SVI ACC VXLAN BRIDGEPORT OTHER IGNORE + ECMP ) // checkNhDB checks the neighbor database @@ -83,23 +90,67 @@ func checkNhDB(nhKey NexthopKey) bool { return false } +// deepCopyMetadata deep copies the metadata +func deepCopyMetadata(originalMap map[interface{}]interface{}) map[interface{}]interface{} { + newMap := make(map[interface{}]interface{}) + for key, value := range originalMap { + newMap[key] = value + } + return newMap +} + // tryResolve resolves the neighbor -func (nexthop *NexthopStruct) tryResolve() *NexthopStruct { +func (nexthop *NexthopStruct) tryResolve() []*NexthopStruct { + var retNexthopSt []*NexthopStruct + if nexthop.Metadata == nil { + nexthop.Metadata = make(map[interface{}]interface{}) + } if nexthop.nexthop.Gw != nil { // Nexthops with a gateway IP need resolution of that IP neighborKey := NeighKey{Dst: nexthop.nexthop.Gw.String(), VrfName: nexthop.Vrf.Name, Dev: nexthop.nexthop.LinkIndex} ch := checkNeigh(neighborKey) - if ch && latestNeighbors[neighborKey].Type != IGNORE { - nexthop.Resolved = true - nh := latestNeighbors[neighborKey] - nexthop.Neighbor = &nh - } else { + if ch { + if nexthop.NhType == VXLAN { + nexthop.Metadata["remote_vtep_ip"] = nexthop.nexthop.Gw.String() + nh := latestNeighbors[neighborKey] + nexthop.Metadata["inner_dmac"] = nh.Neigh0.HardwareAddr.String() + VRF, _ := infradb.GetVrf("//network.opiproject.org/vrfs/GRD") + r, ok := lookupRoute(nexthop.nexthop.Gw, VRF) + if ok { + for _, grdNexthop := range r.Nexthops { + arrayOfNexthops := grdNexthop.tryResolve() + if len(arrayOfNexthops) != 0 { + nexthopSt := *nexthop + nexthopSt.nexthop.Gw = grdNexthop.nexthop.Gw + nexthopSt.nexthop.LinkIndex = grdNexthop.nexthop.LinkIndex + nexthopSt.Key = NexthopKey{nexthopSt.Vrf.Name, nexthopSt.nexthop.Gw.String(), nexthopSt.nexthop.LinkIndex, nexthopSt.Local, nexthopSt.Weight} + nexthopSt.Neighbor = grdNexthop.Neighbor + nexthopSt.Weight = grdNexthop.Weight + nexthopSt.RouteRefs = nexthop.RouteRefs + nexthopSt.Metadata = deepCopyMetadata(nexthop.Metadata) + + nexthopSt.Resolved = true + retNexthopSt = append(retNexthopSt, &nexthopSt) + } + } + } + return retNexthopSt + } else if nexthop.NhType >= 0 { + nexthop.Resolved = true + nh := latestNeighbors[neighborKey] + nexthop.Neighbor = &nh + return []*NexthopStruct{nexthop} + } nexthop.Resolved = false + nexthop.Neighbor = nil + return nil } - } else { - nexthop.Resolved = true + nexthop.Resolved = false + nexthop.Neighbor = nil + return nil } - return nexthop + nexthop.Resolved = true + return []*NexthopStruct{nexthop} } // NHAssignID returns the nexthop id @@ -115,26 +166,40 @@ func NHAssignID(key NexthopKey) int { } // addNexthop adds the nexthop +// +//nolint func (nexthop *NexthopStruct) addNexthop(r *RouteStruct) *RouteStruct { + if len(r.Nexthops) > 0 && !enableEcmp { + log.Printf("ECMP disabled: Ignoring additional nexthop of route") + return nil + } ch := checkNhDB(nexthop.Key) if ch { - nh0 := latestNexthop[nexthop.Key] + NH0 := latestNexthop[nexthop.Key] // Links route with existing nexthop - nh0.RouteRefs = append(nh0.RouteRefs, r) - r.Nexthops = append(r.Nexthops, nh0) - } else { - // Create a new nexthop entry + NH0.RouteRefs = append(NH0.RouteRefs, r) + r.Nexthops = append(r.Nexthops, NH0) + } else if nexthop.Resolved { nexthop.RouteRefs = append(nexthop.RouteRefs, r) nexthop.ID = NHAssignID(nexthop.Key) - nexthop = nexthop.tryResolve() latestNexthop[nexthop.Key] = nexthop r.Nexthops = append(r.Nexthops, nexthop) + } else { + nexthops := nexthop.tryResolve() + for _, nexthop := range nexthops { + r = nexthop.addNexthop(r) + } } return r } // ParseNexthop parses the neighbor +// +//nolint func (nexthop *NexthopStruct) ParseNexthop(v *infradb.Vrf, rc RouteCmdInfo) { + var phyFlag bool + phyFlag = false + nexthop.Weight = 1 nexthop.Vrf = v if rc.Dev != "" { @@ -168,20 +233,27 @@ func (nexthop *NexthopStruct) ParseNexthop(v *infradb.Vrf, rc RouteCmdInfo) { if rc.Weight >= 0 { nexthop.Weight = rc.Weight } - nexthop.Key = NexthopKey{nexthop.Vrf.Name, nexthop.nexthop.Gw.String(), nexthop.nexthop.LinkIndex, nexthop.Local} -} -// nolint -func (nexthop *NexthopStruct) annotate() { - nexthop.Metadata = make(map[interface{}]interface{}) - var phyFlag bool - phyFlag = false for k := range phyPorts { if nameIndex[nexthop.nexthop.LinkIndex] == k { phyFlag = true } } - if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && nexthop.nexthop.LinkIndex != 0 && strings.HasPrefix(nameIndex[nexthop.nexthop.LinkIndex], path.Base(nexthop.Vrf.Name)+"-") && !nexthop.Local { + if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && phyFlag && !nexthop.Local { + nexthop.NhType = PHY + } else if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && nexthop.nexthop.LinkIndex != 0 && strings.HasPrefix(nameIndex[nexthop.nexthop.LinkIndex], path.Base(nexthop.Vrf.Name)+"-") && !nexthop.Local { + nexthop.NhType = VRFNEIGHBOR + } else if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && nameIndex[nexthop.nexthop.LinkIndex] == fmt.Sprintf("br-%s", path.Base(nexthop.Vrf.Name)) && !nexthop.Local { + nexthop.NhType = VXLAN + } else { + nexthop.NhType = ACC + } + nexthop.Key = NexthopKey{nexthop.Vrf.Name, nexthop.nexthop.Gw.String(), nexthop.nexthop.LinkIndex, nexthop.Local, nexthop.Weight} +} + +// nolint +func (nexthop *NexthopStruct) annotate() { + if nexthop.NhType == VRFNEIGHBOR { nexthop.NhType = SVI link, _ := vn.LinkByName(nameIndex[nexthop.nexthop.LinkIndex]) if nexthop.Neighbor != nil { @@ -215,11 +287,10 @@ func (nexthop *NexthopStruct) annotate() { } } else { nexthop.Resolved = false - log.Printf("netlink: Failed to gather data for nexthop on physical port\n") + log.Printf("netlink: Failed to gather data for nexthop on physical port with nexthop is %+v\n", nexthop) } } - } else if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && phyFlag && !nexthop.Local { - nexthop.NhType = PHY + } else if nexthop.NhType == PHY { link1, _ := vn.LinkByName(nameIndex[nexthop.nexthop.LinkIndex]) if link1 == nil { return @@ -233,10 +304,9 @@ func (nexthop *NexthopStruct) annotate() { } } else { nexthop.Resolved = false - log.Printf("netlink: Failed to gather data for nexthop on physical port") + log.Printf("netlink: Failed to gather data for nexthop on physical port with nexthop is %+v\n", nexthop) } - } else if (nexthop.nexthop.Gw != nil && !nexthop.nexthop.Gw.IsUnspecified()) && nameIndex[nexthop.nexthop.LinkIndex] == fmt.Sprintf("br-%s", path.Base(nexthop.Vrf.Name)) && !nexthop.Local { - nexthop.NhType = VXLAN + } else if nexthop.NhType == VXLAN { v, _ := infradb.GetVrf(nexthop.Vrf.Name) var detail map[string]interface{} var rmac net.HardwareAddr @@ -270,34 +340,19 @@ func (nexthop *NexthopStruct) annotate() { } vtepip := v.Spec.VtepIP.IP nexthop.Metadata["local_vtep_ip"] = vtepip.String() - nexthop.Metadata["remote_vtep_ip"] = nexthop.nexthop.Gw.String() nexthop.Metadata["vni"] = *nexthop.Vrf.Spec.Vni - if nexthop.Neighbor != nil { - nexthop.Metadata["inner_dmac"] = nexthop.Neighbor.Neigh0.HardwareAddr.String() - v, err := infradb.GetVrf("//network.opiproject.org/vrfs/GRD") - if err == nil { - r, ok := lookupRoute(nexthop.nexthop.Gw, v) - if ok { - // For now pick the first physical nexthop (no ECMP yet) - phyNh := r.Nexthops[0] - link, _ := vn.LinkByName(nameIndex[phyNh.nexthop.LinkIndex]) - nexthop.Metadata["phy_smac"] = link.Attrs().HardwareAddr.String() - nexthop.Metadata["egress_vport"] = phyPorts[nameIndex[phyNh.nexthop.LinkIndex]] - if phyNh.Neighbor != nil { - nexthop.Metadata["phy_dmac"] = phyNh.Neighbor.Neigh0.HardwareAddr.String() - } else { - // The VXLAN nexthop can only be installed when the phy_nexthops are Resolved. - nexthop.Resolved = false - } - } - } else { - log.Printf("netlink: No GRD found :%v\n", err) + if nexthop.Neighbor.Type == PHY { + r, ok := lookupRoute(nexthop.nexthop.Gw, v) + if ok { + phyNh := r.Nexthops[0] + link, _ := vn.LinkByName(nameIndex[phyNh.nexthop.LinkIndex]) + nexthop.Metadata["phy_smac"] = link.Attrs().HardwareAddr.String() + nexthop.Metadata["egress_vport"] = phyPorts[nameIndex[phyNh.nexthop.LinkIndex]] + nexthop.Metadata["phy_dmac"] = nexthop.Neighbor.Neigh0.HardwareAddr.String() // link.Attrs().HardwareAddr.String() } - } else { - nexthop.Resolved = false } - } else { - nexthop.NhType = ACC + } else if nexthop.NhType == ACC { + //nexthop.NhType = ACC link1, err := vn.LinkByName("rep-" + path.Base(nexthop.Vrf.Name)) if err != nil { log.Printf("netlink: Error in getting rep information: %v\n", err) @@ -313,6 +368,8 @@ func (nexthop *NexthopStruct) annotate() { } else { nexthop.Metadata["vlanID"] = *nexthop.Vrf.Metadata.RoutingTable[0] } + } else { + nexthop.Resolved = false } } diff --git a/pkg/netlink/route.go b/pkg/netlink/route.go index f80cb5b5..3c8d0562 100644 --- a/pkg/netlink/route.go +++ b/pkg/netlink/route.go @@ -102,6 +102,14 @@ type RouteKey struct { Dst string } +// RcNexthop structure of route nexthops +type RcNexthop struct { + Gateway string + Dev string + Flags []string + Weight int +} + // RouteCmdInfo structure type RouteCmdInfo struct { Type string @@ -118,6 +126,7 @@ type RouteCmdInfo struct { VRF *infradb.Vrf Table int NhInfo NhRouteInfo // {id gateway Dev scope protocol flags} + Nexthops []RcNexthop } /*-------------------------------------------------------------------------- @@ -216,6 +225,7 @@ func (route *RouteStruct) addRoute() { nexthops := route.Nexthops route.Nexthops = []*NexthopStruct{} for _, nexthop := range nexthops { + nexthop.Metadata = make(map[interface{}]interface{}) route = nexthop.addNexthop(route) } latestRoutes[route.Key] = route @@ -251,9 +261,18 @@ func ParseRoute(v *infradb.Vrf, rc []RouteCmdInfo, t int) RouteList { r0.Type = routeTypeLocal } var rs RouteStruct - var nh NexthopStruct rs.Vrf = v - if r0.Nhid != 0 || r0.Gateway != "" || r0.Dev != "" { + if len(r0.Nexthops) > 1 { + for _, n := range r0.Nexthops { + var nh NexthopStruct + r0.Gateway = n.Gateway + r0.Dev = n.Dev + r0.Weight = n.Weight + nh.ParseNexthop(v, r0) + rs.Nexthops = append(rs.Nexthops, &nh) + } + } else if r0.Nhid != 0 || r0.Gateway != "" || r0.Dev != "" { + var nh NexthopStruct nh.ParseNexthop(v, r0) rs.Nexthops = append(rs.Nexthops, &nh) } @@ -338,7 +357,11 @@ func ParseRoute(v *infradb.Vrf, rc []RouteCmdInfo, t int) RouteList { rs.Key = RouteKey{Table: rs.Route0.Table, Dst: rs.Route0.Dst.String()} if rs.preFilterRoute() { route.RS = append(route.RS, &rs) + } else if rs.checkRoute() { + rou := latestRoutes[rs.Key] + route.RS = append(route.RS, rou) } + } return route } @@ -421,10 +444,6 @@ func CheckRdup(tmpKey RouteKey) bool { // annotate function annonates the entries func (route *RouteStruct) annotate() { route.Metadata = make(map[interface{}]interface{}) - for i := 0; i < len(route.Nexthops); i++ { - nexthop := route.Nexthops[i] - route.Metadata["nh_ids"] = nexthop.ID - } if route.Vrf.Spec.Vni != nil { route.Metadata["vrf_id"] = *route.Vrf.Spec.Vni } else { @@ -540,7 +559,7 @@ func (route *RouteStruct) installFilterRoute() bool { // preFilterRoute pre filter the routes func (route *RouteStruct) preFilterRoute() bool { - if checkRtype(route.NlType) && !route.Route0.Dst.IP.IsLoopback() && route.Route0.Dst.IP.String() != "0.0.0.0" { + if checkRtype(route.NlType) && !route.Route0.Dst.IP.IsLoopback() && route.Route0.Dst.IP.String() != "0.0.0.0" && !grdDefaultRoute { return true } @@ -578,7 +597,7 @@ func dumpRouteDB() string { } else { via = n.Route0.Gw.String() } - str := fmt.Sprintf("Route(vrf=%s dst=%s type=%s proto=%s metric=%d via=%s dev=%s nhid= %d Table= %d)", n.Vrf.Name, n.Route0.Dst.String(), n.NlType, n.getProto(), n.Route0.Priority, via, nameIndex[n.Route0.LinkIndex], n.Nexthops[0].ID, n.Route0.Table) + str := fmt.Sprintf("Route(vrf=%s dst=%s type=%s proto=%s metric=%d via=%s dev=%s nhid= %+v Table= %d)", n.Vrf.Name, n.Route0.Dst.String(), n.NlType, n.getProto(), n.Route0.Priority, via, nameIndex[n.Route0.LinkIndex], n.Nexthops, n.Route0.Table) log.Println(str) s += str s += "\n"