Fix ARP and NDP next-hop detection with RSS

src/apps/ipv4/README.md

@@ -58,6 +58,17 @@ it will be determined from the *next_ip* via ARP.
 *Optional*.  The IPv4 address of the next-hop host.  Required only if
  *next_mac* is not specified as part of the configuration.
 
+— Key **shared_next_mac_key**
+
+*Optional*.  Path to a shared memory location
+(i.e. */var/run/snabb/PID/PATH*) in which to store the resolved
+next_mac.  This ARP resolver might be part of a set of peer processes
+sharing work via RSS.  In that case, an ARP response will probably
+arrive only to one of the RSS processes, not to all of them.  If you are
+using ARP behind RSS, set *shared_next_mac_key* to, for example,
+`group/arp-next-mac`, to enable the different workers to communicate the
+next-hop MAC address.
+
 ## Reassembler (apps.ipv4.reassemble)
 
 The `Reassembler` app is a filter on incoming IPv4 packets that

src/apps/ipv4/arp.lua

@@ -16,6 +16,7 @@ local ffi      = require("ffi")
 local packet   = require("core.packet")
 local link     = require("core.link")
 local lib      = require("core.lib")
+local shm      = require("core.shm")
 local datagram = require("lib.protocol.datagram")
 local ethernet = require("lib.protocol.ethernet")
 local ipv4     = require("lib.protocol.ipv4")
@@ -65,6 +66,7 @@ struct {
 local ether_arp_header_t = ffi.typeof(
    'struct { $ ether; $ arp; } __attribute__((packed))',
    ether_header_t, arp_header_t)
+local mac_t = ffi.typeof('uint8_t[6]')
 local ether_header_ptr_t = ffi.typeof('$*', ether_header_t)
 local ether_header_len = ffi.sizeof(ether_header_t)
 local ether_arp_header_ptr_t = ffi.typeof('$*', ether_arp_header_t)
@@ -121,7 +123,7 @@ end
 local function ipv4_eq(a, b) return C.memcmp(a, b, 4) == 0 end
 
 local function copy_mac(src)
-   local dst = ffi.new('uint8_t[6]')
+   local dst = mac_t()
    ffi.copy(dst, src, 6)
    return dst
 end
@@ -146,6 +148,12 @@ local arp_config_params = {
    next_ip  = { default=false },
    -- Emits an alarm notification on arp-resolving and arp-resolved.
    alarm_notification = { default=false },
+   -- This ARP resolver might be part of a set of peer processes sharing
+   -- work via RSS.  In that case, a response will probably arrive only
+   -- at one process, not all of them!  In that case we can arrange for
+   -- the ARP app that receives the reply to write the resolved next-hop
+   -- to a shared file.  RSS peers can poll that file.
+   shared_next_mac_key = {},
 }
 
 function ARP:new(conf)
@@ -156,7 +164,7 @@ function ARP:new(conf)
    if not o.next_mac then
       assert(o.next_ip, 'ARP needs next-hop IPv4 address to learn next-hop MAC')
       o.arp_request_pkt = make_arp_request(o.self_mac, o.self_ip, o.next_ip)
-      self.arp_request_interval = 3 -- Send a new arp_request every three seconds.
+      o.arp_request_interval = 3 -- Send a new arp_request every three seconds.
    end
    return setmetatable(o, {__index=ARP})
 end
@@ -182,11 +190,31 @@ function ARP:send_arp_request (output)
    transmit(output, packet.clone(self.arp_request_pkt))
 end
 
-function ARP:arp_resolved (ip, mac)
+function ARP:arp_resolved (ip, mac, provenance)
    print(("ARP: '%s' resolved (%s)"):format(ipv4:ntop(ip), ethernet:ntop(mac)))
    if self.alarm_notification then
       resolve_alarm:clear()
    end
+   self.next_mac = mac
+   if self.shared_next_mac_key then
+      if provenance == 'remote' then
+         -- If we are getting this information from a packet and not
+         -- from the shared key, then update the shared key.
+         local ok, shared = pcall(shm.create, self.shared_next_mac_key, mac_t)
+         if not ok then
+            ok, shared = pcall(shm.open, self.shared_next_mac_key, mac_t)
+         end
+         if not ok then
+            print('warning: arp: failed to update shared next MAC key!')
+         else
+            ffi.copy(shared, mac, 6)
+            shm.unmap(shared)
+         end
+      else
+         assert(provenance == 'peer')
+         -- Pass.
+      end
+   end
 end
 
 function ARP:push()
@@ -213,9 +241,7 @@ function ARP:push()
             end
          elseif ntohs(h.arp.oper) == arp_oper_reply then
             if self.next_ip and ipv4_eq(h.arp.spa, self.next_ip) then
-               local next_mac = copy_mac(h.arp.sha)
-               self:arp_resolved(self.next_ip, next_mac)
-               self.next_mac = next_mac
+               self:arp_resolved(self.next_ip, copy_mac(h.arp.sha), 'remote')
             end
          else
             -- Incoming ARP that isn't handled; drop it silently.
@@ -235,14 +261,20 @@ function ARP:push()
         e.shost = self.self_mac
         transmit(osouth, p)
      end
+   elseif self.shared_next_mac_key then
+      local ok, mac = pcall(shm.open, self.shared_next_mac_key, mac_t)
+      -- Use the shared pointer directly, without copying; if it is ever
+      -- updated, we will get its new value.
+      if ok then self:arp_resolved(self.next_ip, mac, 'peer') end
    end
 end
 
 function selftest()
    print('selftest: arp')
 
    local arp = ARP:new({ self_ip = ipv4:pton('1.2.3.4'),
-                         next_ip = ipv4:pton('5.6.7.8') })
+                         next_ip = ipv4:pton('5.6.7.8'),
+                         shared_next_mac_key = "foo" })
    arp.input  = { south=link.new('south in'),  north=link.new('north in') }
    arp.output = { south=link.new('south out'), north=link.new('north out') }
 

src/apps/lwaftr/ndp.lua

@@ -23,6 +23,7 @@ local ffi      = require("ffi")
 local packet   = require("core.packet")
 local link     = require("core.link")
 local lib      = require("core.lib")
+local shm      = require("core.shm")
 local checksum = require("lib.checksum")
 local datagram = require("lib.protocol.datagram")
 local ethernet = require("lib.protocol.ethernet")
@@ -48,6 +49,7 @@ local htons, ntohs = lib.htons, lib.ntohs
 local htonl, ntohl = lib.htonl, lib.ntohl
 local receive, transmit = link.receive, link.transmit
 
+local mac_t = ffi.typeof('uint8_t[6]')
 local ether_header_t = ffi.typeof [[
 /* All values in network byte order.  */
 struct {
@@ -280,6 +282,12 @@ local ndp_config_params = {
    is_router = { default=true },
    -- Emit alarms if set.
    alarm_notification = { default=false },
+   -- This NDP resolver might be part of a set of peer processes sharing
+   -- work via RSS.  In that case, a response will probably arrive only
+   -- at one process, not all of them!  In that case we can arrange for
+   -- the NDP app that receives the reply to write the resolved next-hop
+   -- to a shared file.  RSS peers can poll that file.
+   shared_next_mac_key = {},
 }
 
 function NDP:new(conf)
@@ -312,11 +320,31 @@ function NDP:maybe_send_ns_request (output)
    end
 end
 
-function NDP:ndp_resolved (ip, mac)
+function NDP:ndp_resolved (ip, mac, provenance)
    print(("NDP: '%s' resolved (%s)"):format(ipv6:ntop(ip), ethernet:ntop(mac)))
    if self.alarm_notification then
       resolve_alarm:clear()
    end
+   self.next_mac = mac
+   if self.shared_next_mac_key then
+      if provenance == 'remote' then
+         -- If we are getting this information from a packet and not
+         -- from the shared key, then update the shared key.
+         local ok, shared = pcall(shm.create, self.shared_next_mac_key, mac_t)
+         if not ok then
+            ok, shared = pcall(shm.open, self.shared_next_mac_key, mac_t)
+         end
+         if not ok then
+            print('warning: ndp: failed to update shared next MAC key!')
+         else
+            ffi.copy(shared, mac, 6)
+            shm.unmap(shared)
+         end
+      else
+         assert(provenance == 'peer')
+         -- Pass.
+      end
+   end
 end
 
 function NDP:resolve_next_hop(next_mac)
@@ -325,8 +353,7 @@ function NDP:resolve_next_hop(next_mac)
    -- link layer address in the NDP options).  Just take the first
    -- one.
    if self.next_mac then return end
-   self:ndp_resolved(self.next_ip, next_mac)
-   self.next_mac = next_mac
+   self:ndp_resolved(self.next_ip, next_mac, 'remote')
 end
 
 local function copy_mac(src)
@@ -430,17 +457,21 @@ function NDP:push()
       end
    end
 
-   for _ = 1, link.nreadable(inorth) do
-      local p = receive(inorth)
-      if not self.next_mac then
-         -- drop all southbound packets until the next hop's ethernet address is known
-         packet.free(p)
-      else
+   -- Don't read southbound packets until the next hop's ethernet
+   -- address is known.
+   if self.next_mac then
+      for _ = 1, link.nreadable(inorth) do
+         local p = receive(inorth)
          local h = ffi.cast(ether_header_ptr_t, p.data)
          h.shost = self.self_mac
          h.dhost = self.next_mac
          transmit(osouth, p)
       end
+   elseif self.shared_next_mac_key then
+      local ok, mac = pcall(shm.open, self.shared_next_mac_key, mac_t)
+      -- Use the shared pointer directly, without copying; if it is ever
+      -- updated, we will get its new value.
+      if ok then self:ndp_resolved(self.next_ip, mac, 'peer') end
    end
 end
 
@@ -451,9 +482,11 @@ function selftest()
    local sink = require("apps.basic.basic_apps").Sink
    local c = config.new()
    config.app(c, "nd1", NDP, { self_ip  = ipv6:pton("2001:DB8::1"),
-                               next_ip  = ipv6:pton("2001:DB8::2") })
+                               next_ip  = ipv6:pton("2001:DB8::2"),
+                               shared_next_mac_key = "foo" })
    config.app(c, "nd2", NDP, { self_ip  = ipv6:pton("2001:DB8::2"),
-                               next_ip  = ipv6:pton("2001:DB8::1") })
+                               next_ip  = ipv6:pton("2001:DB8::1"),
+                               shared_next_mac_key = "bar" })
    config.app(c, "sink1", sink)
    config.app(c, "sink2", sink)
    config.link(c, "nd1.south -> nd2.south")

src/program/lwaftr/setup.lua

@@ -117,12 +117,14 @@ function lwaftr_app(c, conf)
               { self_ip = iinternal_interface.ip,
                 self_mac = iinternal_interface.mac,
                 next_mac = iinternal_interface.next_hop.mac,
+                shared_next_mac_key = "group/ipv6-next-mac",
                 next_ip = iinternal_interface.next_hop.ip,
                 alarm_notification = conf.alarm_notification })
    config.app(c, "arp", arp.ARP,
               { self_ip = convert_ipv4(iexternal_interface.ip),
                 self_mac = iexternal_interface.mac,
                 next_mac = iexternal_interface.next_hop.mac,
+                shared_next_mac_key = "group/ipv4-next-mac",
                 next_ip = convert_ipv4(iexternal_interface.next_hop.ip),
                 alarm_notification = conf.alarm_notification })