From e15b305006b574d9a321bc41b4b438d9dd2055a1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 28 Sep 2017 17:25:24 +0200 Subject: [PATCH 1/5] Workers can communicate next-hop MAC address between themselves Add shared_next_mac_key to ARP configuration to allow workers behind RSS to share next-hop information. --- src/apps/ipv4/README.md | 11 +++++++++++ src/apps/ipv4/arp.lua | 43 ++++++++++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/apps/ipv4/README.md b/src/apps/ipv4/README.md index 2fb121fe6d..97b4914267 100644 --- a/src/apps/ipv4/README.md +++ b/src/apps/ipv4/README.md @@ -58,6 +58,17 @@ it will be determined from the *next_ip* via ARP. *Optional*. The IPv4 address of the next-hop host. Required only if *next_mac* is not specified as part of the configuration. +— Key **shared_next_mac_key** + +*Optional*. Path to a shared memory location +(i.e. */var/run/snabb/PID/PATH*) in which to store the resolved +next_mac. This ARP resolver might be part of a set of peer processes +sharing work via RSS. In that case, an ARP response will probably +arrive only to one of the RSS processes, not to all of them. If you are +using ARP behind RSS, set *shared_next_mac_key* to, for example, +`group/arp-next-mac`, to enable the different workers to communicate the +next-hop MAC address. + ## Reassembler (apps.ipv4.reassemble) The `Reassembler` app is a filter on incoming IPv4 packets that diff --git a/src/apps/ipv4/arp.lua b/src/apps/ipv4/arp.lua index ab2b638e58..f29179adaa 100644 --- a/src/apps/ipv4/arp.lua +++ b/src/apps/ipv4/arp.lua @@ -16,6 +16,7 @@ local ffi = require("ffi") local packet = require("core.packet") local link = require("core.link") local lib = require("core.lib") +local shm = require("core.shm") local datagram = require("lib.protocol.datagram") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") @@ -65,6 +66,7 @@ struct { local ether_arp_header_t = ffi.typeof( 'struct { $ ether; $ arp; } __attribute__((packed))', ether_header_t, arp_header_t) +local mac_t = ffi.typeof('uint8_t[6]') local ether_header_ptr_t = ffi.typeof('$*', ether_header_t) local ether_header_len = ffi.sizeof(ether_header_t) local ether_arp_header_ptr_t = ffi.typeof('$*', ether_arp_header_t) @@ -121,7 +123,7 @@ end local function ipv4_eq(a, b) return C.memcmp(a, b, 4) == 0 end local function copy_mac(src) - local dst = ffi.new('uint8_t[6]') + local dst = mac_t() ffi.copy(dst, src, 6) return dst end @@ -146,6 +148,12 @@ local arp_config_params = { next_ip = { default=false }, -- Emits an alarm notification on arp-resolving and arp-resolved. alarm_notification = { default=false }, + -- This ARP resolver might be part of a set of peer processes sharing + -- work via RSS. In that case, a response will probably arrive only + -- at one process, not all of them! In that case we can arrange for + -- the ARP app that receives the reply to write the resolved next-hop + -- to a shared file. RSS peers can poll that file. + shared_next_mac_key = {}, } function ARP:new(conf) @@ -156,7 +164,7 @@ function ARP:new(conf) if not o.next_mac then assert(o.next_ip, 'ARP needs next-hop IPv4 address to learn next-hop MAC') o.arp_request_pkt = make_arp_request(o.self_mac, o.self_ip, o.next_ip) - self.arp_request_interval = 3 -- Send a new arp_request every three seconds. + o.arp_request_interval = 3 -- Send a new arp_request every three seconds. end return setmetatable(o, {__index=ARP}) end @@ -182,11 +190,28 @@ function ARP:send_arp_request (output) transmit(output, packet.clone(self.arp_request_pkt)) end -function ARP:arp_resolved (ip, mac) +function ARP:arp_resolved (ip, mac, provenance) print(("ARP: '%s' resolved (%s)"):format(ipv4:ntop(ip), ethernet:ntop(mac))) if self.alarm_notification then resolve_alarm:clear() end + self.next_mac = mac + if self.shared_next_mac_key then + local ok, shared_mac = pcall(shm.create, self.shared_next_mac_key, mac_t) + if not ok then + ok, shared_mac = pcall(shm.open, self.shared_next_mac_key, mac_t) + end + if not ok then + print('warning: failed to open shared next MAC key!') + elseif provenance == 'remote' then + -- If we are getting this information from a packet and not + -- from the shared key, then update the shared key. + ffi.copy(shared_mac, mac, 6) + else + assert(provenance == 'peer') + -- Pass. + end + end end function ARP:push() @@ -213,9 +238,7 @@ function ARP:push() end elseif ntohs(h.arp.oper) == arp_oper_reply then if self.next_ip and ipv4_eq(h.arp.spa, self.next_ip) then - local next_mac = copy_mac(h.arp.sha) - self:arp_resolved(self.next_ip, next_mac) - self.next_mac = next_mac + self:arp_resolved(self.next_ip, copy_mac(h.arp.sha), 'remote') end else -- Incoming ARP that isn't handled; drop it silently. @@ -235,6 +258,11 @@ function ARP:push() e.shost = self.self_mac transmit(osouth, p) end + elseif self.shared_next_mac_key then + local ok, mac = pcall(shm.open, self.shared_next_mac_key, mac_t) + -- Use the shared pointer directly, without copying; if it is ever + -- updated, we will get its new value. + if ok then self:arp_resolved(self.next_ip, mac, 'peer') end end end @@ -242,7 +270,8 @@ function selftest() print('selftest: arp') local arp = ARP:new({ self_ip = ipv4:pton('1.2.3.4'), - next_ip = ipv4:pton('5.6.7.8') }) + next_ip = ipv4:pton('5.6.7.8'), + shared_next_mac_key = "foo" }) arp.input = { south=link.new('south in'), north=link.new('north in') } arp.output = { south=link.new('south out'), north=link.new('north out') } From c72c6c2dcb2bd563ecc52cb170104b095c2ba96b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 28 Sep 2017 17:35:26 +0200 Subject: [PATCH 2/5] Workers can communicate next-hop MAC address between themselves (NDP) Add shared_next_mac_key to NDP configuration to allow workers behind RSS to share next-hop information. --- src/apps/lwaftr/ndp.lua | 52 ++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/src/apps/lwaftr/ndp.lua b/src/apps/lwaftr/ndp.lua index dd40f77c43..97f0449572 100644 --- a/src/apps/lwaftr/ndp.lua +++ b/src/apps/lwaftr/ndp.lua @@ -23,6 +23,7 @@ local ffi = require("ffi") local packet = require("core.packet") local link = require("core.link") local lib = require("core.lib") +local shm = require("core.shm") local checksum = require("lib.checksum") local datagram = require("lib.protocol.datagram") local ethernet = require("lib.protocol.ethernet") @@ -48,6 +49,7 @@ local htons, ntohs = lib.htons, lib.ntohs local htonl, ntohl = lib.htonl, lib.ntohl local receive, transmit = link.receive, link.transmit +local mac_t = ffi.typeof('uint8_t[6]') local ether_header_t = ffi.typeof [[ /* All values in network byte order. */ struct { @@ -280,6 +282,12 @@ local ndp_config_params = { is_router = { default=true }, -- Emit alarms if set. alarm_notification = { default=false }, + -- This NDP resolver might be part of a set of peer processes sharing + -- work via RSS. In that case, a response will probably arrive only + -- at one process, not all of them! In that case we can arrange for + -- the NDP app that receives the reply to write the resolved next-hop + -- to a shared file. RSS peers can poll that file. + shared_next_mac_key = {}, } function NDP:new(conf) @@ -312,11 +320,28 @@ function NDP:maybe_send_ns_request (output) end end -function NDP:ndp_resolved (ip, mac) +function NDP:ndp_resolved (ip, mac, provenance) print(("NDP: '%s' resolved (%s)"):format(ipv6:ntop(ip), ethernet:ntop(mac))) if self.alarm_notification then resolve_alarm:clear() end + self.next_mac = mac + if self.shared_next_mac_key then + local ok, shared_mac = pcall(shm.create, self.shared_next_mac_key, mac_t) + if not ok then + ok, shared_mac = pcall(shm.open, self.shared_next_mac_key, mac_t) + end + if not ok then + print('warning: failed to open shared next MAC key!') + elseif provenance == 'remote' then + -- If we are getting this information from a packet and not + -- from the shared key, then update the shared key. + ffi.copy(shared_mac, mac, 6) + else + assert(provenance == 'peer') + -- Pass. + end + end end function NDP:resolve_next_hop(next_mac) @@ -325,8 +350,7 @@ function NDP:resolve_next_hop(next_mac) -- link layer address in the NDP options). Just take the first -- one. if self.next_mac then return end - self:ndp_resolved(self.next_ip, next_mac) - self.next_mac = next_mac + self:ndp_resolved(self.next_ip, next_mac, 'remote') end local function copy_mac(src) @@ -430,17 +454,21 @@ function NDP:push() end end - for _ = 1, link.nreadable(inorth) do - local p = receive(inorth) - if not self.next_mac then - -- drop all southbound packets until the next hop's ethernet address is known - packet.free(p) - else + -- Don't read southbound packets until the next hop's ethernet + -- address is known. + if self.next_mac then + for _ = 1, link.nreadable(inorth) do + local p = receive(inorth) local h = ffi.cast(ether_header_ptr_t, p.data) h.shost = self.self_mac h.dhost = self.next_mac transmit(osouth, p) end + elseif self.shared_next_mac_key then + local ok, mac = pcall(shm.open, self.shared_next_mac_key, mac_t) + -- Use the shared pointer directly, without copying; if it is ever + -- updated, we will get its new value. + if ok then self:ndp_resolved(self.next_ip, mac, 'peer') end end end @@ -451,9 +479,11 @@ function selftest() local sink = require("apps.basic.basic_apps").Sink local c = config.new() config.app(c, "nd1", NDP, { self_ip = ipv6:pton("2001:DB8::1"), - next_ip = ipv6:pton("2001:DB8::2") }) + next_ip = ipv6:pton("2001:DB8::2"), + shared_next_mac_key = "foo" }) config.app(c, "nd2", NDP, { self_ip = ipv6:pton("2001:DB8::2"), - next_ip = ipv6:pton("2001:DB8::1") }) + next_ip = ipv6:pton("2001:DB8::1"), + shared_next_mac_key = "bar" }) config.app(c, "sink1", sink) config.app(c, "sink2", sink) config.link(c, "nd1.south -> nd2.south") From f66b085410661afcacb18c03868800cb1a698a5a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 28 Sep 2017 17:37:10 +0200 Subject: [PATCH 3/5] lwAFTR configures ARP and NDP with shared_next_mac_key Hopefully fixes #981. --- src/program/lwaftr/setup.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index f344ff1295..2a64677778 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -117,12 +117,14 @@ function lwaftr_app(c, conf) { self_ip = iinternal_interface.ip, self_mac = iinternal_interface.mac, next_mac = iinternal_interface.next_hop.mac, + shared_next_mac_key = "group/ipv6-next-mac", next_ip = iinternal_interface.next_hop.ip, alarm_notification = conf.alarm_notification }) config.app(c, "arp", arp.ARP, { self_ip = convert_ipv4(iexternal_interface.ip), self_mac = iexternal_interface.mac, next_mac = iexternal_interface.next_hop.mac, + shared_next_mac_key = "group/ipv4-next-mac", next_ip = convert_ipv4(iexternal_interface.next_hop.ip), alarm_notification = conf.alarm_notification }) From 59fcb911c0cd526853852a97281f3dc1bbe2383e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 29 Sep 2017 12:04:22 +0200 Subject: [PATCH 4/5] Avoid leaking shared next MAC key in ARP --- src/apps/ipv4/arp.lua | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/apps/ipv4/arp.lua b/src/apps/ipv4/arp.lua index f29179adaa..bfd09ba0d3 100644 --- a/src/apps/ipv4/arp.lua +++ b/src/apps/ipv4/arp.lua @@ -197,16 +197,19 @@ function ARP:arp_resolved (ip, mac, provenance) end self.next_mac = mac if self.shared_next_mac_key then - local ok, shared_mac = pcall(shm.create, self.shared_next_mac_key, mac_t) - if not ok then - ok, shared_mac = pcall(shm.open, self.shared_next_mac_key, mac_t) - end - if not ok then - print('warning: failed to open shared next MAC key!') - elseif provenance == 'remote' then + if provenance == 'remote' then -- If we are getting this information from a packet and not -- from the shared key, then update the shared key. - ffi.copy(shared_mac, mac, 6) + local ok, shared = pcall(shm.create, self.shared_next_mac_key, mac_t) + if not ok then + ok, shared = pcall(shm.open, self.shared_next_mac_key, mac_t) + end + if not ok then + print('warning: arp: failed to update shared next MAC key!') + else + ffi.copy(shared, mac, 6) + shm.unmap(shared) + end else assert(provenance == 'peer') -- Pass. From 8216ef6601f7a5514286c420a9db53094c4c2c58 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 29 Sep 2017 12:05:21 +0200 Subject: [PATCH 5/5] Avoid leaking shared next MAC key in NDP --- src/apps/lwaftr/ndp.lua | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/apps/lwaftr/ndp.lua b/src/apps/lwaftr/ndp.lua index 97f0449572..d8a2a9e469 100644 --- a/src/apps/lwaftr/ndp.lua +++ b/src/apps/lwaftr/ndp.lua @@ -327,16 +327,19 @@ function NDP:ndp_resolved (ip, mac, provenance) end self.next_mac = mac if self.shared_next_mac_key then - local ok, shared_mac = pcall(shm.create, self.shared_next_mac_key, mac_t) - if not ok then - ok, shared_mac = pcall(shm.open, self.shared_next_mac_key, mac_t) - end - if not ok then - print('warning: failed to open shared next MAC key!') - elseif provenance == 'remote' then + if provenance == 'remote' then -- If we are getting this information from a packet and not -- from the shared key, then update the shared key. - ffi.copy(shared_mac, mac, 6) + local ok, shared = pcall(shm.create, self.shared_next_mac_key, mac_t) + if not ok then + ok, shared = pcall(shm.open, self.shared_next_mac_key, mac_t) + end + if not ok then + print('warning: ndp: failed to update shared next MAC key!') + else + ffi.copy(shared, mac, 6) + shm.unmap(shared) + end else assert(provenance == 'peer') -- Pass.