Merge with lwaftr v2019.06.01

.version

@@ -1 +1 @@
-2019.01
+2019.06.01

lib/ljsyscall/syscall/linux/syscalls.lua

@@ -483,33 +483,30 @@ local function get_maxnumnodes()
        return math.floor(((#line+1)/9)*32)
      end
    end
+   -- If we don't know, guess that the system has a max of 1024 nodes.
+   return 1024
+end
+
+local function ensure_bitmask(mask, size)
+  if ffi.istype(t.bitmask, mask) then return mask end
+  return t.bitmask(mask, size or get_maxnumnodes())
 end
 
 function S.get_mempolicy(mode, mask, addr, flags)
   mode = mode or t.int1()
-  local size
-  if ffi.istype(t.bitmask, mask) then
-    -- if mask was provided by the caller, then use its size
-    -- and let the syscall error if it's too small
-    size = ffi.cast("uint64_t", tonumber(mask.size))
-  else
-    local mask_for_size = t.bitmask(mask)
-    -- Size should be at least equals to maxnumnodes.
-    size = ffi.cast("uint64_t", math.max(tonumber(mask_for_size.size), get_maxnumnodes()))
-    mask = t.bitmask(mask, tonumber(size))
-  end
-  local ret, err = C.get_mempolicy(mode, mask.mask, size, addr or 0, c.MPOL_FLAG[flags])
+  mask = ensure_bitmask(mask);
+  local ret, err = C.get_mempolicy(mode, mask.mask, mask.size, addr or 0, c.MPOL_FLAG[flags])
   if ret == -1 then return nil, t.error(err or errno()) end
   return { mode=mode[0], mask=mask }
 end
 function S.set_mempolicy(mode, mask)
-  mask = mktype(t.bitmask, mask)
+  mask = ensure_bitmask(mask);
   return retbool(C.set_mempolicy(c.MPOL_MODE[mode], mask.mask, mask.size))
 end
 
 function S.migrate_pages(pid, from, to)
-  from = mktype(t.bitmask, from)
-  to = mktype(t.bitmask, to)
+  from = ensure_bitmask(from);
+  to = ensure_bitmask(to, from.size)
   assert(from.size == to.size, "incompatible nodemask sizes")
   return retbool(C.migrate_pages(pid or 0, from.size, from.mask, to.mask))
 end

src/Makefile

@@ -45,7 +45,7 @@ LUAJIT_A := ../lib/luajit/src/raptorjit.a
 # for each module that has a top-level selftest () function.
 TESTMODS = $(shell find . -regex '[^\#]*\.\(lua\|dasl\)' -printf '%P ' | \
              xargs grep -s -l '^function selftest *[[:punct:]]' | \
-             sed -e 's_\.lua__' -e 's_\.dasl__' -e 's_/_._g')
+             sed -e 's_\.lua__' -e 's_\.dasl__' -e 's_/_._g' -e 's/-/_/g')
 
 # TESTSCRIPTS expands to:
 #   lib/watchdog/selftest.sh ...

src/apps/intel_mp/intel_mp.lua

@@ -133,12 +133,14 @@ EEC         0x10010 -               RW EEPROM/Flash Control Register
 EIMC        0x00888 -               RW Extended Interrupt Mask Clear
 ERRBC       0x04008 -               RC Error Byte Count
 FCCFG       0x03D00 -               RW Flow Control Configuration
+FCOERPDC    0x0241C -               RC Rx Packets Dropped Count
 FCTRL       0x05080 -               RW Filter Control
 HLREG0      0x04240 -               RW MAC Core Control 0
 ILLERRC     0x04004 -               RC Illegal Byte Error Count
 LINKS       0x042A4 -               RO Link Status Register
 MAXFRS      0x04268 -               RW Max Frame Size
 MFLCN       0x04294 -               RW MAC Flow Control Register
+MNGPDC      0x040B8 -               RO Management Packets Dropped Count
 MRQC        0x0EC80 -               RW Multiple Receive Queues Command Register
 MTQC        0x08120 -               RW Multiple Transmit Queues Command Register
 PFVTCTL     0x051B0 -               RW PF Virtual Control Register
@@ -1336,7 +1338,11 @@ function Intel82599:promisc ()
    return band(self.r.FCTRL(), lshift(1, 9)) ~= 0ULL
 end
 function Intel82599:rxbytes  () return self.r.GORC64()   end
-function Intel82599:rxdrop   () return self.r.QPRDC[0]() end
+function Intel82599:rxdrop   ()
+   local rxdrop = self.r.MNGPDC() + self.r.FCOERPDC()
+   for i=0,15 do rxdrop = rxdrop + self.r.QPRDC[i]() end
+   return rxdrop
+end
 function Intel82599:rxerrors ()
    return self.r.CRCERRS() + self.r.ILLERRC() + self.r.ERRBC() +
       self.r.RUC() + self.r.RFC() + self.r.ROC() + self.r.RJC()

src/apps/lwaftr/binding_table.lua

@@ -118,7 +118,6 @@ function BTLookupQueue:get_lookup(n)
       local streamer = self.streamer
       local pkt, b4_ipv6, br_ipv6
       pkt = self.packet_queue[n]
-      self.packet_queue[n] = nil
       if not streamer:is_empty(n) then
          b4_ipv6 = streamer.entries[n].value.b4_ipv6
          br_ipv6 = streamer.entries[n].value.br_address

src/apps/lwaftr/rangemap.lua

@@ -27,6 +27,14 @@ local function make_entry_type(value_type)
       value_type)
 end
 
+local entry_type_cache = {}
+local function get_entry_type(value_type)
+   if not entry_type_cache[value_type] then
+      entry_type_cache[value_type] = make_entry_type(value_type)
+   end
+   return entry_type_cache[value_type]
+end
+
 local function make_entries_type(entry_type)
    return ffi.typeof('$[?]', entry_type)
 end
@@ -63,7 +71,7 @@ end
 function RangeMapBuilder.new(value_type)
    local builder = {}
    builder.value_type = value_type
-   builder.entry_type = make_entry_type(builder.value_type)
+   builder.entry_type = get_entry_type(builder.value_type)
    builder.type = make_entries_type(builder.entry_type)
    builder.equal_fn = make_equal_fn(builder.value_type)
    builder.entries = {}

src/lib/README.numa.md

@@ -12,11 +12,17 @@ for some reason the current process is not bound to a NUMA node.  See
 [../doc/performance-tuning.md] for more notes on getting good
 performance out of your Snabb program.
 
-— Function **bind_to_cpu** *cpu*
+— Function **bind_to_cpu** *cpu* *skip_perf_checks*
 Bind the current process to *cpu*, arranging for it to only ever be
 run on that CPU.  Additionally, call **bind_to_numa_node** on the NUMA
 node corresponding to *cpu*.
 
+Unless the optional argument *skip_perf_checks* is true, also run some
+basic checks to verify that the given core is suitable for processing
+low-latency network traffic: that the CPU has the `performance` scaling
+governor, that it has been reserved from the kernel scheduler, and so
+on, printing out any problems to `stderr`.
+
 — Function **bind_to_numa_node** *node*
 Bind the current process to NUMA node *node*, arranging for it to only
 ever allocate memory local to that NUMA node.  Additionally, migrate
@@ -55,3 +61,18 @@ node bound by **bind_to_numa_node**, if present, and in any case that
 all *addrs* are on the same NUMA node.  If *require_affinity* is true
 (not the default), then error if a problem is detected, otherwise just
 print a warning to the console.
+
+— Function **parse_cpuset** *cpus*
+A helper function to parse a CPU set from a string.  A CPU set is either
+the number of a CPU, a range of CPUs, or two or more CPU sets joined by
+commas.  The result is a table whose keys are the CPUs and whose values
+are true (a set).  For example, q`parse_cpuset("1-3,5")` will return a
+table with keys 1, 2, 3, and 5 bound to `true`.
+
+— Function **node_cpus** *node*
+Return a set of CPUs belonging to NUMA node *node*, in the same format
+as in **parse_cpuset**.
+
+— Function **isolated_cpus**
+Return a set of CPUs that have been "isolated" away from the kernel at
+boot via the `isolcpus` kernel boot parameter.

src/lib/binary_search.dasl

@@ -35,6 +35,7 @@ local function assemble (name, prototype, generator)
    return ffi.cast(prototype, mcode)
 end
 
+local gencache = {} -- Cache for generated variants (reuse if possible.)
 function gen(count, entry_type)
    local function gen_binary_search(Dst)
       if count == 1 then
@@ -80,9 +81,19 @@ function gen(count, entry_type)
       | mov rax, rdi
       | ret
    end
-   return assemble("binary_search_"..count,
-                   ffi.typeof("$*(*)($*, uint32_t)", entry_type, entry_type),
-                   gen_binary_search)
+   -- Assemble binary search variant and cache it unless it has not been
+   -- previously generated.
+   if not gencache[entry_type] then
+      gencache[entry_type] = {}
+   end
+   if not gencache[entry_type][count] then
+      gencache[entry_type][count] =
+         assemble("binary_search_"..count,
+                  ffi.typeof("$*(*)($*, uint32_t)", entry_type, entry_type),
+                  gen_binary_search)
+   end
+   -- Return (now) cached routine.
+   return gencache[entry_type][count]
 end
 
 function selftest ()

src/lib/cpuset.lua

@@ -19,53 +19,7 @@ do
    end
 end
 
-local function trim (str)
-   return str:gsub("^%s", ""):gsub("%s$", "")
-end
-
-local function parse_cpulist (cpus)
-   local ret = {}
-   cpus = trim(cpus)
-   if #cpus == 0 then return ret end
-   for range in cpus:split(',') do
-      local lo, hi = range:match("^%s*([^%-]*)%s*-%s*([^%-%s]*)%s*$")
-      if lo == nil then lo = range:match("^%s*([^%-]*)%s*$") end
-      assert(lo ~= nil, 'invalid range: '..range)
-      lo = assert(tonumber(lo), 'invalid range begin: '..lo)
-      assert(lo == math.floor(lo), 'invalid range begin: '..lo)
-      if hi ~= nil then
-         hi = assert(tonumber(hi), 'invalid range end: '..hi)
-         assert(hi == math.floor(hi), 'invalid range end: '..hi)
-         assert(lo < hi, 'invalid range: '..range)
-      else
-         hi = lo
-      end
-      for cpu=lo,hi do table.insert(ret, cpu) end
-   end
-   return ret
-end
-
-local function parse_cpulist_from_file (path)
-   local fd = assert(io.open(path))
-   if not fd then return {} end
-   local ret = parse_cpulist(fd:read("*all"))
-   fd:close()
-   return ret
-end
-
 local function available_cpus (node)
-   local function set (t)
-      local ret = {}
-      for _,v in pairs(t) do ret[tostring(v)] = true end
-      return ret
-   end
-   local function cpus_in_node (node)
-      local node_path = '/sys/devices/system/node/node'..node
-      return set(parse_cpulist_from_file(node_path..'/cpulist'))
-   end
-   local function isolated_cpus ()
-      return set(parse_cpulist_from_file('/sys/devices/system/cpu/isolated'))
-   end
    local function subtract (s, t)
       local ret = {}
       for k,_ in pairs(s) do
@@ -75,7 +29,7 @@ local function available_cpus (node)
       return ret
    end
    -- XXX: Add sched_getaffinity cpus.
-   return subtract(cpus_in_node(node), isolated_cpus())
+   return subtract(numa.node_cpus(node), numa.isolated_cpus())
 end
 
 function CPUSet:bind_to_numa_node()
@@ -87,7 +41,7 @@ function CPUSet:bind_to_numa_node()
       numa.bind_to_numa_node(nodes[1])
       local cpus = available_cpus(nodes[1])
       assert(#cpus > 0, 'Not available CPUs')
-      numa.bind_to_cpu(cpus)
+      numa.bind_to_cpu(cpus, 'skip-perf-checks')
       print(("Bound main process to NUMA node: %s (CPU %s)"):format(nodes[1], cpus[1]))
    else
       print("CPUs available from multiple NUMA nodes: "..table.concat(nodes, ","))
@@ -96,7 +50,7 @@ function CPUSet:bind_to_numa_node()
 end
 
 function CPUSet:add_from_string(cpus)
-   for _, cpu in ipairs(parse_cpulist(cpus)) do
+   for cpu,_ in pairs(numa.parse_cpuset(cpus)) do
       self:add(cpu)
    end
 end
@@ -157,14 +111,3 @@ function CPUSet:release(cpu)
    end
    error('CPU not found on NUMA node: '..cpu..', '..node)
 end
-
-function selftest ()
-   print('selftest: cpuset')
-   local cpus = parse_cpulist("0-5,7")
-   assert(#cpus == 7 and cpus[6] == 5 and cpus[7] == 7)
-   cpus = parse_cpulist("1")
-   assert(#cpus == 1 and cpus[1] == 1)
-   assert(#parse_cpulist("\n") == 0)
-   assert(#parse_cpulist("") == 0)
-   print('selftest: ok')
-end

src/lib/ctable.lua

@@ -4,13 +4,10 @@ local ffi = require("ffi")
 local C = ffi.C
 local S = require("syscall")
 local lib = require("core.lib")
-local util = require("lib.yang.util")
 local binary_search = require("lib.binary_search")
 local multi_copy = require("lib.multi_copy")
 local siphash = require("lib.hash.siphash")
 
--- TODO: Move to core/lib.lua.
-local memoize = util.memoize
 local min, max, floor, ceil = math.min, math.max, math.floor, math.ceil
 
 CTable = {}
@@ -430,21 +427,6 @@ function CTable:remove(key, missing_allowed)
    return true
 end
 
-local function generate_multi_copy(width, size)
-   return multi_copy.gen(width, size)
-end
-generate_multi_copy = memoize(generate_multi_copy)
-
-local function generate_multi_hash(self, width)
-   return self.make_multi_hash_fn(width)
-end
-generate_multi_hash = memoize(generate_multi_hash)
-
-local function generate_binary_search(entries_per_lookup, entry_type)
-   return binary_search.gen(entries_per_lookup, entry_type)
-end
-generate_binary_search = memoize(generate_binary_search)
-
 function CTable:make_lookup_streamer(width)
    assert(width > 0 and width <= 262144, "Width value out of range: "..width)
    local res = {
@@ -463,6 +445,9 @@ function CTable:make_lookup_streamer(width)
       -- more entry.
       stream_entries = self.type(width * (self.max_displacement + 1) + 1)
    }
+   -- Pointer to first entry key (cache to avoid cdata allocation.)
+   local key_offset = 4 -- Skip past uint32_t hash.
+   res.keys = ffi.cast('uint8_t*', res.entries) + key_offset
    -- Give res.pointers sensible default values in case the first lookup
    -- doesn't fill the pointers vector.
    for i = 0, width-1 do res.pointers[i] = self.entries end
@@ -475,23 +460,23 @@ function CTable:make_lookup_streamer(width)
    -- Compile multi-copy and binary-search procedures that are
    -- specialized for this table and this width.
    local entry_size = ffi.sizeof(self.entry_type)
-   res.multi_copy = generate_multi_copy(width, res.entries_per_lookup * entry_size)
-   res.multi_hash = generate_multi_hash(self, width)
-   res.binary_search = generate_binary_search(res.entries_per_lookup, self.entry_type)
+   res.multi_copy = multi_copy.gen(width, res.entries_per_lookup * entry_size)
+   res.multi_hash = self.make_multi_hash_fn(width)
+   res.binary_search = binary_search.gen(res.entries_per_lookup, self.entry_type)
 
    return setmetatable(res, { __index = LookupStreamer })
 end
 
 function LookupStreamer:stream()
    local width = self.width
    local entries = self.entries
+   local keys = self.keys
    local pointers = self.pointers
    local stream_entries = self.stream_entries
    local entries_per_lookup = self.entries_per_lookup
    local equal_fn = self.equal_fn
 
-   local key_offset = 4 -- Skip past uint32_t hash.
-   self.multi_hash(ffi.cast('uint8_t*', entries) + key_offset, self.hashes)
+   self.multi_hash(self.keys, self.hashes)
 
    for i=0,width-1 do
       local hash = self.hashes[i]

src/lib/fibers/timer.lua

@@ -129,8 +129,8 @@ local function tick_outer(inner, outer)
       local ent = pop_node(head)
       local idx = math.floor((ent.time - outer.now) * inner.rate)
       -- Because of floating-point imprecision it's possible to get an
-      -- index that is too large by 1.
-      idx = math.min(idx, WHEEL_SLOTS-1)
+      -- index that falls just outside [0,WHEEL_SLOTS-1].
+      idx = math.max(math.min(idx, WHEEL_SLOTS-1), 0)
       push_node(ent, inner.slots[idx])
    end
    outer.cur = band(outer.cur + 1, SLOT_INDEX_MASK)