test framework, docs and multi-iovec fixes

src/Makefile

@@ -7,7 +7,7 @@ E= @echo
 SRCDIR = $(shell find . -type d -not -regex './obj.*' -printf '%P ')
 OBJDIR = $(patsubst %,obj/%,$(SRCDIR))
 
-LUASRC = $(shell find . -regex '[^\#]*\.lua' -printf '%P ')
+LUASRC = $(shell find . -regex '[^\#]*\.lua' ! -name '*_t.lua' -printf '%P ')
 CSRC   = $(shell find . -regex '[^\#]*\.c'   -printf '%P ')
 CHDR   = $(shell find . -regex '[^\#]*\.h'   -printf '%P ')
 ASM    = $(shell find . -regex '[^\#]*\.dasc' -printf '%P ')

src/apps/basic/basic_apps.lua

@@ -1,5 +1,6 @@
 module(...,package.seeall)
 
+local lib = require("core.lib")
 local app = require("core.app")
 local buffer = require("core.buffer")
 local packet = require("core.packet")
@@ -221,3 +222,52 @@ function Buzz:pull () print "bzzz pull" end
 function Buzz:push () print "bzzz push" end
 
 
+--- ### Classifier app:
+--- records history of packets
+Classifier = setmetatable({}, {__index = Basic})
+function Classifier:new()
+   return setmetatable(
+      {_patterns={}, _rev={}, _counters={}, _seq={}}, 
+      {__index=Classifier})
+end
+
+function Classifier:setPatterns(pats)
+   self._patterns = pats
+   self._seq = {}
+   self._rev = {}
+   self._counters = {}
+   for k,v in pairs(pats) do
+      self._rev[v] = k
+      self._counters[k] = 0
+   end
+end
+
+function Classifier:count(ln, d)
+   local packet_name = self._rev[d]
+   if packet_name == nil then
+     packet_name = #self._patterns+1
+     self._patterns[packet_name] = d
+     self._rev[d] = packet_name
+   end
+   self._seq[#self._seq+1] = ln..':'..packet_name
+   self._counters[packet_name] = (self._counters[packet_name] or 0) + 1
+end
+
+function Classifier:repConters()
+   return table.concat(self._counters, ',')
+end
+
+function Classifier:repSequence()
+   return table.concat(self._seq, ',')
+end
+
+function Classifier:push ()
+   for ln, l in pairs(self.input) do
+      for _ = 1, link.nreadable(l) do
+        local p = link.receive(l)
+        self:count(ln, packet.tostring(p))
+        packet.deref(p)
+      end
+   end
+end
+

src/apps/intel/intel10g.lua

@@ -168,21 +168,17 @@ end
 
 --- See datasheet section 7.1 "Inline Functions -- Transmit Functionality."
 
-txdesc_flags = bits{eop=24,ifcs=25, dext=29, dtyp0=20, dtyp1=21}
+txdesc_flags = bits{ifcs=25, dext=29, dtyp0=20, dtyp1=21}
 txdesc_flags_last = bits({eop=24}, txdesc_flags)
 function M_sf:transmit (p)
-   if p.niovecs > 1 then
-      packet.coalesce(p)
-   end
    for i = 0, p.niovecs - 1 do
       local iov = p.iovecs[i]
       local flags = (i + 1 < p.niovecs) and txdesc_flags or txdesc_flags_last
       self.txdesc[self.tdt].address = iov.buffer.physical + iov.offset
       self.txdesc[self.tdt].options = bit.bor(iov.length, flags, bit.lshift(p.length+0ULL, 46))
-      self.txpackets[self.tdt] = p
+      self.txpackets[self.tdt] = packet.ref(p)
       self.tdt = (self.tdt + 1) % num_descriptors
    end
-   return packet.ref(p)
 end
 
 function M_sf:sync_transmit ()

src/apps/intel/intel10g_t.lua

@@ -0,0 +1,199 @@
+local lib      = require("core.lib")
+local packet = require "core.packet"
+local link     = require("core.link")
+local config = require("core.config")
+local app      = require("core.app")
+local basic_apps = require("apps.basic.basic_apps")
+local Intel82599 = require "apps.intel.intel_app".Intel82599
+
+
+local PCIdevA, PCIdevB = '0000:05:00.0', '0000:8a:00.0'
+
+
+local function _unhex(s)
+   local d = {}
+   for b in s:gmatch('[0-9a-fA-F][0-9a-fA-F]') do
+      d[#d+1] = tonumber(b, 16)
+   end
+   return string.char(unpack(d))
+end
+
+local function pad(s, n)
+   if not n then return s end
+   return (s..('\0'):rep(n)):sub(1, n)
+end
+
+local function make_packet(s, padlen)
+   return packet.from_data(
+      _unhex('FF:FF:FF:FF:FF:FF')..
+      _unhex('52:54:00:01:01:01')..
+      _unhex('08:00')..
+      pad(s, padlen)..
+      _unhex('00 00 00 00')
+   )
+end
+
+local function make_split_packet(s, padlen)
+   return packet.from_data(
+      _unhex('FF:FF:FF:FF:FF:FF')..
+      _unhex('52:54:00:01:01:01')..
+      _unhex('08:00'),
+      pad(s, padlen)..
+      _unhex('00 00 00 00')
+   )
+end
+
+local function _set_dst(p, dst)
+   if #dst ~= 6 then dst = _unhex(dst) end
+   assert(#dst==6, "wrong address length")
+   packet.fill_data(p, dst, 0)
+   return p
+end
+
+local function _set_src(p, src)
+   if #src ~= 6 then src = _unhex(src) end
+   assert(#src==6, "wrong address length")
+   packet.fill_data(p, src, 6)
+   return p
+end
+
+
+return {   
+   sf_to_sf = {
+      __setup = function ()
+         local c = config.new()
+         config.app(c, 'source1', basic_apps.Join)
+--         config.app(c, 'source2', basic_apps.Source)
+         config.app(c, 'nicA', Intel82599, ([[{pciaddr='%s'}]]):format(PCIdevA))
+         config.app(c, 'nicB', Intel82599, ([[{pciaddr='%s'}]]):format(PCIdevB))
+         config.app(c, 'sink', basic_apps.Classifier)
+         config.link(c, 'source1.out -> nicA.rx')
+--         config.link(c, 'source2.out -> nicB.rx')
+         config.link(c, 'nicA.tx -> sink.in1')
+         config.link(c, 'nicB.tx -> sink.in2')
+         app.configure(c)
+      end,
+
+      t = {
+         single_iovec = {
+            broadcast = function ()
+               local p = make_packet('some payload', 42)
+               _set_dst(p, 'FF:FF:FF:FF:FF:FF')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+            unicast = function ()
+               local p = make_packet('some payload', 42)
+               _set_dst(p, '52:54:00:01:01:01')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+         },
+
+         split_header = {
+            broadcast = function ()
+               local p = make_split_packet('some payload', 42)
+               _set_dst(p, 'FF:FF:FF:FF:FF:FF')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+            unicast = function ()
+               local p = make_split_packet('some payload', 42)
+               _set_dst(p, '52:54:00:01:01:01')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+         },
+      },
+   },
+
+   sf_to_vf = {
+      __setup = function ()
+         local c = config.new()
+         config.app(c, 'source1', basic_apps.Join)
+         config.app(c, 'nicAs', Intel82599, ([[{
+         -- Single App on NIC A
+            pciaddr = '%s',
+            macaddr = '52:54:00:01:01:01',
+         }]]):format(PCIdevA))
+         config.app(c, 'nicBm0', Intel82599, ([[{
+         -- first VF on NIC B
+            pciaddr = '%s',
+            vmdq = true,
+            macaddr = '52:54:00:02:02:02',
+         }]]):format(PCIdevB))
+         config.app(c, 'nicBm1', Intel82599, ([[{
+         -- second VF on NIC B
+            pciaddr = '%s',
+            vmdq = true,
+            macaddr = '52:54:00:03:03:03',
+         }]]):format(PCIdevB))
+         config.app(c, 'sink', basic_apps.Classifier)
+         config.link(c, 'source1.out -> nicAs.rx')
+         config.link(c, 'nicAs.tx -> sink.in1')
+         config.link(c, 'nicBm0.tx -> sink.in2')
+         config.link(c, 'nicBm1.tx -> sink.in3')
+         app.configure(c)
+      end,
+      t = {
+         single_iovec = {
+            broadcast = function ()
+               local p = make_packet('broadcast content', 42)
+               _set_dst(p, 'FF:FF:FF:FF:FF:FF')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same,in3:same')
+            end,
+            unicast = function ()
+               local p = make_packet('some payload', 42)
+               _set_dst(p, '52:54:00:02:02:02')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+         },
+         split_header = {
+            broadcast = function ()
+               local p = make_split_packet('broadcast content', 42)
+               _set_dst(p, 'FF:FF:FF:FF:FF:FF')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same,in3:same')
+            end,
+            unicast = function ()
+               local p = make_split_packet('some payload', 42)
+               _set_dst(p, '52:54:00:02:02:02')
+               app.app_table.sink:setPatterns({same=packet.tostring(p)})
+               link.transmit(app.app_table.source1.output.out, p)
+               app.main({duration = 1, report={showlinks=false}})
+               assert(app.app_table.sink:repSequence() == 'in2:same')
+            end,
+         },
+
+
+      },
+   },
+
+   local_A_dev = {
+      Broadcast = function ()
+         local p = packet.from_data(
+            _unhex('FF:FF:FF:FF:FF:FF')..
+            _unhex('52:54:00:01:01:01')..
+            _unhex('08:00')..
+            'this is the payload'..
+            _unhex('00:00:00:00')
+         )
+      end,
+   },
+}

src/apps/intel/intel_app.lua

@@ -115,6 +115,7 @@ function selftest ()
 
    mq_sq('0000:05:00.0', '0000:8a:00.0')
    app.main({duration = 1, report={showlinks=true, showapps=false}})
+   require ("test")('src/apps/intel/intel10g_t')
 end
 
 -- open two singlequeue drivers on both ends of the wire

src/core/buffer.lua

@@ -20,12 +20,14 @@ buffer_ptr_t = ffi.typeof("struct buffer *")
 -- This is used to return freed buffers to their devices.
 virtio_devices = {}
 
--- Return a ready-to-use buffer, or nil if none is available.
+--- ### Allocation
+
+--- Return a ready-to-use buffer, or nil if none is available.
 function allocate ()
    return freelist.remove(buffers) or new_buffer()
 end
 
--- Return a newly created buffer, or nil if none can be created.
+--- Return a newly created buffer, or nil if none can be created.
 function new_buffer ()
    assert(allocated < max, "out of buffers")
    allocated = allocated + 1
@@ -35,16 +37,16 @@ function new_buffer ()
    return b
 end
 
--- Free a buffer that is no longer in use.
+--- Free a buffer that is no longer in use.
 function free (b)
    freelist.add(buffers, b)
    if b.origin.type == C.BUFFER_ORIGIN_VIRTIO then
       virtio_devices[b.origin.info.virtio.device_id]:return_virtio_buffer(b)
    end
 end
 
--- Create buffers until at least N are ready for use.
--- This is a way to pay the cost of allocating buffer memory in advance.
+--- Create buffers until at least N are ready for use.
+--- This is a way to pay the cost of allocating buffer memory in advance.
 function preallocate (n)
    while freelist.nfree(buffers) < n do free(new_buffer()) end
 end
@@ -59,3 +61,21 @@ function delete_virtio_device (index)
    virtio_devices[index] = nil
 end
 
+-- fill's an allocated buffer with data from a string
+function fill_data (b, d, offset)
+   offset = offset or 0
+   assert (offset+#d <= b.size, "can't fit on buffer")
+   ffi.copy (b.pointer + offset, d, math.min(#d, b.size-offset))
+end
+
+-- creates a buffer from a given binary string
+function from_data (d)
+   local b = allocate()
+   local size = math.min(#d, b.size)
+   fill_data(b, d)
+   return b
+end
+
+function tostring(b, size)
+   return ffi.string(b.pointer, size or b.size)
+end

src/core/config.lua

@@ -60,7 +60,7 @@ end
 -- Example:
 --   parse_app_arg("{ timeout = 5 * 10 }") => { timeout = 50 }
 function parse_app_arg (s)
-   print("s", type(s), s)
+--   print("s", type(s), s)
    assert(type(s) == 'string')
    return loadstring("return " .. s)()
 end

src/core/lib.lua

@@ -277,27 +277,5 @@ function selftest ()
          :match('^45.00.B6.7D.00.FA.40.00.40.11$'), "wrong hex dump")
    assert(hexundump('4500 B67D 00FA400040 11', 10)
          =='\x45\x00\xb6\x7d\x00\xFA\x40\x00\x40\x11', "wrong hex undump")
-
-   local macA = new_mac('00-01-02-0a-0b-0c')
-   local macB = new_mac('0001020A0B0C')
-   local macC = new_mac('0A:0B:0C:00:01:02')
-   print ('macA', macA)
-   assert (tostring(macA) == '00:01:02:0A:0B:0C', "bad canonical MAC")
-   assert (macA == macB, "macA and macB should be equal")
-   assert (macA ~= macC, "macA and macC should be different")
-   assert (macA:subbits(0,31)==0x0a020100, "low A")
-   assert (macA:subbits(32,48)==0x0c0b, ("hi A (%X)"):format(macA:subbits(32,48)))
-   assert (macC:subbits(0,31)==0x000c0b0a, "low C")
-   assert (macC:subbits(32,48)==0x0201," hi C")
-
-   local ndx_set = new_index_set(4, 'test ndx')
-   assert (string.format('%d/%s', ndx_set:add('a'))=='0/true', "indexes start with 0, and is new")
-   assert (string.format('%d/%s', ndx_set:add('b'))=='1/true', "second new index")
-   assert (string.format('%d/%s', ndx_set:add('c'))=='2/true', "third new")
-   assert (string.format('%d/%s', ndx_set:add('b'))=='1/false', "that's an old one")
-   assert (string.format('%d/%s', ndx_set:add('a'))=='0/false', "the very first one")
-   assert (string.format('%d/%s', ndx_set:add('A'))=='3/true', "almost, but new")
-   assert (string.format('%s/%s', pcall(ndx_set.add, ndx_set,'B'))
-         :match('^false/core/lib.lua:%d+: test ndx overflow'), 'should overflow')
 end