diff --git a/src/Makefile.vita b/src/Makefile.vita index b9b5bf8e55..9941d4ad1c 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -2,9 +2,10 @@ INCLUDE = *.* core arch jit syscall pf \ lib/token_bucket.* lib/tsc.* \ lib/lua lib/protocol lib/checksum.* lib/ipsec \ lib/yang lib/stream.* lib/stream lib/buffer.* \ - lib/xsd_regexp.* lib/maxpc.* lib/ctable.* lib/binary_search.* \ + lib/xsd_regexp.* lib/maxpc.* lib/ctable.* lib/cltable.* \ + lib/binary_search.* lib/multi_copy.* lib/hash \ lib/ptree lib/rrd.* lib/fibers \ - lib/multi_copy.* lib/hash lib/cltable.* lib/lpm lib/interlink.* \ + lib/poptrie* lib/interlink.* \ lib/hardware lib/macaddress.* lib/numa.* lib/cpuset.* \ lib/scheduling.* lib/timers \ apps/basic apps/interlink apps/intel_mp \ diff --git a/src/doc/genbook.sh b/src/doc/genbook.sh index c8182f6a84..79274460df 100755 --- a/src/doc/genbook.sh +++ b/src/doc/genbook.sh @@ -84,6 +84,8 @@ $(cat $mdroot/lib/README.checksum.md) $(cat $mdroot/lib/README.ctable.md) +$(cat $mdroot/lib/README.poptrie.md) + $(cat $mdroot/lib/README.pmu.md) $(cat $mdroot/lib/yang/README.md) diff --git a/src/lib/README.poptrie.md b/src/lib/README.poptrie.md new file mode 100644 index 0000000000..9756b6274f --- /dev/null +++ b/src/lib/README.poptrie.md @@ -0,0 +1,86 @@ +### Poptrie (lib.poptrie) + +An implementation of +[Poptrie](http://conferences.sigcomm.org/sigcomm/2015/pdf/papers/p57.pdf). +Includes high-level functions for building the Poptrie data structure, as well +as a hand-written, optimized assembler lookup routine. + +#### Example usage + +```lua +local pt = poptrie.new{direct_pointing=true} +-- Associate prefixes of length to values (uint16_t) +pt:add(0x00FF, 8, 1) +pt:add(0x000F, 4, 2) +pt:build() +pt:lookup64(0x001F) ⇒ 2 +pt:lookup64(0x10FF) ⇒ 1 +-- The value zero denotes "no match" +pt:lookup64(0x0000) ⇒ 0 +-- You can create a pre-built poptrie from its backing memory. +local pt2 = poptrie.new{ + nodes = pt.nodes, + leaves = pt.leaves, + directmap = pt.directmap +} +``` + +#### Known bugs and limitations + + - Only supports keys up to 64 bits wide + +#### Performance + +- Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz (Haswell, Turbo off) + +``` +PMU analysis (numentries=10000, keysize=32) +build: 0.1290 seconds +lookup: 13217.09 cycles/lookup 28014.35 instructions/lookup +lookup64: 122.94 cycles/lookup 133.22 instructions/lookup +build(direct_pointing): 0.1056 seconds +lookup(direct_pointing): 5519.01 cycles/lookup 11412.01 instructions/lookup +lookup64(direct_pointing): 89.82 cycles/lookup 70.72 instructions/lookup +``` + +#### Interface + +— Function **new** *init* + +Creates and returns a new `Poptrie` object. + +*Init* is a table with the following keys: + +* `direct_pointing` - *Optional*. Boolean that governs whether to use the + *direct pointing* optimization. Default is `false`. +* `s` - *Optional*. Bits to use for the *direct pointing* optimization. + Default is 18. Note that the direct map array will be 2×2ˢ bytes in size. +* `leaves` - *Optional*. An array of leaves. When *leaves* is supplied *nodes* + must be supplied as well. +* `nodes` - *Optional*. An array of nodes. When *nodes* is supplied *leaves* + must be supplied as well. +* `directmap` - *Optional*. A direct map array. When *directmap* is supplied, + *nodes* and *leaves* must be supplied as well and *direct_pointing* is + implicit. + +— Method **Poptrie:add** *prefix* *length* *value* + +Associates *value* to *prefix* of *length*. *Prefix* must be an unsigned +integer (little-endian) of up to 64 bits. *Length* must be an an unsigned +integer between 1 and 64. *Value* must be a 16‑bit unsigned integer, and should +be greater than zero (see `lookup64` as to why.) + +— Method **Poptrie:build** + +Compiles the optimized poptrie data structure used by `lookup64`. After calling +this method, the *leaves* and *nodes* fields of the `Poptrie` object will +contain the leaves and nodes arrays respectively. These arrays can be used to +construct a `Poptrie` object. + +— Method **Poptrie:lookup64** *key* + +Looks up *key* in the `Poptrie` object and returns the associated value or +zero. *Key* must be an unsigned, little-endian integer of up to 64 bits. + +Unless the `Poptrie` object was initialized with leaves and nodes arrays, the +user must call `Poptrie:build` before calling `Poptrie:lookup64`. diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua new file mode 100644 index 0000000000..ab768eb651 --- /dev/null +++ b/src/lib/poptrie.lua @@ -0,0 +1,501 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local debug = false + +-- Poptrie, see +-- http://conferences.sigcomm.org/sigcomm/2015/pdf/papers/p57.pdf + +local ffi = require("ffi") +local bit = require("bit") +local band, bor, lshift, rshift, bnot = + bit.band, bit.bor, bit.lshift, bit.rshift, bit.bnot + +local poptrie_lookup = require("lib.poptrie_lookup") + +local Poptrie = { + leaf_compression = true, + direct_pointing = false, + k = 6, + s = 18, + leaf_tag = lshift(1, 31), + leaf_t = ffi.typeof("uint16_t"), + vector_t = ffi.typeof("uint64_t"), + base_t = ffi.typeof("uint32_t"), + num_leaves = 100, + num_nodes = 10 +} +Poptrie.node_t = ffi.typeof([[struct { + $ leafvec, vector; + $ base0, base1; +} __attribute__((packed))]], Poptrie.vector_t, Poptrie.base_t) + +local function array (t, n) + return ffi.new(ffi.typeof("$[?]", t), n) +end + +function new (init) + local self = setmetatable({}, {__index=Poptrie}) + if init.leaves and init.nodes then + self.leaves, self.num_leaves = init.leaves, assert(init.num_leaves) + self.nodes, self.num_nodes = init.nodes, assert(init.num_nodes) + elseif init.nodes or init.leaves or init.directmap then + error("partial init") + else + self.leaves = array(Poptrie.leaf_t, Poptrie.num_leaves) + self.nodes = array(Poptrie.node_t, Poptrie.num_nodes) + end + if init.directmap then + self.directmap = init.directmap + self.direct_pointing = true + else + if init.direct_pointing ~= nil then + self.direct_pointing = init.direct_pointing + end + if init.s ~= nil then + self.s = init.s + end + if self.direct_pointing then + self.directmap = array(Poptrie.base_t, 2^self.s) + end + end + self.asm_lookup64 = poptrie_lookup.generate(self, 64) + return self +end + +function Poptrie:grow_nodes () + self.num_nodes = self.num_nodes * 2 + local new_nodes = array(Poptrie.node_t, self.num_nodes) + ffi.copy(new_nodes, self.nodes, ffi.sizeof(self.nodes)) + self.nodes = new_nodes +end + +function Poptrie:grow_leaves () + self.num_leaves = self.num_leaves * 2 + local new_leaves = array(Poptrie.leaf_t, self.num_leaves) + ffi.copy(new_leaves, self.leaves, ffi.sizeof(self.leaves)) + self.leaves = new_leaves +end + +-- XXX - Generalize for key=uint8_t[?] +local function extract (key, offset, length) + return band(rshift(key+0ULL, offset), lshift(1, length) - 1) +end + +-- Add key/value pair to RIB (intermediary binary trie) +-- key=uint64_t, length=uint16_t, value=uint16_t +function Poptrie:add (key, length, value) + assert(value) + local function add (node, offset) + if offset == length then + node.value = value + elseif extract(key, offset, 1) == 0 then + node.left = add(node.left or {}, offset + 1) + elseif extract(key, offset, 1) == 1 then + node.right = add(node.right or {}, offset + 1) + else error("invalid state") end + return node + end + self.rib = add(self.rib or {}, 0) +end + +-- Longest prefix match on RIB +function Poptrie:rib_lookup (key, length, root) + local function lookup (node, offset, value) + value = node.value or value + if offset == length then + return value, (node.left or node.right) and node + elseif node.left and extract(key, offset, 1) == 0 then + return lookup(node.left, offset + 1, value) + elseif node.right and extract(key, offset, 1) == 1 then + return lookup(node.right, offset + 1, value) + else + -- No match: return longest prefix key value, but no node. + return value + end + end + return lookup(root or self.rib, 0) +end + +-- Map f over keys of length in RIB +function Poptrie:rib_map (f, length, root) + local function map (node, offset, key, value) + value = (node and node.value) or value + local left, right = node and node.left, node and node.right + if offset == length then + f(key, value, (left or right) and node) + else + map(left, offset + 1, key, value) + map(right, offset + 1, bor(key, lshift(1, offset)), value) + end + end + return map(root or self.rib, 0, 0) +end + +function Poptrie:clear_fib () + self.leaf_base, self.node_base = 0, 0 + ffi.fill(self.leaves, ffi.sizeof(self.leaves), 0) + ffi.fill(self.nodes, ffi.sizeof(self.nodes), 0) +end + +function Poptrie:allocate_leaf () + while self.leaf_base >= self.num_leaves do + self:grow_leaves() + end + self.leaf_base = self.leaf_base + 1 + return self.leaf_base - 1 +end + +function Poptrie:allocate_node () + if self.direct_pointing then + -- When using direct_pointing, the node index space is split into half in + -- favor of a bit used for disambiguation in Poptrie:build_directmap. + assert(band(self.node_base, Poptrie.leaf_tag) == 0, "Node overflow") + end + while self.node_base >= self.num_nodes do + self:grow_nodes() + end + self.node_base = self.node_base + 1 + return self.node_base - 1 +end + +function Poptrie:build_node (rib, node_index, default) + -- Initialize node base pointers. + do local node = self.nodes[node_index] + -- Note: have to be careful about keeping direct references of nodes + -- around as they can get invalidated when the backing array is grown. + node.base0 = self.leaf_base + node.base1 = self.node_base + end + -- Compute leaves and children + local leaves, children = {}, {} + local function collect (key, value, node) + leaves[key], children[key] = value, node + end + self:rib_map(collect, Poptrie.k, rib) + -- Allocate and initialize node.leafvec and leaves. + local last_leaf_value = nil + for index = 0, 2^Poptrie.k - 1 do + if not children[index] then + local value = leaves[index] or default or 0 + if value ~= last_leaf_value then -- always true when leaf_compression=false + if Poptrie.leaf_compression then + local node = self.nodes[node_index] + node.leafvec = bor(node.leafvec, lshift(1ULL, index)) + last_leaf_value = value + end + local leaf_index = self:allocate_leaf() + self.leaves[leaf_index] = value + end + end + end + -- Allocate child nodes (this has to be done before recursing into build() + -- because their indices into the nodes array need to be node.base1 + index, + -- and build() will advance the node_base.) + local child_nodes = {} + for index = 0, 2^Poptrie.k - 1 do + if children[index] then + child_nodes[index] = self:allocate_node() + end + end + -- Initialize node.vector and child nodes. + for index = 0, 2^Poptrie.k - 1 do + if children[index] then + local node = self.nodes[node_index] + node.vector = bor(node.vector, lshift(1ULL, index)) + self:build_node(children[index], + child_nodes[index], + leaves[index] or default) + end + end +end + +-- Build direct index array for RIB +function Poptrie:build_directmap (rib) + local function build (index, value, node) + if node then + self.directmap[index] = self:allocate_node() + self:build_node(node, self.directmap[index], value) + else + self.directmap[index] = bor(value or 0, Poptrie.leaf_tag) + end + end + self:rib_map(build, self.s, rib) +end + +-- Compress RIB into Poptrie +function Poptrie:build () + self:clear_fib() + if self.direct_pointing then + self:build_directmap(self.rib) + else + self:build_node(self.rib, self:allocate_node()) + end +end + +-- http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetNaive +local function popcnt (v) -- XXX - popcaan is 64-bit only + local c = 0 + while v > 0 do + c = c + band(v, 1ULL) + v = rshift(v, 1ULL) + end + return c +end + +-- [Algorithm 1] lookup(t = (N , L), key); the lookup procedure for the address +-- key in the tree t (when k = 6). The function extract(key, off, len) extracts +-- bits of length len, starting with the offset off, from the address key. +-- N and L represent arrays of internal nodes and leaves, respectively. +-- << denotes the shift instruction of bits. Numerical literals with the UL and +-- ULL suffixes denote 32-bit and 64-bit unsigned integers, respectively. +-- Vector and base are the variables to hold the contents of the node’s fields. +-- +-- if [direct_pointing] then +-- index = extract(key, 0, t.s); +-- dindex = t.D[index].direct index; +-- if (dindex & (1UL << 31)) then +-- return dindex & ((1UL << 31) - 1); +-- end if +-- index = dindex; +-- offset = t.s; +-- else +-- index = 0; +-- offset = 0; +-- end if +-- vector = t.N [index].vector; +-- v = extract(key, offset, 6); +-- while (vector & (1ULL << v)) do +-- base = t.N [index].base1; +-- bc = popcnt(vector & ((2ULL << v) - 1)); +-- index = base + bc - 1; +-- vector = t.N [index].vector; +-- offset += 6; +-- v = extract(key, offset, 6); +-- end while +-- base = t.N [index].base0; +-- if [leaf_compression] then +-- bc = popcnt(t.N [index].leafvec & ((2ULL << v) - 1)); +-- else +-- bc = popcnt((∼t.N [index].vector) & ((2ULL << v) - 1)); +-- end if +-- return t.L[base + bc - 1]; +-- +function Poptrie:lookup (key) + local N, L, D = self.nodes, self.leaves, self.directmap + local index, offset = 0, 0 + if self.direct_pointing then + offset = self.s + index = D[extract(key, 0, offset)] + if debug then print(bin(index), band(index, Poptrie.leaf_tag - 1)) end + if band(index, Poptrie.leaf_tag) ~= 0 then + return band(index, Poptrie.leaf_tag - 1) -- direct leaf, strip tag + end + end + local node = N[index] + local v = extract(key, offset, Poptrie.k) + if debug then print(index, bin(node.vector), bin(v)) end + while band(node.vector, lshift(1ULL, v)) ~= 0 do + local base = N[index].base1 + local bc = popcnt(band(node.vector, lshift(2ULL, v) - 1)) + index = base + bc - 1 + node = N[index] + offset = offset + Poptrie.k + v = extract(key, offset, Poptrie.k) + if debug then print(index, bin(node.vector), bin(v)) end + end + if debug then print(node.base0, bin(node.leafvec), bin(v)) end + local base = node.base0 + local bc + if Poptrie.leaf_compression then + bc = popcnt(band(node.leafvec, lshift(2ULL, v) - 1)) + else + bc = popcnt(band(bnot(node.vector), lshift(2ULL, v) - 1)) + end + if debug then print(base + bc - 1) end + return L[base + bc - 1] +end + +function Poptrie:lookup64 (key) + return self.asm_lookup64(self.leaves, self.nodes, key, self.directmap) +end + +function Poptrie:fib_info () + for i=0, self.node_base-1 do + print("node:", i) + print(self.nodes[i].base0, bin(self.nodes[i].leafvec)) + print(self.nodes[i].base1, bin(self.nodes[i].vector)) + end + for i=0, self.leaf_base-1 do + if self.leaves[i] > 0 then + print("leaf:", i, self.leaves[i]) + end + end +end + +function selftest () + -- To test direct pointing: Poptrie.direct_pointing = true + local t = new{} + -- Tets building empty RIB + t:build() + -- Test RIB + t:add(0x00, 8, 1) -- 00000000 + t:add(0x0F, 8, 2) -- 00001111 + t:add(0x07, 4, 3) -- 0111 + t:add(0xFF, 8, 4) -- 11111111 + t:add(0xFF, 5, 5) -- 11111 + local v, n = t:rib_lookup(0x0, 1) + assert(not v and n.left and not n.right) + local v, n = t:rib_lookup(0x00, 8) + assert(v == 1 and not n) + local v, n = t:rib_lookup(0x07, 3) + assert(not v and (n.left and n.right)) + local v, n = t:rib_lookup(0x0, 1, n) + assert(v == 3 and not n) + local v, n = t:rib_lookup(0xFF, 5) + assert(v == 5 and (not n.left) and n.right) + local v, n = t:rib_lookup(0x0F, 3, n) + assert(v == 4 and not n) + local v, n = t:rib_lookup(0x3F, 8) + assert(v == 5 and not n) + -- Test FIB + t:build() + if debug then t:fib_info() end + assert(t:lookup(0x00) == 1) -- 00000000 + assert(t:lookup(0x03) == 0) -- 00000011 + assert(t:lookup(0x07) == 3) -- 00000111 + assert(t:lookup(0x0F) == 2) -- 00001111 + assert(t:lookup(0x1F) == 5) -- 00011111 + assert(t:lookup(0x3F) == 5) -- 00111111 + assert(t:lookup(0xFF) == 4) -- 11111111 + assert(t:lookup64(0x00) == 1) + assert(t:lookup64(0x03) == 0) + assert(t:lookup64(0x07) == 3) + assert(t:lookup64(0x0F) == 2) + assert(t:lookup64(0x1F) == 5) + assert(t:lookup64(0x3F) == 5) + assert(t:lookup64(0xFF) == 4) + + -- Random testing + local function reproduce (cases) + debug = true + print("repoducing...") + local t = new{} + for entry, case in ipairs(cases) do + print("key:", entry, bin(case[1])) + print("prefix:", entry, bin(case[1], case[2])) + t:add(case[1], case[2], entry) + end + t:build() + t:fib_info() + for _, case in ipairs(cases) do + print("rib:", t:rib_lookup(case[1])) + print("fib:", t:lookup(case[1])) + print("64:", t:lookup64(case[1])) + end + end + local function r_assert (condition, cases) + if condition then return end + reproduce(cases) + print("selftest failed") + main.exit(1) + end + local lib = require("core.lib") + local seed = lib.getenv("SNABB_RANDOM_SEED") or 0 + for keysize = 1, 64 do + print("keysize:", keysize) + -- ramp up the geometry below to crank up test coverage + for entries = 1, 3 do + for i = 1, 10 do + math.randomseed(seed+i) + cases = {} + local t = new{} + local k = {} + for entry = 1, entries do + local a, l = math.random(2^keysize - 1), math.random(keysize) + cases[entry] = {a, l} + t:add(a, l, entry) + k[entry] = a + end + local v = {} + for entry, a in ipairs(k) do + v[entry] = t:rib_lookup(a, keysize) + r_assert(v[entry] > 0, cases) + end + t:build() + for entry, a in ipairs(k) do + r_assert(t:lookup(a) == v[entry], cases) + r_assert(t:lookup64(a) == v[entry], cases) + end + end + end + end + + -- PMU analysis + local pmu = require("lib.pmu") + local function measure (description, f, iterations) + local set = pmu.new_counter_set() + pmu.switch_to(set) + f(iterations) + pmu.switch_to(nil) + local tab = pmu.to_table(set) + print(("%s: %.2f cycles/lookup %.2f instructions/lookup") + :format(description, + tab.cycles / iterations, + tab.instructions / iterations)) + end + local function time (description, f) + local start = os.clock(); f() + print(("%s: %.4f seconds"):format(description, os.clock() - start)) + end + if pmu.is_available() then + local t = new{direct_pointing=false} + local k = {} + local numentries = 10000 + local keysize = 64 + for entry = 1, numentries do + local a, l = math.random(2^keysize - 1), math.random(keysize) + t:add(a, l, entry) + k[entry] = a + end + local function build () + t:build() + end + local function lookup (iter) + for i=1,iter do t:lookup(k[i%#k+1]) end + end + local function lookup64 (iter) + for i=1,iter do t:lookup64(k[i%#k+1]) end + end + print("PMU analysis (numentries="..numentries..", keysize="..keysize..")") + pmu.setup() + time("build", build) + measure("lookup", lookup, 1e5) + measure("lookup64", lookup64, 1e7) + do local rib = t.rib + t = new{direct_pointing=true} + t.rib = rib + end + time("build(direct_pointing)", build) + measure("lookup(direct_pointing)", lookup, 1e5) + measure("lookup64(direct_pointing)", lookup64, 1e7) + else + print("No PMU available.") + end +end + +-- debugging utils +function bin (number, length) + local digits = {"0", "1"} + local s = "" + local i = 0 + repeat + local remainder = number % 2 + s = digits[tonumber(remainder+1)]..s + number = (number - remainder) / 2 + i = i + 1 + if i % Poptrie.k == 0 then s = " "..s end + until number == 0 or (i == length) + return s +end diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl new file mode 100644 index 0000000000..6664396b5b --- /dev/null +++ b/src/lib/poptrie_lookup.dasl @@ -0,0 +1,146 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(..., package.seeall) + +local debug = false + +local ffi = require("ffi") +local dasm = require("dasm") + +|.arch x64 +|.actionlist actions +|.globalnames globalnames + +-- Table keeping machine code alive to the GC. +local anchor = {} + +-- Assemble a lookup routine +function generate (Poptrie, keysize) + -- Assert assumptions about lib.poptrie + assert(Poptrie.k == 6) + if Poptrie.direct_pointing then + assert(Poptrie.leaf_tag == bit.lshift(1, 31)) + end + assert(ffi.sizeof(Poptrie.leaf_t) == 2) + assert(ffi.sizeof(Poptrie.vector_t) == 8) + assert(ffi.sizeof(Poptrie.base_t) == 4) + assert(ffi.offsetof(Poptrie.node_t, 'leafvec') == 0) + assert(ffi.offsetof(Poptrie.node_t, 'vector') == 8) + assert(ffi.offsetof(Poptrie.node_t, 'base0') == 16) + assert(ffi.offsetof(Poptrie.node_t, 'base1') == 20) + + local name = "poptrie_lookup(k="..Poptrie.k..", keysize="..keysize..")" + + local Dst = dasm.new(actions) + lookup(Dst, Poptrie, keysize) + local mcode, size = Dst:build() + table.insert(anchor, mcode) + + if debug then + print("mcode dump: "..name) + dasm.dump(mcode, size) + end + + local prototype + if keysize <= 64 then + prototype = ffi.typeof( + "$ (*) ($ *, $ *, uint64_t, $ *)", + Poptrie.leaf_t, Poptrie.leaf_t, Poptrie.node_t, Poptrie.base_t + ) + else error("NYI") end + + return ffi.cast(prototype, mcode) +end + +|.define leaves, rdi -- pointer to leaves array +|.define nodes, rsi -- pointer to nodes array +|.define key, rdx -- key to look up +|.define dmap, rcx -- pointer to directmap +|.define index, r8d -- index into node array +|.define node, r8 -- pointer into node array +|.define offset, r9 -- offset into key +|.define v, r10 -- k or s bits extracted from key +|.define vec, r11 -- 64-bit vector or leafvec + +-- lookup(leaf_t *leaves, node_t *nodes, key) -> leaf_t +function lookup (Dst, Poptrie, keysize) + if Poptrie.direct_pointing then + -- v = extract(key, 0, Poptrie.s) + local direct_mask = bit.lshift(1ULL, Poptrie.s) - 1 + -- v = band(key, direct_mask) + | mov v, key + | and v, direct_mask + -- index = dmap[v] + | mov index, dword [dmap+v*4] + -- eax = band(index, leaf_tag - 1) (tag inverted) + | mov eax, index + -- is leaf_tag set? (unsets bit) + | btr eax, 31 + | jnc >1 -- leaf_tag not set, index is a node + | ret + -- node, offset = nodes[index], s + |1: + | imul index, 24 -- multiply by node size + | lea node, [nodes+index] + -- offset = s + | mov offset, Poptrie.s + else + -- index, node, offset = 0, nodes[index], 0 + | xor index, index + | lea node, [nodes+0] -- nodes[0] + | xor offset, offset + end + -- while band(vec, lshift(1ULL, v)) ~= 0 + |2: + -- v = extract(key, offset, k=6) + if keysize <= 64 then + -- v = rshift(key, offset) + | mov v, key + | mov rcx, offset + | shr v, cl + -- v = band(v, lshift(1, k=6) - 1) + | and v, 0x3F + else error("NYI") end + -- vec = nodes[index].vector + | mov vec, qword [node+8] + -- is bit v set in vec? + | bt vec, v + | jnc >3 -- reached leaf, exit loop + -- rax = lshift(2ULL, v) - 1 + | mov rax, 2 + | mov rcx, v + | shl rax, cl + | sub rax, 1 + -- rax = popcnt(band(vec, rax)) + | and rax, vec + | popcnt rax, rax + -- index = base + bc - 1 + | mov index, dword [node+20] -- nodes[index].base1 + | sub index, 1 + | add index, eax + -- node = nodes[index] + | imul index, 24 -- multiply by node size + | lea node, [nodes+index] + -- offset = offset + k + | add offset, 6 + | jmp <2 -- loop + -- end while + |3: + -- rax = lshift(2ULL, v) - 1 + | mov rax, 2 + | mov rcx, v + | shl rax, cl + | sub rax, 1 + if Poptrie.leaf_compression then + -- vec = nodes[index].leafvec + | mov vec, qword [node+0] + else error("NYI") end + -- rax = popcnt(band(vec, rax)) - 1 + | and rax, vec + | popcnt rax, rax + -- return leaves[base + bc - 1] + | mov index, dword [node+16] -- nodes[index].base0 + | add index, eax + | movzx eax, word [leaves+index*2-2] -- leaves[index] + | ret +end diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 9e008af461..a9d7f3e786 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -213,6 +213,12 @@ Returns the binary representation of IPv4 address denoted by *string*. Returns the string representation of *ip* address. +— Function **ipv4:pton** *string* + +Returns the binary representation of the IPv4 address prefix and prefix length +encoded denoted by *string* of the form `/`. +See [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). + ### IPv6 (lib.protocol.ipv6) diff --git a/src/lib/protocol/ipv4.lua b/src/lib/protocol/ipv4.lua index 0f050ff540..9bb19515ee 100644 --- a/src/lib/protocol/ipv4.lua +++ b/src/lib/protocol/ipv4.lua @@ -91,6 +91,13 @@ function ipv4:ntop (n) return ffi.string(c_str) end +function ipv4:pton_cidr (p) + local prefix, length = p:match("([^/]*)/([0-9]*)") + return + ipv4:pton(prefix), + assert(tonumber(length), "Invalid length "..length) +end + function ipv4:set(addr) return ipv4:pton(addr) end diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index 75b8ff5d3c..003858df45 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -6,7 +6,7 @@ local counter = require("core.counter") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") local esp = require("lib.protocol.esp") -local lpm = require("lib.lpm.lpm4_248").LPM4_248 +local poptrie = require("lib.poptrie") local ffi = require("ffi") @@ -26,29 +26,21 @@ PrivateRouter = { } function PrivateRouter:new (conf) - local keybits = 15 -- see lib/lpm/README.md local o = { ports = {}, routes = {}, mtu = conf.mtu, ip4 = ipv4:new({}), - routing_table4 = lpm:new({keybits=keybits}) + routing_table4 = poptrie.new{direct_pointing=true, s=24} } for id, route in pairs(conf.routes) do local index = #o.ports+1 - assert(index < 2^keybits, "index overflow") - o.routing_table4:add_string( - assert(route.net_cidr4, "Missing net_cidr4"), - index - ) + assert(ffi.cast("uint16_t", index) == index, "index overflow") + assert(route.net_cidr4, "Missing net_cidr4") + local prefix, length = ipv4:pton_cidr(route.net_cidr4) + o.routing_table4:add(ffi.cast("uint32_t *", prefix)[0], length, index) o.ports[index] = id end - -- NB: need to add default LPM entry until #1238 is fixed, see - -- https://github.com/snabbco/snabb/issues/1238#issuecomment-345362030 - -- Zero maps to nil in o.routes (which is indexed starting at one), hence - -- packets that match the default entry will be dropped (and route_errors - -- incremented.) - o.routing_table4:add_string("0.0.0.0/0", 0) o.routing_table4:build() return setmetatable(o, {__index = PrivateRouter}) end @@ -60,7 +52,8 @@ function PrivateRouter:link () end function PrivateRouter:find_route4 (dst) - return self.routes[self.routing_table4:search_bytes(dst)] + local address = ffi.cast("uint32_t *", dst)[0] + return self.routes[self.routing_table4:lookup64(address)] end function PrivateRouter:route (p)