Merge PR #888 (v2016.05 release) onto master

.gitignore

@@ -16,3 +16,4 @@ obj
 /deps/*.vsn
 /src/programs.inc
 .images
+/lib/luajit/usr

README.md

@@ -10,10 +10,10 @@ ways to do things.
 The Snabb community are active in
 [applying modern programming techniques](http://blog.ipspace.net/2014/09/snabb-switch-deep-dive-on-software-gone.html),
 [do-it-yourself operator networking](http://blog.ipspace.net/2014/12/l2vpn-over-ipv6-with-snabb-switch-on.html),
-[high-level device drivers](https://github.com/SnabbCo/snabbswitch/blob/master/src/apps/intel/intel10g.lua),
+[high-level device drivers](https://github.com/snabbco/snabb/blob/master/src/apps/intel/intel10g.lua),
 [fast userspace virtio networking](http://www.virtualopensystems.com/en/solutions/guides/snabbswitch-qemu/),
 [universal SIMD protocol offloads](https://groups.google.com/d/msg/snabb-devel/aez4pEnd4ow/WrXi5N7nxfkJ), and
-[applying compiler technology to networking](https://fosdem.org/2015/schedule/event/packet_filtering_pflua/).
+[applying compiler technology to networking](https://archive.fosdem.org/2015/schedule/event/packet_filtering_pflua/).
 
 You are welcome to join our community. If you have an application that
 you want to build, or you want to use one that we are already
@@ -32,8 +32,9 @@ Snabb is written using these main techniques:
 
 Snabb compiles into a stand-alone executable called
 `snabb`. This single binary includes multiple applications and runs on
-any modern Linux distribution. (You could think of it as a
-[busybox](http://en.wikipedia.org/wiki/BusyBox#Single_binary) for
+any modern [Linux/x86-64](src/doc/porting.md) distribution. (You could
+think of it as a
+[busybox](https://en.wikipedia.org/wiki/BusyBox#Single_binary) for
 networking.)
 
 ## How is it being used?
@@ -67,14 +68,14 @@ documentation](src/program/lwaftr/doc/) for more details.
 ### VPWS
 
 VPWS (Virtual Private Wire Service) is a Layer-2 VPN application being
-developed by Alexander Gall at [SWITCH](http://switch.ch). His Github
+developed by Alexander Gall at [SWITCH](http://www.switch.ch/). His Github
 [`vpn` branch](https://github.com/alexandergall/snabbswitch/tree/vpn)
 is the master line of development.
 
 ### packetblaster
 
 [packetblaster](src/program/packetblaster/) generates load by
-replaying a [pcap format](http://en.wikipedia.org/wiki/Pcap) trace
+replaying a [pcap format](https://en.wikipedia.org/wiki/Pcap) trace
 file or synthesizing customizable packets onto any number of Intel 82599 10-Gigabit network
 interfaces. This is very efficient: only a small % of one core per CPU
 is required even for hundreds of Gbps of traffic. Because so little

default.nix

@@ -0,0 +1,39 @@
+# Run like this:
+#   nix-build /path/to/this/directory
+# ... and the files are produced in ./result/
+
+{ pkgs ? (import <nixpkgs> {})
+}:
+
+with pkgs;
+
+stdenv.mkDerivation rec {
+  # TODO: get the version from somewhere?
+  name = "snabb";
+
+  src = ./.;
+
+  buildInputs = [ makeWrapper ];
+
+  patchPhase = ''
+    patchShebangs .
+
+    # some hardcodeism
+    for f in $(find src/program/snabbnfv/ -type f); do
+      substituteInPlace $f --replace "/bin/bash" "${bash}/bin/bash"
+    done
+
+    # We need a way to pass $PATH to the scripts
+    sed -i '2iexport PATH=${git}/bin:${mariadb}/bin:${which}/bin:${procps}/bin:${coreutils}/bin' src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh.inc
+    sed -i '2iexport PATH=${git}/bin:${coreutils}/bin:${diffutils}/bin:${nettools}/bin' src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh.inc
+  '';
+
+  preBuild = ''
+    make clean
+  '';
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp src/snabb $out/bin
+  '';
+}

release.nix

@@ -0,0 +1,67 @@
+# Run like this:
+#   nix-build /path/to/this/directory/release.nix -A manual
+# ... and the files are produced in ./result/
+
+{ pkgs ? (import <nixpkgs> {})
+}:
+
+with pkgs;
+
+let
+  # see https://github.com/snabbco/snabb/blob/master/src/doc/testing.md
+  test_env = fetchurl {
+    url = "http://lab1.snabb.co:2008/~max/assets/vm-ubuntu-trusty-14.04-dpdk-snabb.tar.gz";
+    sha256 = "0323591i925jhd6wv8h268wc3ildjpa6j57n4p9yg9d6ikwkw06j";
+  };
+  optionalGetEnv = first: default: let
+      maybeEnv = builtins.getEnv first;
+    in if (maybeEnv != "") then maybeEnv else default;
+in rec {
+  manual = import ./src/doc {};
+  snabb = import ./default.nix {};
+  tests = stdenv.mkDerivation rec {
+    name = "snabb-tests";
+
+    src = snabb.src;
+
+    # allow sudo
+    __noChroot = true;
+    requiredSystemFeatures = [ "performance" ];
+
+    buildInputs = [ git telnet tmux numactl bc iproute which qemu ];
+
+    buildPhase = ''
+      export PATH=$PATH:/var/setuid-wrappers/
+      export HOME=$TMPDIR
+
+      # make sure we reuse the snabb built in another derivation
+      ln -s ${snabb}/bin/snabb src/snabb
+      sed -i 's/testlog snabb/testlog/' src/Makefile
+
+      # setup the environment
+      mkdir ~/.test_env
+      tar xvzf ${test_env} -C ~/.test_env/
+    '';
+
+    doCheck = true;
+    checkPhase = ''
+      export SNABB_PCI0=${ optionalGetEnv "SNABB_PCI0" "0000:01:00.0"}
+      export SNABB_PCI_INTEL0=${ optionalGetEnv "SNABB_PCI_INTEL0" "0000:01:00.0"}
+      export SNABB_PCI_INTEL1=${ optionalGetEnv "SNABB_PCI_INTEL1" "0000:01:00.1"}
+      export FAIL_ON_FIRST=true
+
+      # run tests
+      sudo -E make test -C src/
+    '';
+
+    installPhase = ''
+      mkdir -p $out/nix-support
+
+      # keep the logs
+      cp src/testlog/* $out/
+      for f in $(ls $out/* | sort); do
+        echo "file log $f"  >> $out/nix-support/hydra-build-products
+      done
+    '';
+   };
+}

src/Makefile

@@ -5,6 +5,7 @@ E= @echo
 #E= @:
 
 export TEST_SKIPPED=43
+CC ?= gcc
 
 SRCDIR = $(shell find . -type d -not -regex './obj.*' -printf '%P ')
 OBJDIR = $(patsubst %,obj/%,$(SRCDIR))
@@ -50,7 +51,7 @@ PATH := ../lib/luajit/usr/local/bin:$(PATH)
 
 snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(INCOBJ) $(LUAJIT_A)
 	$(E) "LINK      $@"
-	$(Q) gcc $(DEBUG) -Wl,--no-as-needed -Wl,-E -Werror -Wall -o $@ $^ \
+	$(Q) $(CC) $(DEBUG) -Wl,--no-as-needed -Wl,-E -Werror -Wall -o $@ $^ \
 	    ../lib/luajit/src/libluajit.a \
 	    -lrt -lc -ldl -lm -lpthread
 	@echo -n "BINARY    "
@@ -124,15 +125,15 @@ $(PFLUAOBJ): obj/%_lua.o: ../lib/pflua/src/%.lua Makefile
 
 $(COBJ): obj/%_c.o: %.c $(CHDR) Makefile | $(OBJDIR)
 	$(E) "C         $@"
-	$(Q) gcc $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
+	$(Q) $(CC) $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
 
 obj/arch/avx2_c.o: arch/avx2.c Makefile
 	$(E) "C(AVX2)   $@"
-	$(Q) gcc -O2 -mavx2 $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
+	$(Q) $(CC) -O2 -mavx2 $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
 
 obj/arch/sse2_c.o: arch/sse2.c Makefile
 	$(E) "C(SSE2)   $@"
-	$(Q) gcc -O2 -msse2 $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
+	$(Q) $(CC) -O2 -msse2 $(DEBUG) -Wl,-E -I ../lib/luajit/src -I . -include $(CURDIR)/../gcc-preinclude.h -c -Wall -Werror -o $@ $<
 
 $(HOBJ): obj/%_h.o: %.h Makefile | $(OBJDIR)
 	$(E) "H         $@"
@@ -178,7 +179,7 @@ obj/jit_tprof.o: extra/tprof.lua | $(OBJDIR)
 
 obj/jit_vmprof.o: extra/vmprof.c | $(OBJDIR)
 	$(E) "C         $@"
-	$(Q) gcc $(DEBUG) -Wl,-E -O2 -I ../lib/luajit/src -c -Wall -Werror -o $@ $<
+	$(Q) $(CC) $(DEBUG) -Wl,-E -O2 -I ../lib/luajit/src -c -Wall -Werror -o $@ $<
 
 book: obj/doc/snabb.pdf obj/doc/snabb.html obj/doc/snabb.epub
 
@@ -187,7 +188,7 @@ obj/doc/snabb.markdown: markdown Makefile doc/genbook.sh
 
 obj/doc/snabb.pdf: obj/doc/snabb.markdown
 	$(E) "PANDOC    $@"
-	$(Q) (cd obj/doc; pandoc --template=template.latex --latex-engine=lualatex -V fontsize=10pt -V monofont=droidsansmono -V monoscale=.70 -V verbatimspacing=.85 -V mainfont=droidserif -V sansfont=droidsans -V documentclass:book -V geometry:top=1.0in -V geometry:bottom=0.75in -S --toc --chapters  -o ../../$@ ../../$<)
+	$(Q) (cd obj/doc; pandoc --template=../../doc/template.latex --latex-engine=lualatex -V fontsize=10pt -V monofont=droidsansmono -V monoscale=.70 -V verbatimspacing=.85 -V mainfont=droidserif -V sansfont=droidsans -V documentclass:book -V geometry:top=1.0in -V geometry:bottom=0.75in -S --toc --chapters  -o ../../$@ ../../$<)
 
 obj/doc/snabb.html: obj/doc/snabb.markdown
 	$(E) "PANDOC    $@"

src/apps/basic/README.md

@@ -6,8 +6,10 @@ functionality for use in you app networks.
 ## Source
 
 The `Source` app is a synthetic packet generator. On each breath it fills
-each attached output link with new packets. The packet data is
-uninitialized garbage and each packet is 60 bytes long.
+each attached output link with new packets. It accepts a number as its
+configuration argument which is the byte size of the generated packets. By
+default, each packet is 60 bytes long. The packet data is initialized with
+zero bytes.
 
     DIAGRAM: Source
     +--------+

src/apps/basic/basic_apps.lua

@@ -3,7 +3,6 @@
 module(...,package.seeall)
 
 local app = require("core.app")
-local freelist = require("core.freelist")
 local packet = require("core.packet")
 local link = require("core.link")
 local transmit, receive = link.transmit, link.receive

src/apps/intel/intel10g.lua

@@ -23,8 +23,7 @@ local timer = require("core.timer")
 local bits, bitset = lib.bits, lib.bitset
 local band, bor, lshift = bit.band, bit.bor, bit.lshift
 
-num_descriptors = 512
---num_descriptors = 32
+num_descriptors = 1024
 
 -- Defaults for configurable items
 local default = {

src/apps/intel/intel_app.lua

@@ -9,7 +9,6 @@ local lib      = require("core.lib")
 local pci      = require("lib.hardware.pci")
 local register = require("lib.hardware.register")
 local intel10g = require("apps.intel.intel10g")
-local freelist = require("core.freelist")
 local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty
 Intel82599 = {}
 Intel82599.__index = Intel82599

src/apps/keyed_ipv6_tunnel/tunnel.lua

@@ -243,9 +243,8 @@ prepare_header_template()
 function selftest ()
    print("Keyed IPv6 tunnel selftest")
    local ok = true
-
-   local input_file = "apps/keyed_ipv6_tunnel/selftest.cap.input"
-   local output_file = "apps/keyed_ipv6_tunnel/selftest.cap.output"
+   local Synth = require("apps.test.synth").Synth
+   local Match = require("apps.test.match").Match
    local tunnel_config = {
       local_address = "00::2:1",
       remote_address = "00::2:1",
@@ -255,19 +254,19 @@ function selftest ()
    } -- should be symmetric for local "loop-back" test
 
    local c = config.new()
-   config.app(c, "source", pcap.PcapReader, input_file)
    config.app(c, "tunnel", SimpleKeyedTunnel, tunnel_config)
-   config.app(c, "sink", pcap.PcapWriter, output_file)
+   config.app(c, "match", Match)
+   config.app(c, "comparator", Synth)
+   config.app(c, "source", Synth)
    config.link(c, "source.output -> tunnel.decapsulated")
+   config.link(c, "comparator.output -> match.comparator")
    config.link(c, "tunnel.encapsulated -> tunnel.encapsulated")
-   config.link(c, "tunnel.decapsulated -> sink.input")
+   config.link(c, "tunnel.decapsulated -> match.rx")
    app.configure(c)
 
-   app.main({duration = 0.25}) -- should be long enough...
+   app.main({duration = 0.0001, report = {showapps=true,showlinks=true}})
    -- Check results
-   if io.open(input_file):read('*a') ~=
-      io.open(output_file):read('*a')
-   then
+   if #engine.app_table.match:errors() ~= 0 then
       ok = false
    end