From b34c3eee2bf5cb05a6f47c0ed9e318632c8dc534 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 22 Feb 2016 13:06:18 +0000 Subject: [PATCH 001/103] engine: Set shm path to "app/$name" Set the shared memory path (shm.path) to a private namespace for each app with prefix "app/$name". This means that apps can create shm objects such as counters and by default these will appear in a local namespace for that app. --- src/core/app.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index 35a5305631..099fc3125a 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -5,6 +5,7 @@ local lib = require("core.lib") local link = require("core.link") local config = require("core.config") local timer = require("core.timer") +local shm = require("core.shm") local counter = require("core.counter") local zone = require("jit.zone") local ffi = require("ffi") @@ -65,6 +66,8 @@ end -- Run app:methodname() in protected mode (pcall). If it throws an -- error app will be marked as dead and restarted eventually. local function with_restart (app, method) + local oldshm = shm.path + shm.path = app.shmpath if use_restart then -- Run fn in protected mode using pcall. local status, err = pcall(method, app) @@ -75,6 +78,7 @@ local function with_restart (app, method) else method(app) end + shm.path = oldshm end -- Restart dead apps. @@ -162,7 +166,11 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg + local shmpath, shmorig = "app/"..name, shm.path + shm.path = shmpath local app = class:new(arg) + shm.path = shmorig + local shmpath = "app/"..name if type(app) ~= 'table' then error(("bad return value from app '%s' start() method: %s"):format( name, tostring(app))) @@ -171,6 +179,7 @@ function apply_config_actions (actions, conf) app.appname = name app.output = {} app.input = {} + app.shmpath = shmpath new_app_table[name] = app table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array From 80614e1bd4b34d6c4ebf35289417cb00508fb745 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 18 Mar 2016 06:03:29 +0000 Subject: [PATCH 002/103] doc/git-workflow.md: Rewritten based on new experience I rewrote the git-workflow explanation with the goal of briefly but clearly explaining both how to submit a change to Snabb Switch and also how to be a subsystem maintainer. This is intended to be a chapter in the manual. --- src/doc/git-workflow.md | 110 +++++++++++++++------------------------- 1 file changed, 42 insertions(+), 68 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 353384cf3e..7560bd927a 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -1,86 +1,60 @@ -# Snabb Switch Git Workflow +## Git workflow -This document explains the Git workflows that we use to develop and -release Snabb Switch. +Snabb Switch is developed on Github using the distributed workflow +that was pioneered by the Linux kernel. -## Overview +The latest Snabb Switch release can always be found on the `master` +branch of the Github `SnabbCo/snabbswitch` repository. -Snabb Switch development follows a few well-known patterns: +### Being a contributor -- We follow the distributed development model [described by Linus - Torvald](https://www.youtube.com/watch?v=4XpnKHJAok8) and pioneered - by the Linux kernel. -- We use a [merge based workflow](https://www.atlassian.com/git/articles/git-team-workflows-merge-or-rebase/). -- We use the [fork and pull](https://help.github.com/articles/using-pull-requests/#fork--pull) - model of collaboration. -- We aim to support ad-hoc collaboration inspired by the - [DMZ Flow](https://gist.github.com/djspiewak/9f2f91085607a4859a66). +So you have decided to contribute an improvement to Snabb Switch. Great! This is what to do: -## HOWTO +1. Create your own "fork" of Snabb Switch on Github. Do this by clicking the "Fork" button on the Github UI. (You only have to do this the first time you are making a change.) +2. Create a topic branch based on `master`. Example: `git branch -b my-feature master`. +3. Develop your feature and push your branch to your Github fork. Example: `emacs`, `git commit`, `git push`. +4. Open a Github Pull Request from your branch to the SnabbCo/snabbswitch master branch. Clicking the Pull Request button on Github should do the right thing. -### Download and update the latest release +Once your Pull Request is open you can wait a short while, usually a day or two, for a maintainer to engage with you and take responsibility for merging your change "upstream". -1. Clone the [SnabbCo/snabbswitch](https://github.com/SnabbCo/snabbswitch) repository. -2. Check out and build the `master` branch. -3. Pull when you want to update to the latest stable release. +Tips for making your contribution smoothly: -### Develop and contribute an improvement +- Explain why your change is a good idea using the text area of the Pull Request. +- Don't rebase your branch after the pull request is open; make corrections by pushing new commits. +- If your change is a work in progress then prefix the Pull Request name with `[wip]`. +- If your change is a rough draft for early feedback then prefix the Pull Request name with `[sketch]`. -1. [Create your own fork](https://help.github.com/articles/fork-a-repo/) of Snabb Switch on Github. -2. Develop and debug your contribution on a new [topic branch](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows#Topic-Branches) based on the latest `master`. -3. Make a final cleanup of your code before review. (Last chance to rebase.) -4. Submit a Github [Pull Request](https://help.github.com/articles/using-pull-requests/#initiating-the-pull-request) - to the `master` branch. -5. Respond to feedback and correct problems by pushing additional commits. +### Being a maintainer -There are two milestones in the process of accepting your change: +So you have decided to help with the upstream maintenance of Snabb +Switch. Fantastic! You can do this by creating and maintaining a +"subsystem branch" where you take responsibility for reviewing and +merging some of the Pull Requests submitted to Snabb Switch. -1. Your change is merged onto a branch that feeds `master`, for - example `next`, `fixes`, `documentation-fixes`, or `nfv`. From this - point the owner of that branch will push your work upstream - together with other related changes. They might ask you for help - but otherwise your work is done. -2. Your change is merged onto `master`. This could happen in a series - of merge steps, for example `nfv->next->master`. Once this happens - your code has been officially released as part of Snabb Switch. +#### Registering a subsystem branch -### Develop and maintain a new program +The first step is to create and register your subsystem branch: -Snabb Switch includes programs like `snabbnfv`, `packetblaster`, and -`snsh`. Here is how you can create a new program and take charge of -its development. +1. Pick your technical area of interest. What kind of changes will you be responsible for reviewing and merging? Try to pick an area that is easy to identify, for example "the packetblaster program", "the Intel I350 device driver", or "the Git Workflow chapter of the manual". +2. Create a branch with a suitable name on your Github fork, for example `packetblaster`, `i350`, or `git-workflow`. This is where you will merge relevant changes. +3. Describe this branch in the file `src/doc/branches.md` and open a Github Pull Request. This will kick off two discussions: how to clearly identify the changes that you are responsible for, and to which "next-hop" upstream branch you should send the changes that you have accepted by merging. -1. [Fork](https://help.github.com/articles/fork-a-repo/) your own - repository on Github. -2. Create a [long-lived branch](branches.md) where new development of your program will be done. -3. Create a directory `src/program/myapplication/` and develop your program. -4. `git merge master` regularly to stay synchronized with the main line of development. -5. Optional: Send releases of your application to `master` with Pull Requests. +Once those details are worked out and your branch is registered then +you are a Snabb Switch maintainer. Congratulations! -The code in your `src/program/myapplication/` directory is developed -according to your own rules and tastes. If there are parts of this -code that you especially want to have reviewed (or do not want to have -reviewed) then please explain this in your Pull Request. The only -necessary review is to make sure that programs do not negatively -impact each other or change shared code without enough review. +#### Being "the upstream" for Pull Requests -Pull Requests that make changes to your application will be referred -to you for merge onto your branch. +Now as a maintainer your job is to watch for Pull Requests opened to +the Github repository and act as the responsible person (the +"upstream") when your branch most specifically matches a change. -Use the *Develop and contribute an improvement* workflow to make -changes to the core Snabb Switch code. Please do not bundle -substantial changes to the core software with updates to your program. - -If you do not want to include your program in the main Snabb Switch -release then this is no problem. You can simply pull from `master` to -receive updates and skip the step of pushing back. - -### To help maintain Snabb Switch - -Here are the best ways to help maintain Snabb Switch: - -1. Review Pull Requests to help people quickly improve them. -2. Test the `next` branch and help fix problems before releases. -3. Contribute new `selftest` cases to make our CI more effective. -4. Maintain a [branch](branches.md) where you accept Pull Requests and push them upstream. +Here is how to be the upstream for a change: +1. Set yourself as the *Assignee* of the Pull Request. This clearly signals to everybody that you are the one responsible for reviewing and merging the change. +2. Review the submitted changes: + 1. Does it all look good? If so then merge the changes onto your branch and add the label `merged` to the Pull Request. + 2. Do you see some serious problems? Tell the contributor exactly what they need to change in order for you to merge the changes. + 3. Do you see minor ways that the change could be improved? Suggest those and ask the contributor whether they want to do that before you merge the change. + 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. + 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. +3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. From 12d00096604974380d48ebf921344c392014f96f Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 18 Mar 2016 10:32:24 +0000 Subject: [PATCH 003/103] doc/git-workflow.md: Added section on upstreaming subsystems --- src/doc/git-workflow.md | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 7560bd927a..5a3835a30a 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -58,3 +58,56 @@ Here is how to be the upstream for a change: 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. 3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. + +#### Sending collected changes upstream to your next-hop + +So you merge some good changes onto your subsystem branch. What next? + +The next step is to open a Pull Request from your specific subsystem +branch to the more general "next hop" upstream branch. This is your +way to say "hey, I have collected some good changes here, please merge +them!" + +The upstreaming process is the same one described above, but now you +are the one submitting the changes and expecting clear feedback on +what actions you need to take for them to be accepted. + +#### Putting it all together + +Here is a complete example of how this can all fit together: + +You decide that you want to be the maintainer of the Intel I350 +ethernet driver. You create a branch called `i350` and open a Pull +Request to describe this branch in `src/doc/branches.md`. The other +maintainers are happy that you want to join in and gladly agree to +refer Pull Requests concerned the Intel I350 driver to you. You agree +that once you have good changes on your `i350` branch you will open a +Pull Request to the more general `drivers` branch as your "next hop" +upstream. + +People in the community start contributing improvements to the I350 +driver. You set yourself as the *Assignee* to these Pull Requests and +engage with the contributors to get the changes in good shape. You +merge the good changes onto your `i350` branch and periodically open a +Pull Request to the `drivers` branch to send this code upstream. The +`drivers` branch will in turn be merged to its next hop upstream and +step-by-step the changes will make their way towards release on the +`master` branch. + +The time scales involved are not written in stone but it is important +to find a rhythm that is comfortable for everybody involved. You might +aim to set yourself as the *Assignee* for relevant Pull Requests +within one day, to provide a review within a few days, and to send +Pull Requests to your next-hop upstream once or twice per week when +you have changes. + +#### Don't be shy! + +Becoming a Snabb Switch maintainer is a great service to the community +and you can learn all the skills that you need "on the job." If you are +tempted to give it a try then please do! + +The more subsystem maintainers we have the more capacity we have to +incorporate improvements into Snabb Switch. The Linux kernel has +more than one thousand registered subsystems. The sky is the limit! + From af8ae46df1f42e95ac7311cb1bd5e8e37acef952 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 21 Mar 2016 06:00:18 +0000 Subject: [PATCH 004/103] doc/git-workflow.md: Partly rewritten draft Contains "XXX rewrite" where old sections have been removed but replacements not written yet. --- src/doc/git-workflow.md | 190 +++++++++++++++++++--------------------- 1 file changed, 92 insertions(+), 98 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 5a3835a30a..16dae03e6e 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -1,113 +1,107 @@ ## Git workflow -Snabb Switch is developed on Github using the distributed workflow -that was pioneered by the Linux kernel. - -The latest Snabb Switch release can always be found on the `master` -branch of the Github `SnabbCo/snabbswitch` repository. - -### Being a contributor - -So you have decided to contribute an improvement to Snabb Switch. Great! This is what to do: - -1. Create your own "fork" of Snabb Switch on Github. Do this by clicking the "Fork" button on the Github UI. (You only have to do this the first time you are making a change.) -2. Create a topic branch based on `master`. Example: `git branch -b my-feature master`. -3. Develop your feature and push your branch to your Github fork. Example: `emacs`, `git commit`, `git push`. -4. Open a Github Pull Request from your branch to the SnabbCo/snabbswitch master branch. Clicking the Pull Request button on Github should do the right thing. - -Once your Pull Request is open you can wait a short while, usually a day or two, for a maintainer to engage with you and take responsibility for merging your change "upstream". - -Tips for making your contribution smoothly: - -- Explain why your change is a good idea using the text area of the Pull Request. -- Don't rebase your branch after the pull request is open; make corrections by pushing new commits. -- If your change is a work in progress then prefix the Pull Request name with `[wip]`. -- If your change is a rough draft for early feedback then prefix the Pull Request name with `[sketch]`. - -### Being a maintainer - -So you have decided to help with the upstream maintenance of Snabb -Switch. Fantastic! You can do this by creating and maintaining a -"subsystem branch" where you take responsibility for reviewing and -merging some of the Pull Requests submitted to Snabb Switch. +How do you engage with the Snabb Switch developer community? The answer depends on what you want to do: + +- Use the software, ask questions, report bugs. +- Contribute fixes and improvements. +- Maintain Snabb Switch by reviewing and merging pull requests. +- Create a new application to develop together with the community. + +### Using the software + +The recommended way to download Snabb Switch is with `git` directly +from from the `master` branch of the `snabbco` repository. This branch +always contains the latest release. + +``` +$ git checkout https://github.com/snabbco/snabbswitch +$ cd snabbswitch +$ make -j +``` + +The `master` branch is updated with a new release each month. You can +upgrade by pulling in the latest changes to your local copy: + +``` +$ git pull +$ make -j +``` + +This is safe to do at any time because the `master` branch is only +used for publishing the latest release to users. (The active software +development is done on different branches and only merged onto +`master` once it is ready for release.) + +You can also switch back and forth between different versions based on +their release tags: + +``` +$ git checkout v2016.01 # switch to a specific version +$ make -j +... +$ git checkout master # switch back to latest +$ make -j +``` + +### Contributing fixes and improvements + +The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. + +1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. +2. Push your proposed change to a branch on your repository. +3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. + +Here are some tips for making your contribution smoothly: + +- Use a dedicated "topic branch" for each feature or fix. +- Use the Pull Request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. + +### Becoming a maintainer + +Snabb Switch maintainers are the people who review and merge the Pull +Requests on Github. Each maintainer takes care of one or more specific +aspects of Snabb Switch. These aspects are called *subsystems*. + +Each subsystem has a dedicated branch and these branches are organized +as a network: + + DIAGRAM: Branches + +--lisper + +--max next<----+--documentation<--pdf manual + | + fixes | + | | + master<--+--next<--+--kbara next<--+--nix + | +--mellanox + | + | + +--wingo next<--+--lwaftr + +--multiproc + +Pull Requests are first merged onto the subsystem branch that most +specifically matches their subject matter. For example, a change to +the PDF formatting of the manual would first be reviewed and accepted +by the maintainer of the `pdf-manual` branch. Later, that whole +subsystem branch is merged onto its "next-hop upstream" branch. For +example, the maintainer of the `documentation` branch would merge the +entire `pdf-manual` branch at regular intervals. #### Registering a subsystem branch -The first step is to create and register your subsystem branch: - -1. Pick your technical area of interest. What kind of changes will you be responsible for reviewing and merging? Try to pick an area that is easy to identify, for example "the packetblaster program", "the Intel I350 device driver", or "the Git Workflow chapter of the manual". -2. Create a branch with a suitable name on your Github fork, for example `packetblaster`, `i350`, or `git-workflow`. This is where you will merge relevant changes. -3. Describe this branch in the file `src/doc/branches.md` and open a Github Pull Request. This will kick off two discussions: how to clearly identify the changes that you are responsible for, and to which "next-hop" upstream branch you should send the changes that you have accepted by merging. - -Once those details are worked out and your branch is registered then -you are a Snabb Switch maintainer. Congratulations! +XXX rewrite. #### Being "the upstream" for Pull Requests -Now as a maintainer your job is to watch for Pull Requests opened to -the Github repository and act as the responsible person (the -"upstream") when your branch most specifically matches a change. - -Here is how to be the upstream for a change: - -1. Set yourself as the *Assignee* of the Pull Request. This clearly signals to everybody that you are the one responsible for reviewing and merging the change. -2. Review the submitted changes: - 1. Does it all look good? If so then merge the changes onto your branch and add the label `merged` to the Pull Request. - 2. Do you see some serious problems? Tell the contributor exactly what they need to change in order for you to merge the changes. - 3. Do you see minor ways that the change could be improved? Suggest those and ask the contributor whether they want to do that before you merge the change. - 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. - 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. -3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. +XXX rewrite. #### Sending collected changes upstream to your next-hop -So you merge some good changes onto your subsystem branch. What next? - -The next step is to open a Pull Request from your specific subsystem -branch to the more general "next hop" upstream branch. This is your -way to say "hey, I have collected some good changes here, please merge -them!" - -The upstreaming process is the same one described above, but now you -are the one submitting the changes and expecting clear feedback on -what actions you need to take for them to be accepted. +XXX rewrite. #### Putting it all together -Here is a complete example of how this can all fit together: - -You decide that you want to be the maintainer of the Intel I350 -ethernet driver. You create a branch called `i350` and open a Pull -Request to describe this branch in `src/doc/branches.md`. The other -maintainers are happy that you want to join in and gladly agree to -refer Pull Requests concerned the Intel I350 driver to you. You agree -that once you have good changes on your `i350` branch you will open a -Pull Request to the more general `drivers` branch as your "next hop" -upstream. - -People in the community start contributing improvements to the I350 -driver. You set yourself as the *Assignee* to these Pull Requests and -engage with the contributors to get the changes in good shape. You -merge the good changes onto your `i350` branch and periodically open a -Pull Request to the `drivers` branch to send this code upstream. The -`drivers` branch will in turn be merged to its next hop upstream and -step-by-step the changes will make their way towards release on the -`master` branch. - -The time scales involved are not written in stone but it is important -to find a rhythm that is comfortable for everybody involved. You might -aim to set yourself as the *Assignee* for relevant Pull Requests -within one day, to provide a review within a few days, and to send -Pull Requests to your next-hop upstream once or twice per week when -you have changes. - -#### Don't be shy! - -Becoming a Snabb Switch maintainer is a great service to the community -and you can learn all the skills that you need "on the job." If you are -tempted to give it a try then please do! - -The more subsystem maintainers we have the more capacity we have to -incorporate improvements into Snabb Switch. The Linux kernel has -more than one thousand registered subsystems. The sky is the limit! +XXX rewrite. From eccb61405a1c14c68c871f7a2b60b1ff9e74bdad Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 21 Mar 2016 06:04:03 +0000 Subject: [PATCH 005/103] doc/git-workflow.md: Created .src.md This file now contains a diagram so it needs to have separate .src.md and .md versions. This can be cleaned up when these changes eventually merge with the on-demand markdown (#829). --- src/doc/.images/Branches.png | Bin 0 -> 9218 bytes src/doc/git-workflow.md | 13 +---- src/doc/git-workflow.src.md | 107 +++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 src/doc/.images/Branches.png create mode 100644 src/doc/git-workflow.src.md diff --git a/src/doc/.images/Branches.png b/src/doc/.images/Branches.png new file mode 100644 index 0000000000000000000000000000000000000000..7793578f6932b5113ec39ecebeef57d30daacc0c GIT binary patch literal 9218 zcmaia2UJsA*DWf74NwsgA)ulnA|M??iHM36X-X$VrAjXeNLTQJw4flp1rShrF9}7Z z2}lUNgMdIF^w8_u!F#`Z-+TA}{xKLCC+F<5v-jGo%sChSYAW({CoZ0#qN1Wxcz92P zit6wNxNbjo7<}dtIa{cxI2#r2$v%16KR4v2{KRap!Jg?CeZZ6QnC_xronnLj3>=?% z{-IQhN4Eph=F+bBJ7vOiV?}IQY~ZPu!OR~mG?9_r7ES_>Wbb9%^0(sUMTS50*0!?L z4eh7Nc*zsFxm(q|9!Zvztchc}ap~vwDsA*soRFDm!qli#qL8C$!lC^mR8$02It*ov zvb0oGm{uAHD;3q9>N|u(R8-UuaNC>y&u6bNVDQ2HOT_^-j6c)7!+Lo%OuC3 zl$(D~b@0kxgNYA1(jKwie)`3SL8jN{3}sp15%?pJd{lIGbv->j?d|PbVPONMwjFGg z$5;VkFB1jxLIJ^u*-=fj@=4PRVioCqr+}zwZ zZhRjYaNF5f!a2=3xEmbT%+JqHNJyY%VE-sV+3<=Qq1%Hty< zOoug=gZ=#EpFH`H6JCvQU;4x5U3|PWCO9|O*3K?(*s$Cob#s1iZ?8a08zOG^YVu&! za0(&AMNkdshrOJhD zYH3MoPgP14Hz#14Bl%+&WIVQ3PBQa#c6Pe&uJ#Di0^_}6KLPE@frlAfsoU7tVB*m- zHZwz1ylk6C%gD%_eT8i7{QTHN;o-x#gM55^-E2apKcZe(kByBX8>>9FgK!LN;&~+{ z5n7Z*E_x-Wq|hQGjL)q5pnJ>gQz9cHT^K@N{?PaJ^JB%uNIK`(I8Nc?<^1R++~{LP ztp(?0p4T3`cn)~Nm;Eqz!XIvqo2$zU)%!9q7~*Ven5xWYh18rJd~bV42f03&6+6MG zD)p=)#c{0m_>j1eP)B^j(St3}LFwq|^h<5JbU2;n2S4|vVzK?E*zc3YJ%*Xt*&=+> zCAbG>ouAZQ)m0upehd!Y!Xjff0&@ho%ujY&h`)FY{P5w!D;$u+f}g$1%*bAe&RLGZkxffrFJF$AqLrWELJb~qi(4U9Ci{4v9J%s zt@NIZq^&Hi%o)Pxd2FxAv%B2$LSB-VuFU12-Fj}wMWnI|3^H^*UF|Rtuo{t?n#v%( zqQ>vSk-APD?>RWu?K~Zwq%PPky?{c;kCIzqVmYcD(ym2IDT>q(Z~9j<60;&ABH4zO zdS;@Rh^LWHoK+Y3Ds8$lG%J&OB%FLJ-PUJ9jT+FaA%?PqLr-G`O#;}DsU%3TFfxvl zNU@=z8v6SBnwsJ7-u;}KVrFL6*3+}F;`jU37{*0%?{VlCvmdLCi(`Xps;Ob*8-s#Q zahDVoDaQya1iXL$e$F%IDVvYX-p2UG#!C*4MEmi2CMKrnQhQNY$*R$`>OgcbM6!G7 z1_vvvo(`yu>5Z?6E%5&kehzP1T z?^*j;yH0NP9+ofiX87(yh(&%yg~56SFd-hr8|&7QR63v^S6p`A8Rf*>kkpp!IXGQo zYSx-CS7kWm@XdvUzm#pHec85p*Qu0UVDR&u$e|xCyNt+J+tNg88k#17Z{lAXzuu*1 zdmwQ2>c{+i^-{0B%_-E0s}csTFJBUAINTRs=fLxxzE&}VKYkn*9sSudgk9Rg-XP+u z+#083XYj=7ogwFuX_{`bk-k(!GbSuN+(hd6K0VZHkDS<&tL}{{40~BPmEfk*(-f5i zi!d7YaiI=XZ-ElaMv4cc>4$V}?Yqa$4|TO*jNmZ7?YzG3Y<*mDPkEf(Fjfe8gNW=p zQ)6-zJ8L>Wn@EX;J)^ed`{Ci?jMtS3$#HizpFFvxTW~MX0c~Cp?l3&Pr>$Gmd5TICD1|!K8E_8Vo7$?Ydn30&LYU^k3ar! zaHeAmm7-^t(m@~)aQJSwzHPMpCI6G%vJEXc6MIPF+}e3FUYj8}+%N@fZ(fQgrAM0l zLuhD-l*JY2+{a>-hG^JZ=}>i>d^=ij815wGyrxPxbPT#LF^BWq#h#n6OKEsd$)V5g zGsKV058)gls$VXRt*opR6$$U&hf72YIq5ukV$4=Mv5|}`7UyGNm)Zap6zyO-!YXQ& zS1v$t!D30@#z5DfuW9329`u3z#Wj(Xw&!fUGL>X@7Ujsjrpmgj^{lo;8WR%}tnudZ z!~@2MlXipUu?O3jL=y3?fraNw=X3ac(5T%qa#m5#3tv(+X58wVOJsZ!Nc)-S3w%5T z6ER1#eQF3MWV*jEdo_>g%$cM*5Gz9(_^y?YYJm=ECYCs~AUAiTeW^pY<*b@YbC6k? zY0JgP1`~-R9nT*<3LaMS<>1D~aSUwb_d{xaq3iEsDkGNGwpDFMIOWmAW#?fR`N#`& z8Ah?{jSE%VIW~(eVk5**X2X|Xbu3TmotJHRNaghRs+DlO^94BUDQNfVIKHsmaB9O} zQ+szGx+fmy}j+7|}yPZLS}!Z*5%wg{!;4WIV|vPk>|G03Tw#tAm%phH49gTsXXg zR$&%Y85?mYKY~B6upe87#G5on=vJpNa&XvIZ8UzTMl*BbI*OrC2o`SpqO=a~)RVZ3 zMt6otwl)z_PCb{LMl8v!+7)A(y zVoB=BEiEb8TaK(oIt8oTzkeSNkGDV6(Uu|)$!#mQ8~q0A8fvQ0thHMS0SuwUqRXWc zxAE9pMLu25ne7Zjf6&C)x-?82_n;x)bC3brPtwZP>k{fA7Dn*#;BFvaCBK zo9;G{eW<+Afi>NwBc811&RxCF5;plGTF8CF4hDmb=Uwr`*Vf*HFd3@}qh({oY(Yq4 z>vnqiI7pz<3NUcxRxAe$r8GBU4NH4Ne9$c`_gPM2I84Tehp)~SwqEpO8sFJi9Ie4FRk)B>c3+XFTAi;Dd*fO|=I}<1qxv*V z5J`~Kv**M%f&(vXSrXb3RagX`zv=w3GTjk;&wLLO58U~o*s){B2na*nYR}ye*q%9i zY-+!cj}L%mw^qxI8_&Xr+uIAA=K9~hJtpB7VIiDjHHReRql}>|PHurlf%-^oGiQ6Y zi@Y=GDYEiz@oQc4+YpEuBOW(<3eJu+5PjWz0@>FBE%NfFw}a6@D)#rbw~~%)zGim> z=!3R1A!(tI3O+y6l?fjpQ9vtzz?hLMpp;TTBKSf9t_OFi01%>J zT3K2OHm0E8|2hCbE&xLS00I}kKf#39hPAl?4wqkN`1$!+LckeW|M;u~VEAd|mR~Du z-~CSPt5>gp_vPy3dt>B&Kkx@N+=3wCG#izlmz8x32CFVBi`u#j)^H5!p{c0Y^lE=? zZNPS5ZyOR1{2U(s{{7{;h=>SGI8+ ziV?~QQWCdstJ&N#C{xoG5f!Bea4X^}7uQIIOJQ*_YPvmDgH70+0Kz#lBLg`#F)>kK z@90Q2OZ13{js0yYg1$HU#?708!>jKndU6px6LocUSi2ul0v-Leb#>FB03v|xJvtjQ zhz|P*(1Bx_%#RvdzD4nRy;3tHE z7?BwV4hCec8d}eBzR_M2}x4l3nldmmFZT4X*48kw># zA>h?d7t?Y;(8c3#aC2|U#cpkF@r{(vl2aoMv9`1e^j+2?8JRCS(i-;wO8cskRSDZS z0cEGHcy6F(%Q^Nib^M+(y5~s00pM!Q4L*MCuAeOZ2CA5lgSi7T+S==mgz4I9{?n{5s8Ti1A`v|jF*VWVD86hL^b>6UyV^w7Z8Y%4QpK{ zd-hC2LqkpNU1{n1#?rX+T>pf4P@_7(sHhhG7|55x-3iieTIXZd`(kNr$BW%}mKv6p zmO#oLuMg${F=9KOb@RlD6Dvnejg24Xr>2^;e^k&23JS`~%`N>jAfu;!Ufr}VfN5)I z#FfNL?r)XZR>mfG>LZazL&JpFq2eKWQ;$f)t}Ne~nVEUZ5%xX#+ofeFCV|6;sN?CW zDk0FnI$$uH(S5jBv!(R-_^Ln zO{E-NUW|CG$J;!GGo3tti=YW;PjqxNc8%DU(tWugq&vJ2Q~0cg#$(yGYBSBn`YG#p zszSr|SRlWFleQV;#};2@x-K2SFGKxR(mNqfnXP#tvabi}zKUFl}CMRDR4{y4SnKKc%vNnXZSf`_!)63M#b}%3T6-VH1BvGFA^5x5X zI7=Ml#{_)dr7&h-{WUE+@%7qVqZhTlTMu0K$r*s0aI#!iuG|e|VPSE-^EoT)!&tPC zS|vpyY#|wPzNgx7wu5Er4bABvtX)ft$Ygw}mgO7I50+z>K!pR&)j&VF%(q^l+cIZq;5wLn^#k(RwqB18PPQTW zPxIG5t2+jDNV`_ok**r+ga7IEl($397QeXr;Ld&9tt*j%TU&!tJjmG)pYZE?a zvcHr!1NbaOJ|u$m(4j+sjiZ6UF>u6bfmk-1gNFQwXZLK4?Ha5C`GKTAX9+i&6Aie} zbO@v|dqQJ#b4Ri#!$f9o)$;nfnD1*}Utdt4EV9l>I?oF)SO6rgck9Lt0$mW#g$KbG zQzLJE-$ec?*ECJ;G}n~cju;kOXnLK1@@3vVZ=umB;DwMF0had_&4C?>iu`)}UaY2wNK{%v=AokYvAtUdqSee-^` zq{S`|_qA)+T*IHkjzPOM+xq)sqM|+}wmodS5F}BPkDt>u7y?&fB{>I+fFx#Wm#81o#La1LXGR|>NYnv zzAIR0X>}L2%2=K{;;^?hlbKszTT4w#>*nIp_qtlmqQ+VVb>pFUC3$qC0SXspVJfCN7^=tDIq_Ox9U2hy1!;Ev=799L$6;| z*u73Hdwd_%*xivNIlq-7hcm_!I+;X9+$bQqh#z2L7;<$+Gje1^R;-KZ$g^CBi6*9g zW&oR>Ub>fPi@-5_i#kn_1gRmQ;vW=!3M2qv_%zZSvCe`ZOSvo?D=P;9y;MY4GC3=S@C*nwK*Raxe}z>^ixbp6IE^v6bJz-u(T$xrIf6KsFT2cjU4; zA`ruP;lho^nhn8py5f2n*bp`%zilir*D%bjEW zrMno;^SpXb0J?qljheTg^Q&cR?tCSvUgNH9zO>SP6^Cj9cMMOq64?Wh< zK>B?9c9*py!v}NZ_G#}m)Onb21+{30&QUzha%XU&AXbB zKe3WOR4&!CVm)+qqxp2J+{^i%>7jsKXzs4K)Fb)ef^OwB@qoziUWC;d>gsaW37O%e zyaI+MRu%whZz(6i@lyNqxP=)SK$nC0Tt@$3Mjk&{!)fFzCvSADBfczdZ|DjUi3GnV z)GAq(QfSpD2>(_zb8re6vTVb_TCjQ3Ae?!23Axji+^Hy+jN&G?x8DS65g0#HxUlsl;E|X^Bu;7wPz$ zv-Nqoy1L4f=nj@W$SU}R!Q2r92#HroNr}Uim^8e@t}iCiF_q|*qj~Q1>D#U=L?GHD zKMjLWc{R+AY-(y+CX+!aU1-I0baocGu3D(7stzrCo`kxs&v!9<&i3Y`X=*{+gN!%@ zUU55(l-c-s1oBBGQHHk;aAh4(j151GjqI(7@g_h~P4?6x0S>wUlwVmSZLO{0K!FDE ztGXJjcME8_DBA);Z(3g7;&gf<{fQIHYipjf`3-`Cf>%{_LPqfHD(ev+K74dzg?aAe zz;Q?yS=rYzMYD(iz*|6S34LyCZ0zB&3-o<~pVbsicfKt%Gt zW%<_ulgtGS7UajtU%!YlAgE?Dh;yYQPFlA0jPI0hgqhxrIKMv@PI-!`6 zcM@73OKsXJy&hM$VZCJ6*Vl(7^z?}R#Y}tnbn@F>Gg6j|ho|{v zEjV>(XB7pco|_0)6b)DG=Dl_k#YcSD@z9#I3=qxYU#5R!Xuq6^*i~#Oe0_vgajB?tWJvFR(gAGlDto{&|EB6S1(^rN`@#$rK10grZ$a-4V@ z8}cG6E6`7`)GaP9E-x=*u}~vn-dnejqMnj&;(Y^zB)Qb6xHxz6pjP|6c~e_kJ6P^0 zUyb1OiM~%@k$RbDd{KYEPZC;T-J2&HKtY;CWx!d7K<{US4XF|*o>V6`{8CQrozzFA z2!6!8)sEAe9U$U*lpa)(C_n{R1 zDb(aI&=WJdcVc~{O}E4@YzEIYnepCTe?$AG3m&p9G8}s2AYMS;ppp2 z!`*Zj(67Fydpn!1y2!hC?qb)3afunC{-np7&K**PHC*1_F-F9bM)vKM(X<>No!X;CD9N-}hGjNnvkQ50y0iu0+gv5kbl_5dkP>P{I1~DA5+6Un?1PS-)7Q{A%gj7AJ}zX@ z`6(+a3kX=lTLxL|9K$IwaCh!4U&{? zm>d>?)>BVXkX!L1$246tU@#}v3qo;;lFs<&qT4`gr=g`Ksd7L-+N-Ln!eY0wONW!m zAjGKFjR^0bCUq17Dxfuf*TyDCqQRVcx*Ib-KHf1nn56_LWX8hPXtCR^l>Jr;NbZ8h z$n25}rSJ;A;b!4h^mE12==X_xVOI(Rbk zdT(Wl(H<1LYPD98La{AINS3lc*U`mDgeP}-ZItn_E`X}_4&!IzdJNsNOG-p3s*6(( zG0F!G-a_{f4a)4E0Xirw1{~uKA^!vvq~yQ)2OAeZ6k^8lK@)nQl> Date: Mon, 21 Mar 2016 09:54:31 +0000 Subject: [PATCH 006/103] git-workflow.md: Rewrote section on becoming a maintainer --- src/doc/git-workflow.md | 82 ++++++++++++++++++++++++++++--------- src/doc/git-workflow.src.md | 82 ++++++++++++++++++++++++++++--------- 2 files changed, 126 insertions(+), 38 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 293942d2fb..f298d2eddb 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -4,7 +4,7 @@ How do you engage with the Snabb Switch developer community? The answer depends - Use the software, ask questions, report bugs. - Contribute fixes and improvements. -- Maintain Snabb Switch by reviewing and merging pull requests. +- Maintain a part of Snabb Switch by reviewing and merging pull requests. - Create a new application to develop together with the community. ### Using the software @@ -63,34 +63,78 @@ Here are some tips for making your contribution smoothly: Snabb Switch maintainers are the people who review and merge the Pull Requests on Github. Each maintainer takes care of one or more specific -aspects of Snabb Switch. These aspects are called *subsystems*. +part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized -as a network: +as a tree: ![Branches](.images/Branches.png) -Pull Requests are first merged onto the subsystem branch that most -specifically matches their subject matter. For example, a change to -the PDF formatting of the manual would first be reviewed and accepted -by the maintainer of the `pdf-manual` branch. Later, that whole -subsystem branch is merged onto its "next-hop upstream" branch. For -example, the maintainer of the `documentation` branch would merge the -entire `pdf-manual` branch at regular intervals. +Pull Requests are merged onto the most specifically relevant branch to +begin with. Later, whole child branches are merged "upstream" onto +their parent branches. This way once a change has been merged onto any +branch in the tree it will naturally flow upstream to the `master` +branch for release. -#### Registering a subsystem branch - -XXX rewrite. +For example, a change to the LaTeX template for the PDF edition of the +manual would first be reviewed and merged by the maintainer of the +`pdf-manual` branch. The change would then be included in successive +merges upstream to branches `documentation`, `max-next`, `next`, and +finally `master`. Each of these steps provides the opportunity for a +maintainer to improve some aspect of the change before release. -#### Being "the upstream" for Pull Requests +#### Registering a subsystem branch -XXX rewrite. +So you are interested in becoming a Snabb Switch maintainer. Great! +The other maintainers are looking forward to working with you and will +be more than happy to help you learn the ropes. You can learn +everything you need to know on the job: no special qualifications are +required. + +The first step to becoming a maintainer is to decide which kind of +Pull Requests you want to be responsible for. For example you could be +responsible for changes to the Makefiles, or the Intel 10G device +driver, or the `packetblaster` load generator program, or any other +part of the software that is easy to identify when assigning Pull +Requests. You do not have to be an expert in this area already: it is +enough that you are committed to learning about it and being +responsive to contributors. + +Once you have worked out which part you want to maintain then you can +propose this to the community by sending a Pull Request that registers +your new branch in the file `src/doc/branches.md`. This Pull Request +will be the vehicle for talking with the other maintainers about where +to fit your new branch into the tree. + +The moment this Pull Request is merged onto the `next` branch then you +are officially a Snabb Switch maintainer. (Congratulations in advance!) + +#### Being the assigned maintainer for Pull Requests + +Now that you are a registered maintainer you will watch Github for new +Pull Requests and mark yourself as the *Assignee* for changes that +match your area of responsibility. You are the reviewer for these +changes and you decide when to accept them from the contributor. + +Here is the basic criteria for merging a Pull Request: + +- Does the change improve Snabb Switch? It does not have to be perfect + but it should clearly have a net-positive impact. +- Will the next-hop upstream maintainer agree? If not then getting + your branch merged upstream may require you to make some + improvements or even revert the change. + +Beyond this the most important thing is to communicate with the +contributor to make sure they understand exactly what they have to do +in order for their change to be merged. Contributors can easily be +overwhelmed by comments on Pull Requests, often from many different +people, so the assigned maintainer needs to explain very clearly what +the requirements are for merge. (Contributors will be frustrated if +they do not know what feedback they are required to act on, especially +when receiving conflicting ideas from different people, so you need to +decide for yourself what is required and clearly explain this.) #### Sending collected changes upstream to your next-hop XXX rewrite. -#### Putting it all together - -XXX rewrite. - diff --git a/src/doc/git-workflow.src.md b/src/doc/git-workflow.src.md index 16dae03e6e..f882b040b9 100644 --- a/src/doc/git-workflow.src.md +++ b/src/doc/git-workflow.src.md @@ -4,7 +4,7 @@ How do you engage with the Snabb Switch developer community? The answer depends - Use the software, ask questions, report bugs. - Contribute fixes and improvements. -- Maintain Snabb Switch by reviewing and merging pull requests. +- Maintain a part of Snabb Switch by reviewing and merging pull requests. - Create a new application to develop together with the community. ### Using the software @@ -63,10 +63,10 @@ Here are some tips for making your contribution smoothly: Snabb Switch maintainers are the people who review and merge the Pull Requests on Github. Each maintainer takes care of one or more specific -aspects of Snabb Switch. These aspects are called *subsystems*. +part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized -as a network: +as a tree: DIAGRAM: Branches +--lisper @@ -81,27 +81,71 @@ as a network: +--wingo next<--+--lwaftr +--multiproc -Pull Requests are first merged onto the subsystem branch that most -specifically matches their subject matter. For example, a change to -the PDF formatting of the manual would first be reviewed and accepted -by the maintainer of the `pdf-manual` branch. Later, that whole -subsystem branch is merged onto its "next-hop upstream" branch. For -example, the maintainer of the `documentation` branch would merge the -entire `pdf-manual` branch at regular intervals. +Pull Requests are merged onto the most specifically relevant branch to +begin with. Later, whole child branches are merged "upstream" onto +their parent branches. This way once a change has been merged onto any +branch in the tree it will naturally flow upstream to the `master` +branch for release. -#### Registering a subsystem branch - -XXX rewrite. +For example, a change to the LaTeX template for the PDF edition of the +manual would first be reviewed and merged by the maintainer of the +`pdf-manual` branch. The change would then be included in successive +merges upstream to branches `documentation`, `max-next`, `next`, and +finally `master`. Each of these steps provides the opportunity for a +maintainer to improve some aspect of the change before release. -#### Being "the upstream" for Pull Requests +#### Registering a subsystem branch -XXX rewrite. +So you are interested in becoming a Snabb Switch maintainer. Great! +The other maintainers are looking forward to working with you and will +be more than happy to help you learn the ropes. You can learn +everything you need to know on the job: no special qualifications are +required. + +The first step to becoming a maintainer is to decide which kind of +Pull Requests you want to be responsible for. For example you could be +responsible for changes to the Makefiles, or the Intel 10G device +driver, or the `packetblaster` load generator program, or any other +part of the software that is easy to identify when assigning Pull +Requests. You do not have to be an expert in this area already: it is +enough that you are committed to learning about it and being +responsive to contributors. + +Once you have worked out which part you want to maintain then you can +propose this to the community by sending a Pull Request that registers +your new branch in the file `src/doc/branches.md`. This Pull Request +will be the vehicle for talking with the other maintainers about where +to fit your new branch into the tree. + +The moment this Pull Request is merged onto the `next` branch then you +are officially a Snabb Switch maintainer. (Congratulations in advance!) + +#### Being the assigned maintainer for Pull Requests + +Now that you are a registered maintainer you will watch Github for new +Pull Requests and mark yourself as the *Assignee* for changes that +match your area of responsibility. You are the reviewer for these +changes and you decide when to accept them from the contributor. + +Here is the basic criteria for merging a Pull Request: + +- Does the change improve Snabb Switch? It does not have to be perfect + but it should clearly have a net-positive impact. +- Will the next-hop upstream maintainer agree? If not then getting + your branch merged upstream may require you to make some + improvements or even revert the change. + +Beyond this the most important thing is to communicate with the +contributor to make sure they understand exactly what they have to do +in order for their change to be merged. Contributors can easily be +overwhelmed by comments on Pull Requests, often from many different +people, so the assigned maintainer needs to explain very clearly what +the requirements are for merge. (Contributors will be frustrated if +they do not know what feedback they are required to act on, especially +when receiving conflicting ideas from different people, so you need to +decide for yourself what is required and clearly explain this.) #### Sending collected changes upstream to your next-hop XXX rewrite. -#### Putting it all together - -XXX rewrite. - From 4a37b98f4a1f347e94f28f995dadc2efb63d809d Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 22 Mar 2016 09:10:05 +0000 Subject: [PATCH 007/103] git-workflow.md: Wrote about upstreaming subsystem branches also stopped capitalizing "Pull Request" because it seemed awkward. --- src/doc/git-workflow.md | 73 ++++++++++++++++++++++++++----------- src/doc/git-workflow.src.md | 73 ++++++++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 44 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index f298d2eddb..16e4cce577 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -45,24 +45,24 @@ $ make -j ### Contributing fixes and improvements -The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. +The recommended way to contribute improvements to Snabb Switch is with Github *pull requests*. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. 1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. 2. Push your proposed change to a branch on your repository. -3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) -4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. +3. Open a pull request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your pull request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the pull request. Then your work is done and the change is in the pipeline leading to release on the master branch. Here are some tips for making your contribution smoothly: - Use a dedicated "topic branch" for each feature or fix. -- Use the Pull Request text to explain why you are proposing the change. -- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. -- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. +- Use the pull request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the pull request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the pull request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. ### Becoming a maintainer -Snabb Switch maintainers are the people who review and merge the Pull -Requests on Github. Each maintainer takes care of one or more specific +Snabb Switch maintainers are the people who review and merge the pull +requests on Github. Each maintainer takes care of one or more specific part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized @@ -70,7 +70,7 @@ as a tree: ![Branches](.images/Branches.png) -Pull Requests are merged onto the most specifically relevant branch to +Pull requests are merged onto the most specifically relevant branch to begin with. Later, whole child branches are merged "upstream" onto their parent branches. This way once a change has been merged onto any branch in the tree it will naturally flow upstream to the `master` @@ -92,31 +92,31 @@ everything you need to know on the job: no special qualifications are required. The first step to becoming a maintainer is to decide which kind of -Pull Requests you want to be responsible for. For example you could be +pull requests you want to be responsible for. For example you could be responsible for changes to the Makefiles, or the Intel 10G device driver, or the `packetblaster` load generator program, or any other -part of the software that is easy to identify when assigning Pull -Requests. You do not have to be an expert in this area already: it is +part of the software that is easy to identify when assigning pull +requests. You do not have to be an expert in this area already: it is enough that you are committed to learning about it and being responsive to contributors. Once you have worked out which part you want to maintain then you can -propose this to the community by sending a Pull Request that registers -your new branch in the file `src/doc/branches.md`. This Pull Request +propose this to the community by sending a pull request that registers +your new branch in the file `src/doc/branches.md`. This pull request will be the vehicle for talking with the other maintainers about where to fit your new branch into the tree. -The moment this Pull Request is merged onto the `next` branch then you +The moment this pull request is merged onto the `next` branch then you are officially a Snabb Switch maintainer. (Congratulations in advance!) -#### Being the assigned maintainer for Pull Requests +#### Being the assigned maintainer for pull requests Now that you are a registered maintainer you will watch Github for new -Pull Requests and mark yourself as the *Assignee* for changes that +pull requests and mark yourself as the *Assignee* for changes that match your area of responsibility. You are the reviewer for these changes and you decide when to accept them from the contributor. -Here is the basic criteria for merging a Pull Request: +Here is the basic criteria for merging a pull request: - Does the change improve Snabb Switch? It does not have to be perfect but it should clearly have a net-positive impact. @@ -127,14 +127,43 @@ Here is the basic criteria for merging a Pull Request: Beyond this the most important thing is to communicate with the contributor to make sure they understand exactly what they have to do in order for their change to be merged. Contributors can easily be -overwhelmed by comments on Pull Requests, often from many different +overwhelmed by comments on pull requests, often from many different people, so the assigned maintainer needs to explain very clearly what the requirements are for merge. (Contributors will be frustrated if they do not know what feedback they are required to act on, especially when receiving conflicting ideas from different people, so you need to decide for yourself what is required and clearly explain this.) -#### Sending collected changes upstream to your next-hop - -XXX rewrite. +#### Sending your accepted changes upstream + +Once you have merged one or more changes onto your branch the next +step is to open a pull request asking the next upstream maintainer to +merge your whole branch. This way all of the changes you merged will +make their way upstream step-by-step and ultimately be released by +being merged onto the `master` branch. + +This is where it comes in handy that the branches are organized in the +tree structure shown above. When your subsystem branch contains +changes that are ready to merge further upstream then you would open a +pull request to the parent branch in the tree. For example, if you are +the maintainer of the `pdf-manual` branch and you have changes ready +for integration then you would open a pull request to the +`documentation` parent branch. + +How do you decide when to open this pull request? This is a latency +verses throughput trade-off that you need to agree on with the +upstream maintainer. On the one hand it is important for changes to +move upstream quickly when they are beneficial to users or likely to +conflict with other work, on the other hand you may overwhelm the +upstream maintainer if you open pull requests too often. One +suggestion to use as a starting point would be to open a pull request +once or twice per week whenever your branch contains new changes. + +The upstream branch maintainer will be responsible for considering the +broader impact of the changes on your branch. They may need to resolve +conflicts with other changes that they have merged, or want to ask for +input from people who will be affected by the changes, or they may +want to refactor the changes together with some other related code. +They will tell you if they need your help with this and if they have +ideas for how you can make their merging work easier in the future. diff --git a/src/doc/git-workflow.src.md b/src/doc/git-workflow.src.md index f882b040b9..c888819cc3 100644 --- a/src/doc/git-workflow.src.md +++ b/src/doc/git-workflow.src.md @@ -45,24 +45,24 @@ $ make -j ### Contributing fixes and improvements -The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. +The recommended way to contribute improvements to Snabb Switch is with Github *pull requests*. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. 1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. 2. Push your proposed change to a branch on your repository. -3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) -4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. +3. Open a pull request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your pull request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the pull request. Then your work is done and the change is in the pipeline leading to release on the master branch. Here are some tips for making your contribution smoothly: - Use a dedicated "topic branch" for each feature or fix. -- Use the Pull Request text to explain why you are proposing the change. -- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. -- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. +- Use the pull request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the pull request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the pull request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. ### Becoming a maintainer -Snabb Switch maintainers are the people who review and merge the Pull -Requests on Github. Each maintainer takes care of one or more specific +Snabb Switch maintainers are the people who review and merge the pull +requests on Github. Each maintainer takes care of one or more specific part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized @@ -81,7 +81,7 @@ as a tree: +--wingo next<--+--lwaftr +--multiproc -Pull Requests are merged onto the most specifically relevant branch to +Pull requests are merged onto the most specifically relevant branch to begin with. Later, whole child branches are merged "upstream" onto their parent branches. This way once a change has been merged onto any branch in the tree it will naturally flow upstream to the `master` @@ -103,31 +103,31 @@ everything you need to know on the job: no special qualifications are required. The first step to becoming a maintainer is to decide which kind of -Pull Requests you want to be responsible for. For example you could be +pull requests you want to be responsible for. For example you could be responsible for changes to the Makefiles, or the Intel 10G device driver, or the `packetblaster` load generator program, or any other -part of the software that is easy to identify when assigning Pull -Requests. You do not have to be an expert in this area already: it is +part of the software that is easy to identify when assigning pull +requests. You do not have to be an expert in this area already: it is enough that you are committed to learning about it and being responsive to contributors. Once you have worked out which part you want to maintain then you can -propose this to the community by sending a Pull Request that registers -your new branch in the file `src/doc/branches.md`. This Pull Request +propose this to the community by sending a pull request that registers +your new branch in the file `src/doc/branches.md`. This pull request will be the vehicle for talking with the other maintainers about where to fit your new branch into the tree. -The moment this Pull Request is merged onto the `next` branch then you +The moment this pull request is merged onto the `next` branch then you are officially a Snabb Switch maintainer. (Congratulations in advance!) -#### Being the assigned maintainer for Pull Requests +#### Being the assigned maintainer for pull requests Now that you are a registered maintainer you will watch Github for new -Pull Requests and mark yourself as the *Assignee* for changes that +pull requests and mark yourself as the *Assignee* for changes that match your area of responsibility. You are the reviewer for these changes and you decide when to accept them from the contributor. -Here is the basic criteria for merging a Pull Request: +Here is the basic criteria for merging a pull request: - Does the change improve Snabb Switch? It does not have to be perfect but it should clearly have a net-positive impact. @@ -138,14 +138,43 @@ Here is the basic criteria for merging a Pull Request: Beyond this the most important thing is to communicate with the contributor to make sure they understand exactly what they have to do in order for their change to be merged. Contributors can easily be -overwhelmed by comments on Pull Requests, often from many different +overwhelmed by comments on pull requests, often from many different people, so the assigned maintainer needs to explain very clearly what the requirements are for merge. (Contributors will be frustrated if they do not know what feedback they are required to act on, especially when receiving conflicting ideas from different people, so you need to decide for yourself what is required and clearly explain this.) -#### Sending collected changes upstream to your next-hop - -XXX rewrite. +#### Sending your accepted changes upstream + +Once you have merged one or more changes onto your branch the next +step is to open a pull request asking the next upstream maintainer to +merge your whole branch. This way all of the changes you merged will +make their way upstream step-by-step and ultimately be released by +being merged onto the `master` branch. + +This is where it comes in handy that the branches are organized in the +tree structure shown above. When your subsystem branch contains +changes that are ready to merge further upstream then you would open a +pull request to the parent branch in the tree. For example, if you are +the maintainer of the `pdf-manual` branch and you have changes ready +for integration then you would open a pull request to the +`documentation` parent branch. + +How do you decide when to open this pull request? This is a latency +verses throughput trade-off that you need to agree on with the +upstream maintainer. On the one hand it is important for changes to +move upstream quickly when they are beneficial to users or likely to +conflict with other work, on the other hand you may overwhelm the +upstream maintainer if you open pull requests too often. One +suggestion to use as a starting point would be to open a pull request +once or twice per week whenever your branch contains new changes. + +The upstream branch maintainer will be responsible for considering the +broader impact of the changes on your branch. They may need to resolve +conflicts with other changes that they have merged, or want to ask for +input from people who will be affected by the changes, or they may +want to refactor the changes together with some other related code. +They will tell you if they need your help with this and if they have +ideas for how you can make their merging work easier in the future. From 94ff2347175cc2e56153f0af3d16f9af86c2a367 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 12 Apr 2016 19:41:12 +0200 Subject: [PATCH 008/103] Amendments to #766: - Use "apps/" instead of "app/" for uniformity - Set shm path to "apps/$name" when calling `app:stop' too - Unlink "apps/$name" after `app:stop' using `shm.unlink' - Add a test case to core.app selftest --- src/core/app.lua | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 099fc3125a..5ff7d476eb 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -156,7 +156,13 @@ function apply_config_actions (actions, conf) -- Table of functions that execute config actions local ops = {} function ops.stop (name) - if app_table[name].stop then app_table[name]:stop() end + if app_table[name].stop then + local shmorig = shm.path + shm.path = app_table[name].shmpath + app_table[name]:stop() + shm.path = shmorig + shm.unlink(app_table[name].shmpath) + end end function ops.keep (name) new_app_table[name] = app_table[name] @@ -166,11 +172,10 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg - local shmpath, shmorig = "app/"..name, shm.path + local shmpath, shmorig = "apps/"..name, shm.path shm.path = shmpath local app = class:new(arg) shm.path = shmorig - local shmpath = "app/"..name if type(app) ~= 'table' then error(("bad return value from app '%s' start() method: %s"):format( name, tostring(app))) @@ -492,6 +497,34 @@ function selftest () assert(app_table.app3 == orig_app3) -- should be the same main({duration = 4, report = {showapps = true}}) assert(app_table.app3 ~= orig_app3) -- should be restarted + -- Test shm.path management + print("shm.path management") + local S = require("syscall") + local App4 = {zone="test"} + function App4:new () + local c = counter.open('test') + counter.set(c, 42) + counter.commit() + return setmetatable({test_counter = c}, + {__index = App4}) + end + function App4:pull () + assert(counter.read(self.test_counter) == 42, "Invalid counter value") + counter.add(self.test_counter) + end + function App4:stop () + assert(counter.read(self.test_counter) == 43, "Invalid counter value") + counter.delete('test') + end + local c_counter = config.new() + config.app(c_counter, "App4", App4) + configure(c_counter) + main({done = function () return app_table.App4.test_counter end}) + assert(S.stat(shm.root.."/"..shm.resolve("apps/App4/test")), + "Missing : apps/App4/test") + configure(config.new()) + assert(not S.stat(shm.root.."/"..shm.resolve("apps/App4")), + "Failed to unlink apps/App4") print("OK") end From fad0f43921d8f226f199f1cbaed02e5e6efc6d30 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 13 Apr 2016 16:44:14 +0200 Subject: [PATCH 009/103] core.counter: Qualify counter names using `shm.resolve'. --- src/core/counter.lua | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/core/counter.lua b/src/core/counter.lua index a479df4040..881cea5717 100644 --- a/src/core/counter.lua +++ b/src/core/counter.lua @@ -44,7 +44,8 @@ local private = {} local numbers = {} -- name -> number function open (name, readonly) - if numbers[name] then error("counter already opened: " .. name) end + local qname = shm.resolve(name) + if numbers[qname] then error("counter already opened: " .. qname) end local n = #public+1 if readonly then public[n] = shm.open(name, counter_t, readonly) @@ -53,13 +54,14 @@ function open (name, readonly) public[n] = shm.create(name, counter_t) private[n] = ffi.new(counter_t) end - numbers[name] = n + numbers[qname] = n return private[n] end function delete (name) - local number = numbers[name] - if not number then error("counter not found for deletion: " .. name) end + local qname = shm.resolve(name) + local number = numbers[qname] + if not number then error("counter not found for deletion: " .. qname) end -- Free shm object shm.unmap(public[number]) -- If we "own" the counter for writing then we unlink it too. @@ -67,7 +69,7 @@ function delete (name) shm.unlink(name) end -- Free local state - numbers[name] = false + numbers[qname] = false public[number] = false private[number] = false end From 7ed4ed0383bb24fd780fb6ecda3197f5c1106165 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 12 Apr 2016 20:21:33 +0200 Subject: [PATCH 010/103] snabb top: add `--app' option to print app counters. --- src/program/top/README | 8 ++++-- src/program/top/top.lua | 61 +++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 365c357326..66471f390c 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -3,10 +3,12 @@ Usage: -h, --help Print usage information. + -a, --app + Print counters of app by and exit. -Display realtime performance statistics for a running Snabb -instance with . If is not supplied and there is only one Snabb -Switch instance, top will connect to that instance. +Display realtime performance statistics for a running Snabb instance with +. If is not supplied and there is only one Snabb instance, top will +connect to that instance. The following global metrics will be displayed: diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 6617027b1b..f66af6df0d 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,20 +12,57 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h" + help = "h", app = "a" } function clearterm () io.write('\027[2J') end function run (args) local opt = {} + local app_name = nil function opt.h (arg) print(usage) main.exit(1) end - args = lib.dogetopt(args, opt, "h", long_opts) + function opt.a (arg) app_name = arg end + args = lib.dogetopt(args, opt, "ha:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = args[1] - local instance_tree = "//"..(select_snabb_instance(target_pid)) + if app_name then app(select_snabb_instance(target_pid), app_name) + else top(select_snabb_instance(target_pid)) end +end + +function select_snabb_instance (pid) + local instances = shm.children("//") + if pid then + -- Try to use given pid + for _, instance in ipairs(instances) do + if instance == pid then return pid end + end + print("No such Snabb instance: "..pid) + elseif #instances == 2 then + -- Two means one is us, so we pick the other. + local own_pid = tostring(S.getpid()) + if instances[1] == own_pid then return instances[2] + else return instances[1] end + elseif #instances == 1 then print("No Snabb instance found.") + else print("Multple Snabb instances found. Select one.") end + os.exit(1) +end + +function app (instance_pid, app_name) + local app_tree = "//"..instance_pid.."/apps/"..app_name + local cnames = shm.children(app_tree) + table.sort(cnames, function (a, b) return a < b end) + for _, cname in ipairs(cnames) do + local cpath = app_tree.."/"..cname + local value = counter.read(counter.open(cpath, 'readonly')) + print_row({30, 30}, {cname, lib.comma_value(value)}) + counter.delete(cpath) + end +end + +function top (instance_pid) + local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) local configs = 0 local last_stats = nil @@ -48,24 +85,6 @@ function run (args) end end -function select_snabb_instance (pid) - local instances = shm.children("//") - if pid then - -- Try to use given pid - for _, instance in ipairs(instances) do - if instance == pid then return pid end - end - print("No such Snabb instance: "..pid) - elseif #instances == 2 then - -- Two means one is us, so we pick the other. - local own_pid = tostring(S.getpid()) - if instances[1] == own_pid then return instances[2] - else return instances[1] end - elseif #instances == 1 then print("No Snabb instance found.") - else print("Multple Snabb instances found. Select one.") end - os.exit(1) -end - function open_counters (tree) local counters = {} for _, name in ipairs({"configs", "breaths", "frees", "freebytes"}) do From eb9005b5c5ec4aa637b8a327f556029f8ef4cf29 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 13 Apr 2016 17:24:44 +0200 Subject: [PATCH 011/103] snabb top: unlink own shm tree to avoid clutter. --- src/program/top/top.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index f66af6df0d..54308c63f3 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -29,6 +29,7 @@ function run (args) if app_name then app(select_snabb_instance(target_pid), app_name) else top(select_snabb_instance(target_pid)) end + ordered_exit(0) end function select_snabb_instance (pid) @@ -46,7 +47,12 @@ function select_snabb_instance (pid) else return instances[1] end elseif #instances == 1 then print("No Snabb instance found.") else print("Multple Snabb instances found. Select one.") end - os.exit(1) + ordered_exit(1) +end + +function ordered_exit (value) + shm.unlink("//"..S.getpid()) -- Unlink own shm tree to avoid clutter + os.exit(value) end function app (instance_pid, app_name) From 5fbe0d6fba0a1c0fd785102bb6da0a184f95dfd5 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 16:39:55 +0200 Subject: [PATCH 012/103] vhost_user: Add RFC 7223 app counters. --- src/apps/vhost/vhost_user.lua | 36 +++++++++++++++++++++++++++++++++++ src/lib/virtio/net_device.lua | 4 ++++ 2 files changed, 40 insertions(+) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 7890b495f5..96e82780eb 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -13,7 +13,9 @@ local lib = require("core.lib") local link = require("core.link") local main = require("core.main") local memory = require("core.memory") +local counter = require("core.counter") local pci = require("lib.hardware.pci") +local ethernet = require("lib.protocol.ethernet") local net_device= require("lib.virtio.net_device") local timer = require("core.timer") local ffi = require("ffi") @@ -52,6 +54,15 @@ function VhostUser:new (args) else self.qemu_connect = self.client_connect end + -- initialize counters + self.counters = {} + for _, name in ipairs({'type', 'discontinuity-time', + 'in-octets', 'in-unicast', 'in-multicast', 'in-discards', + 'out-octets', 'out-unicast', 'out-multicast'}) do + self.counters[name] = counter.open(name) + end + counter.set(self.counters['type'], 0x1001) -- Virtual interface + counter.set(self.counters['discontinuity-time'], C.get_unix_time()) return self end @@ -68,6 +79,9 @@ function VhostUser:stop() self:free_mem_table() if self.link_down_proc then self.link_down_proc() end + + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end function VhostUser:pull () @@ -86,6 +100,28 @@ function VhostUser:push () end end +function VhostUser:tx_callback (p) + counter.add(self.counters['out-octets'], packet.length(p)) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters['out-multicast']) + else + counter.add(self.counters['out-unicast']) + end +end + +function VhostUser:rx_callback (p) + counter.add(self.counters['in-octets'], packet.length(p)) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters['in-multicast']) + else + counter.add(self.counters['in-unicast']) + end +end + +function VhostUser:rxdrop_callback (p) + counter.add(self.counters['in-discards']) +end + -- Try to connect to QEMU. function VhostUser:client_connect () return C.vhost_user_connect(self.socket_path) diff --git a/src/lib/virtio/net_device.lua b/src/lib/virtio/net_device.lua index 1f9f321191..0cbdcd13c3 100644 --- a/src/lib/virtio/net_device.lua +++ b/src/lib/virtio/net_device.lua @@ -163,9 +163,11 @@ function VirtioNetDevice:rx_packet_end(header_id, total_size, rx_p) rx_p.length - self.rx_hdr_csum_start, self.rx_hdr_csum_offset) end + self.owner:rx_callback(rx_p) link.transmit(l, rx_p) else debug("droprx", "len", rx_p.length) + self.owner:rxdrop_callback(rx_p) packet.free(rx_p) end self.virtq[self.ring_id]:put_buffer(header_id, total_size) @@ -253,6 +255,7 @@ function VirtioNetDevice:tx_buffer_add(tx_p, addr, len) end function VirtioNetDevice:tx_packet_end(header_id, total_size, tx_p) + self.owner:tx_callback(tx_p) packet.free(tx_p) self.virtq[self.ring_id]:put_buffer(header_id, total_size) end @@ -313,6 +316,7 @@ end function VirtioNetDevice:tx_packet_end_mrg_rxbuf(header_id, total_size, tx_p) -- free the packet only when all its data is processed if self.tx.finished then + self.owner:tx_callback(tx_p) packet.free(tx_p) self.tx.p = nil self.tx.data_sent = nil From 8bb3215bf759f6c53af07c5c13afd2ea60003334 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 17:57:09 +0200 Subject: [PATCH 013/103] Intel_app: Add RFC 7223 app counters. --- src/apps/intel/intel_app.lua | 45 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 80cace3d9f..f104fad553 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,8 +6,10 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") +local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") +local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local freelist = require("core.freelist") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty @@ -36,6 +38,7 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) + local self = setmetatable({counters = {}}, Intel82599) if conf.vmdq then if devices[conf.pciaddr] == nil then @@ -45,12 +48,26 @@ function Intel82599:new (arg) local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - return setmetatable({dev=vf:open(conf)}, Intel82599) + self.dev = vf:open(conf) else - local dev = intel10g.new_sf(conf):open() - if not dev then return null end - return setmetatable({dev=dev, zone="intel"}, Intel82599) + self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") + self.zone = "intel" end + + self.counters['type'] = counter.open('type') + self.counters['discontinuity-time'] = counter.open('discontinuity-time') + self.counters['in-octets'] = counter.open('in-octets') + self.counters['in-discards'] = counter.open('in-discards') + self.counters['out-octets'] = counter.open('out-octets') + counter.set(self.counters['type'], 0x1000) -- Hardware interface + counter.set(self.counters['discontinuity-time'], C.get_unix_time()) + if conf.vmdq then + self.counters['phys-address'] = counter.open('phys-address') + counter.set(self.counters['phys-address'], + macaddress:new(conf.macaddr).bits) + end + + return self end function Intel82599:stop() @@ -71,6 +88,9 @@ function Intel82599:stop() if close_pf then close_pf:close() end + + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end @@ -79,6 +99,11 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) + + if conf.vmdq then + counter.set(self.counters['phys-address'], + macaddress:new(conf.macaddr).bits) + end end -- Allocate receive buffers from the given freelist. @@ -96,6 +121,13 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() + if self.dev.rxstats then + counter.set(self.counters['in-octets'], + bit.lshift(self.dev.pf.qs.QBRC_H[self.dev.rxstats]()+0LL, 32) + + self.dev.pf.qs.QBRC_L[self.dev.rxstats]()) + counter.set(self.counters['in-discards'], + self.dev.pf.qs.QPRDC[self.dev.rxstats]()) + end end function Intel82599:add_receive_buffers () @@ -116,6 +148,11 @@ function Intel82599:push () end end self.dev:sync_transmit() + if self.dev.txstats then + counter.set(self.counters['out-octets'], + bit.lshift(self.dev.pf.qs.QBTC_H[self.dev.txstats]()+0LL, 32) + + self.dev.pf.qs.QBTC_L[self.dev.txstats]()) + end end -- Report on relevant status and statistics. From 7a5547859a92ba935ed93ef1d144f1a9d8a1ab39 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 18:13:51 +0200 Subject: [PATCH 014/103] snabb top: Add --link parameter to list link counters. --- src/program/top/README | 3 ++- src/program/top/top.lua | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 66471f390c..ae91f6e4c4 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -4,7 +4,8 @@ Usage: -h, --help Print usage information. -a, --app - Print counters of app by and exit. + -l, --link + Print counters of app or link by and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 54308c63f3..3b1f57a600 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,7 +12,7 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h", app = "a" + help = "h", app = "a", link = "l" } function clearterm () io.write('\027[2J') end @@ -20,15 +20,18 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local app_name = nil + local link_name = nil function opt.h (arg) print(usage) main.exit(1) end function opt.a (arg) app_name = arg end - args = lib.dogetopt(args, opt, "ha:", long_opts) + function opt.l (arg) link_name = arg end + args = lib.dogetopt(args, opt, "ha:l:", long_opts) if #args > 1 then print(usage) main.exit(1) end - local target_pid = args[1] + local target_pid = select_snabb_instance(args[1]) - if app_name then app(select_snabb_instance(target_pid), app_name) - else top(select_snabb_instance(target_pid)) end + if app_name then list_counters("//"..target_pid.."/apps/"..app_name) + elseif link_name then list_counters("//"..target_pid.."/counters/"..link_name) + else top(target_pid) end ordered_exit(0) end @@ -55,12 +58,11 @@ function ordered_exit (value) os.exit(value) end -function app (instance_pid, app_name) - local app_tree = "//"..instance_pid.."/apps/"..app_name - local cnames = shm.children(app_tree) +function list_counters (path) + local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do - local cpath = app_tree.."/"..cname + local cpath = path.."/"..cname local value = counter.read(counter.open(cpath, 'readonly')) print_row({30, 30}, {cname, lib.comma_value(value)}) counter.delete(cpath) From dde5da2ac77cda91ed806b68399d7680c92004e3 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 19:17:20 +0200 Subject: [PATCH 015/103] core.app: Put app counters under "counters/", update snabb top. --- src/core/app.lua | 10 +++++----- src/program/top/README | 5 ++--- src/program/top/top.lua | 18 ++++++++---------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 352f5999f3..fa251390bb 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -175,7 +175,7 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg - local shmpath, shmorig = "apps/"..name, shm.path + local shmpath, shmorig = "counters/"..name, shm.path shm.path = shmpath local app = class:new(arg) shm.path = shmorig @@ -530,11 +530,11 @@ function selftest () config.app(c_counter, "App4", App4) configure(c_counter) main({done = function () return app_table.App4.test_counter end}) - assert(S.stat(shm.root.."/"..shm.resolve("apps/App4/test")), - "Missing : apps/App4/test") + assert(S.stat(shm.root.."/"..shm.resolve("counters/App4/test")), + "Missing : counters/App4/test") configure(config.new()) - assert(not S.stat(shm.root.."/"..shm.resolve("apps/App4")), - "Failed to unlink apps/App4") + assert(not S.stat(shm.root.."/"..shm.resolve("counters/App4")), + "Failed to unlink counters/App4") print("OK") end diff --git a/src/program/top/README b/src/program/top/README index ae91f6e4c4..ce815d6f89 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -3,9 +3,8 @@ Usage: -h, --help Print usage information. - -a, --app - -l, --link - Print counters of app or link by and exit. + -c, --counters + Print counters of object by and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 3b1f57a600..d69a999695 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,26 +12,23 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h", app = "a", link = "l" + help = "h", counters = "c" } function clearterm () io.write('\027[2J') end function run (args) local opt = {} - local app_name = nil - local link_name = nil + local object = nil function opt.h (arg) print(usage) main.exit(1) end - function opt.a (arg) app_name = arg end - function opt.l (arg) link_name = arg end - args = lib.dogetopt(args, opt, "ha:l:", long_opts) + function opt.c (arg) object = arg end + args = lib.dogetopt(args, opt, "hc:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) - if app_name then list_counters("//"..target_pid.."/apps/"..app_name) - elseif link_name then list_counters("//"..target_pid.."/counters/"..link_name) - else top(target_pid) end + if object then list_counters(target_pid, object) + else top(target_pid) end ordered_exit(0) end @@ -58,7 +55,8 @@ function ordered_exit (value) os.exit(value) end -function list_counters (path) +function list_counters (pid, object) + local path = "//"..pid.."/counters/"..object local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do From 7a6296f74b777b9c3203beb091cbd3f904f8b818 Mon Sep 17 00:00:00 2001 From: Pete Bristow Date: Thu, 14 Apr 2016 21:08:37 +0100 Subject: [PATCH 016/103] Add a test demonstrating pci device ids, with capital A-F --- src/lib/hardware/pci.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index d75ee4a046..88182c7307 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -176,6 +176,7 @@ function selftest () assert(qualified("0000:01:00.0") == "0000:01:00.0", "qualified 1") assert(qualified( "01:00.0") == "0000:01:00.0", "qualified 2") assert(qualified( "0a:00.0") == "0000:0a:00.0", "qualified 3") + assert(qualified( "0A:00.0") == "0000:0A:00.0", "qualified 4") assert(canonical("0000:01:00.0") == "01:00.0", "canonical 1") assert(canonical( "01:00.0") == "01:00.0", "canonical 2") scan_devices() From 924ff4e6e82880d296d6be9ddae4dda3aef4dff0 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 19:30:15 +0200 Subject: [PATCH 017/103] lib.json: Import JSON4Lua 1.0.0, include encode functionality. --- src/lib/json.lua | 658 ++++++++++++++++++++++++++++++----------------- 1 file changed, 417 insertions(+), 241 deletions(-) diff --git a/src/lib/json.lua b/src/lib/json.lua index 86cbf85e43..9ab4abd38e 100644 --- a/src/lib/json.lua +++ b/src/lib/json.lua @@ -1,241 +1,417 @@ --- JSON4Lua: JSON encoding / decoding support for the Lua language. --- json Module. --- Author: Craig Mason-Jones --- Homepage: http://json.luaforge.net/ --- Version: 0.9.40 --- This module is released under the MIT License (MIT). --- --- NOTE: This is only the decode functionality ripped out from JSON4Lua. --- See: https://github.com/craigmj/json4lua - -module(..., package.seeall) - -local math = require('math') -local string = require("string") -local table = require("table") - -local base = _G - --- Private functions -local decode_scanArray -local decode_scanComment -local decode_scanConstant -local decode_scanNumber -local decode_scanObject -local decode_scanString -local decode_scanWhitespace - ---- Decodes a JSON string and returns the decoded value as a Lua data structure / value. --- @param s The string to scan. --- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. --- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, --- and the position of the first character after --- the scanned JSON object. -function decode(s, startPos) - startPos = startPos and startPos or 1 - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') - local curChar = string.sub(s,startPos,startPos) - -- Object - if curChar=='{' then - return decode_scanObject(s,startPos) - end - -- Array - if curChar=='[' then - return decode_scanArray(s,startPos) - end - -- Number - if string.find("+-0123456789.e", curChar, 1, true) then - return decode_scanNumber(s,startPos) - end - -- String - if curChar==[["]] or curChar==[[']] then - return decode_scanString(s,startPos) - end - if string.sub(s,startPos,startPos+1)=='/*' then - return decode(s, decode_scanComment(s,startPos)) - end - -- Otherwise, it must be a constant - return decode_scanConstant(s,startPos) -end - ---- The null function allows one to specify a null value in an associative array (which is otherwise --- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } -function null() - return null -- so json.null() will also return null ;-) -end ------------------------------------------------------------------------------ --- Internal, PRIVATE functions. --- Following a Python-like convention, I have prefixed all these 'PRIVATE' --- functions with an underscore. ------------------------------------------------------------------------------ - ---- Scans an array from JSON into a Lua object --- startPos begins at the start of the array. --- Returns the array and the next starting position --- @param s The string being scanned. --- @param startPos The starting position for the scan. --- @return table, int The scanned array as a table, and the position of the next character to scan. -function decode_scanArray(s,startPos) - local array = {} -- The return value - local stringLen = string.len(s) - base.assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) - startPos = startPos + 1 - -- Infinite loop for array elements - repeat - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') - local curChar = string.sub(s,startPos,startPos) - if (curChar==']') then - return array, startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - base.assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') - object, startPos = decode(s,startPos) - table.insert(array,object) - until false -end - ---- Scans a comment and discards the comment. --- Returns the position of the next character following the comment. --- @param string s The JSON string to scan. --- @param int startPos The starting position of the comment -function decode_scanComment(s, startPos) - base.assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) - local endPos = string.find(s,'*/',startPos+2) - base.assert(endPos~=nil, "Unterminated comment in string at " .. startPos) - return endPos+2 -end - ---- Scans for given constants: true, false or null --- Returns the appropriate Lua type, and the position of the next character to read. --- @param s The string being scanned. --- @param startPos The position in the string at which to start scanning. --- @return object, int The object (true, false or nil) and the position at which the next character should be --- scanned. -function decode_scanConstant(s, startPos) - local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } - local constNames = {"true","false","null"} - - for i,k in base.pairs(constNames) do - --print ("[" .. string.sub(s,startPos, startPos + string.len(k) -1) .."]", k) - if string.sub(s,startPos, startPos + string.len(k) -1 )==k then - return consts[k], startPos + string.len(k) - end - end - base.assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) -end - ---- Scans a number from the JSON encoded string. --- (in fact, also is able to scan numeric +- eqns, which is not --- in the JSON spec.) --- Returns the number, and the position of the next character --- after the number. --- @param s The string being scanned. --- @param startPos The position at which to start scanning. --- @return number, int The extracted number and the position of the next character to scan. -function decode_scanNumber(s,startPos) - local endPos = startPos+1 - local stringLen = string.len(s) - local acceptableChars = "+-0123456789.e" - while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) - and endPos<=stringLen - ) do - endPos = endPos + 1 - end - local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) - local stringEval = base.loadstring(stringValue) - base.assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON object into a Lua object. --- startPos begins at the start of the object. --- Returns the object and the next starting position. --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return table, int The scanned object as a table and the position of the next character to scan. -function decode_scanObject(s,startPos) - local object = {} - local stringLen = string.len(s) - local key, value - base.assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) - startPos = startPos + 1 - repeat - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') - local curChar = string.sub(s,startPos,startPos) - if (curChar=='}') then - return object,startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') - -- Scan the key - key, startPos = decode(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - base.assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) - startPos = decode_scanWhitespace(s,startPos+1) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - value, startPos = decode(s,startPos) - object[key]=value - until false -- infinite loop while key-value pairs are found -end - ---- Scans a JSON string from the opening inverted comma or single quote to the --- end of the string. --- Returns the string extracted as a Lua string, --- and the position of the next non-string character --- (after the closing inverted comma or single quote). --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return string, int The extracted string as a Lua string, and the next character to parse. -function decode_scanString(s,startPos) - base.assert(startPos, 'decode_scanString(..) called without start position') - local startChar = string.sub(s,startPos,startPos) - base.assert(startChar==[[']] or startChar==[["]],'decode_scanString called for a non-string') - local escaped = false - local endPos = startPos + 1 - local bEnded = false - local stringLen = string.len(s) - repeat - local curChar = string.sub(s,endPos,endPos) - -- Character escaping is only used to escape the string delimiters - if not escaped then - if curChar==[[\]] then - escaped = true - else - bEnded = curChar==startChar - end - else - -- If we're escaped, we accept the current character come what may - escaped = false - end - endPos = endPos + 1 - base.assert(endPos <= stringLen+1, "String decoding failed: unterminated string at position " .. endPos) - until bEnded - local stringValue = 'return ' .. string.sub(s, startPos, endPos-1) - local stringEval = base.loadstring(stringValue) - base.assert(stringEval, 'Failed to load string [ ' .. stringValue .. '] in JSON4Lua.decode_scanString at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON string skipping all whitespace from the current start position. --- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. --- @param s The string being scanned --- @param startPos The starting position where we should begin removing whitespace. --- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string --- was reached. -function decode_scanWhitespace(s,startPos) - local whitespace=" \n\r\t" - local stringLen = string.len(s) - while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do - startPos = startPos + 1 - end - return startPos -end +----------------------------------------------------------------------------- +-- JSON4Lua: JSON encoding / decoding support for the Lua language. +-- json Module. +-- Author: Craig Mason-Jones +-- Homepage: http://github.com/craigmj/json4lua/ +-- Version: 1.0.0 +-- This module is released under the MIT License (MIT). +-- Please see LICENCE.txt for details. +-- +-- USAGE: +-- This module exposes two functions: +-- json.encode(o) +-- Returns the table / string / boolean / number / nil / json.null value as a JSON-encoded string. +-- json.decode(json_string) +-- Returns a Lua object populated with the data encoded in the JSON string json_string. +-- +-- REQUIREMENTS: +-- compat-5.1 if using Lua 5.0 +-- +-- CHANGELOG +-- 0.9.20 Introduction of local Lua functions for private functions (removed _ function prefix). +-- Fixed Lua 5.1 compatibility issues. +-- Introduced json.null to have null values in associative arrays. +-- json.encode() performance improvement (more than 50%) through table.concat rather than .. +-- Introduced decode ability to ignore /**/ comments in the JSON string. +-- 0.9.10 Fix to array encoding / decoding to correctly manage nil/null values in arrays. +----------------------------------------------------------------------------- + +----------------------------------------------------------------------------- +-- Imports and dependencies +----------------------------------------------------------------------------- +local math = require('math') +local string = require("string") +local table = require("table") + +----------------------------------------------------------------------------- +-- Module declaration +----------------------------------------------------------------------------- +local json = {} -- Public namespace +local json_private = {} -- Private namespace + +-- Public functions + +-- Private functions +local decode_scanArray +local decode_scanComment +local decode_scanConstant +local decode_scanNumber +local decode_scanObject +local decode_scanString +local decode_scanWhitespace +local encodeString +local isArray +local isEncodable + +----------------------------------------------------------------------------- +-- PUBLIC FUNCTIONS +----------------------------------------------------------------------------- +--- Encodes an arbitrary Lua object / variable. +-- @param v The Lua object / variable to be JSON encoded. +-- @return String containing the JSON encoding in internal Lua string format (i.e. not unicode) +function json.encode (v) + -- Handle nil values + if v==nil then + return "null" + end + + local vtype = type(v) + + -- Handle strings + if vtype=='string' then + return '"' .. json_private.encodeString(v) .. '"' -- Need to handle encoding in string + end + + -- Handle booleans + if vtype=='number' or vtype=='boolean' then + return tostring(v) + end + + -- Handle tables + if vtype=='table' then + local rval = {} + -- Consider arrays separately + local bArray, maxCount = isArray(v) + if bArray then + for i = 1,maxCount do + table.insert(rval, json.encode(v[i])) + end + else -- An object, not an array + for i,j in pairs(v) do + if isEncodable(i) and isEncodable(j) then + table.insert(rval, '"' .. json_private.encodeString(i) .. '":' .. json.encode(j)) + end + end + end + if bArray then + return '[' .. table.concat(rval,',') ..']' + else + return '{' .. table.concat(rval,',') .. '}' + end + end + + -- Handle null values + if vtype=='function' and v==null then + return 'null' + end + + assert(false,'encode attempt to encode unsupported type ' .. vtype .. ':' .. tostring(v)) +end + + +--- Decodes a JSON string and returns the decoded value as a Lua data structure / value. +-- @param s The string to scan. +-- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. +-- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, +-- and the position of the first character after +-- the scanned JSON object. +function json.decode(s, startPos) + startPos = startPos and startPos or 1 + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') + local curChar = string.sub(s,startPos,startPos) + -- Object + if curChar=='{' then + return decode_scanObject(s,startPos) + end + -- Array + if curChar=='[' then + return decode_scanArray(s,startPos) + end + -- Number + if string.find("+-0123456789.e", curChar, 1, true) then + return decode_scanNumber(s,startPos) + end + -- String + if curChar==[["]] or curChar==[[']] then + return decode_scanString(s,startPos) + end + if string.sub(s,startPos,startPos+1)=='/*' then + return decode(s, decode_scanComment(s,startPos)) + end + -- Otherwise, it must be a constant + return decode_scanConstant(s,startPos) +end + +--- The null function allows one to specify a null value in an associative array (which is otherwise +-- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } +function null() + return null -- so json.null() will also return null ;-) +end +----------------------------------------------------------------------------- +-- Internal, PRIVATE functions. +-- Following a Python-like convention, I have prefixed all these 'PRIVATE' +-- functions with an underscore. +----------------------------------------------------------------------------- + +--- Scans an array from JSON into a Lua object +-- startPos begins at the start of the array. +-- Returns the array and the next starting position +-- @param s The string being scanned. +-- @param startPos The starting position for the scan. +-- @return table, int The scanned array as a table, and the position of the next character to scan. +function decode_scanArray(s,startPos) + local array = {} -- The return value + local stringLen = string.len(s) + assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) + startPos = startPos + 1 + -- Infinite loop for array elements + repeat + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') + local curChar = string.sub(s,startPos,startPos) + if (curChar==']') then + return array, startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') + object, startPos = json.decode(s,startPos) + table.insert(array,object) + until false +end + +--- Scans a comment and discards the comment. +-- Returns the position of the next character following the comment. +-- @param string s The JSON string to scan. +-- @param int startPos The starting position of the comment +function decode_scanComment(s, startPos) + assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) + local endPos = string.find(s,'*/',startPos+2) + assert(endPos~=nil, "Unterminated comment in string at " .. startPos) + return endPos+2 +end + +--- Scans for given constants: true, false or null +-- Returns the appropriate Lua type, and the position of the next character to read. +-- @param s The string being scanned. +-- @param startPos The position in the string at which to start scanning. +-- @return object, int The object (true, false or nil) and the position at which the next character should be +-- scanned. +function decode_scanConstant(s, startPos) + local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } + local constNames = {"true","false","null"} + + for i,k in pairs(constNames) do + if string.sub(s,startPos, startPos + string.len(k) -1 )==k then + return consts[k], startPos + string.len(k) + end + end + assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) +end + +--- Scans a number from the JSON encoded string. +-- (in fact, also is able to scan numeric +- eqns, which is not +-- in the JSON spec.) +-- Returns the number, and the position of the next character +-- after the number. +-- @param s The string being scanned. +-- @param startPos The position at which to start scanning. +-- @return number, int The extracted number and the position of the next character to scan. +function decode_scanNumber(s,startPos) + local endPos = startPos+1 + local stringLen = string.len(s) + local acceptableChars = "+-0123456789.e" + while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) + and endPos<=stringLen + ) do + endPos = endPos + 1 + end + local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) + local stringEval = loadstring(stringValue) + assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON object into a Lua object. +-- startPos begins at the start of the object. +-- Returns the object and the next starting position. +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return table, int The scanned object as a table and the position of the next character to scan. +function decode_scanObject(s,startPos) + local object = {} + local stringLen = string.len(s) + local key, value + assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) + startPos = startPos + 1 + repeat + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') + local curChar = string.sub(s,startPos,startPos) + if (curChar=='}') then + return object,startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') + -- Scan the key + key, startPos = json.decode(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) + startPos = decode_scanWhitespace(s,startPos+1) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + value, startPos = json.decode(s,startPos) + object[key]=value + until false -- infinite loop while key-value pairs are found +end + +-- START SoniEx2 +-- Initialize some things used by decode_scanString +-- You know, for efficiency +local escapeSequences = { + ["\\t"] = "\t", + ["\\f"] = "\f", + ["\\r"] = "\r", + ["\\n"] = "\n", + ["\\b"] = "\b" +} +setmetatable(escapeSequences, {__index = function(t,k) + -- skip "\" aka strip escape + return string.sub(k,2) +end}) +-- END SoniEx2 + +--- Scans a JSON string from the opening inverted comma or single quote to the +-- end of the string. +-- Returns the string extracted as a Lua string, +-- and the position of the next non-string character +-- (after the closing inverted comma or single quote). +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return string, int The extracted string as a Lua string, and the next character to parse. +function decode_scanString(s,startPos) + assert(startPos, 'decode_scanString(..) called without start position') + local startChar = string.sub(s,startPos,startPos) + -- START SoniEx2 + -- PS: I don't think single quotes are valid JSON + assert(startChar == [["]] or startChar == [[']],'decode_scanString called for a non-string') + --assert(startPos, "String decoding failed: missing closing " .. startChar .. " for string at position " .. oldStart) + local t = {} + local i,j = startPos,startPos + while string.find(s, startChar, j+1) ~= j+1 do + local oldj = j + i,j = string.find(s, "\\.", j+1) + local x,y = string.find(s, startChar, oldj+1) + if not i or x < i then + i,j = x,y-1 + end + table.insert(t, string.sub(s, oldj+1, i-1)) + if string.sub(s, i, j) == "\\u" then + local a = string.sub(s,j+1,j+4) + j = j + 4 + local n = tonumber(a, 16) + assert(n, "String decoding failed: bad Unicode escape " .. a .. " at position " .. i .. " : " .. j) + -- math.floor(x/2^y) == lazy right shift + -- a % 2^b == bitwise_and(a, (2^b)-1) + -- 64 = 2^6 + -- 4096 = 2^12 (or 2^6 * 2^6) + local x + if n < 0x80 then + x = string.char(n % 0x80) + elseif n < 0x800 then + -- [110x xxxx] [10xx xxxx] + x = string.char(0xC0 + (math.floor(n/64) % 0x20), 0x80 + (n % 0x40)) + else + -- [1110 xxxx] [10xx xxxx] [10xx xxxx] + x = string.char(0xE0 + (math.floor(n/4096) % 0x10), 0x80 + (math.floor(n/64) % 0x40), 0x80 + (n % 0x40)) + end + table.insert(t, x) + else + table.insert(t, escapeSequences[string.sub(s, i, j)]) + end + end + table.insert(t,string.sub(j, j+1)) + assert(string.find(s, startChar, j+1), "String decoding failed: missing closing " .. startChar .. " at position " .. j .. "(for string at position " .. startPos .. ")") + return table.concat(t,""), j+2 + -- END SoniEx2 +end + +--- Scans a JSON string skipping all whitespace from the current start position. +-- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. +-- @param s The string being scanned +-- @param startPos The starting position where we should begin removing whitespace. +-- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string +-- was reached. +function decode_scanWhitespace(s,startPos) + local whitespace=" \n\r\t" + local stringLen = string.len(s) + while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do + startPos = startPos + 1 + end + return startPos +end + +--- Encodes a string to be JSON-compatible. +-- This just involves back-quoting inverted commas, back-quotes and newlines, I think ;-) +-- @param s The string to return as a JSON encoded (i.e. backquoted string) +-- @return The string appropriately escaped. + +local escapeList = { + ['"'] = '\\"', + ['\\'] = '\\\\', + ['/'] = '\\/', + ['\b'] = '\\b', + ['\f'] = '\\f', + ['\n'] = '\\n', + ['\r'] = '\\r', + ['\t'] = '\\t' +} + +function json_private.encodeString(s) + local s = tostring(s) + return s:gsub(".", function(c) return escapeList[c] end) -- SoniEx2: 5.0 compat +end + +-- Determines whether the given Lua type is an array or a table / dictionary. +-- We consider any table an array if it has indexes 1..n for its n items, and no +-- other data in the table. +-- I think this method is currently a little 'flaky', but can't think of a good way around it yet... +-- @param t The table to evaluate as an array +-- @return boolean, number True if the table can be represented as an array, false otherwise. If true, +-- the second returned value is the maximum +-- number of indexed elements in the array. +function isArray(t) + -- Next we count all the elements, ensuring that any non-indexed elements are not-encodable + -- (with the possible exception of 'n') + local maxIndex = 0 + for k,v in pairs(t) do + if (type(k)=='number' and math.floor(k)==k and 1<=k) then -- k,v is an indexed pair + if (not isEncodable(v)) then return false end -- All array elements must be encodable + maxIndex = math.max(maxIndex,k) + else + if (k=='n') then + if v ~= table.getn(t) then return false end -- False if n does not hold the number of elements + else -- Else of (k=='n') + if isEncodable(v) then return false end + end -- End of (k~='n') + end -- End of k,v not an indexed pair + end -- End of loop across all pairs + return true, maxIndex +end + +--- Determines whether the given Lua object / table / variable can be JSON encoded. The only +-- types that are JSON encodable are: string, boolean, number, nil, table and json.null. +-- In this implementation, all other types are ignored. +-- @param o The object to examine. +-- @return boolean True if the object should be JSON encoded, false if it should be ignored. +function isEncodable(o) + local t = type(o) + return (t=='string' or t=='boolean' or t=='number' or t=='nil' or t=='table') or (t=='function' and o==null) +end + +return json \ No newline at end of file From 8e34093991bb1c4579c84fd0569b54cafc762889 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 21:05:00 +0200 Subject: [PATCH 018/103] lib.macaddress: Support numeric initialization; add method to get numeric representation. --- src/apps/intel/intel_app.lua | 4 ++-- src/lib/macaddress.lua | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index f104fad553..a4224cf0d0 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -64,7 +64,7 @@ function Intel82599:new (arg) if conf.vmdq then self.counters['phys-address'] = counter.open('phys-address') counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr).bits) + macaddress:new(conf.macaddr):int()) end return self @@ -102,7 +102,7 @@ function Intel82599:reconfig(arg) if conf.vmdq then counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr).bits) + macaddress:new(conf.macaddr):int()) end end diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index e003fa1cc7..e55bdcfb3c 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -13,17 +13,21 @@ function mac_mt:new (m) return m end local macobj = mac_t() - local i = 0; - for b in m:gmatch('%x%x') do - if i == 6 then - -- avoid out of bound array index + if type(m) == 'string' then + local i = 0; + for b in m:gmatch('%x%x') do + if i == 6 then + -- avoid out of bound array index + return nil, "malformed MAC address: " .. m + end + macobj.bytes[i] = tonumber(b, 16) + i = i + 1 + end + if i < 6 then return nil, "malformed MAC address: " .. m end - macobj.bytes[i] = tonumber(b, 16) - i = i + 1 - end - if i < 6 then - return nil, "malformed MAC address: " .. m + else + macobj.bits = m end return macobj end @@ -38,6 +42,10 @@ function mac_mt.__eq (a, b) return a.bits == b.bits end +function mac_mt:int () + return self.bits +end + function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From 5f9efd280f683e394c677e6d606b2d4624831d5f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 21:05:34 +0200 Subject: [PATCH 019/103] =?UTF-8?q?core.link:=20Create=20=E2=80=9Cdisconti?= =?UTF-8?q?nuity-time=E2=80=9D=20counters.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/link.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/link.lua b/src/core/link.lua index ed825e2606..34e2ffc942 100644 --- a/src/core/link.lua +++ b/src/core/link.lua @@ -28,6 +28,8 @@ function new (name) for _, c in ipairs(counternames) do r.stats[c] = counter.open("counters/"..name.."/"..c) end + counter.set(counter.open("counters/"..name.."/discontinuity-time"), + C.get_unix_time()) return r end @@ -35,6 +37,7 @@ function free (r, name) for _, c in ipairs(counternames) do counter.delete("counters/"..name.."/"..c) end + counter.delete("counters/"..name.."/discontinuity-time") shm.unmap(r) shm.unlink("links/"..name) end From 7b391482949cf10caac34871bb5890af80ab2f6b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 19 Apr 2016 17:44:30 +0200 Subject: [PATCH 020/103] snabb top: add `--yang' option to print YANG model as JSON. --- src/program/top/README | 2 + src/program/top/top.lua | 109 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index ce815d6f89..9fa3500804 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -5,6 +5,8 @@ Usage: Print usage information. -c, --counters Print counters of object by and exit. + -y, --yang + Print YANG model encoded in JSON and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index d69a999695..34ae80be6e 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -9,10 +9,12 @@ local shm = require("core.shm") local counter = require("core.counter") local S = require("syscall") local histogram = require("core.histogram") +local json = require("lib.json") +local macaddress = require("lib.macaddress") local usage = require("program.top.README_inc") local long_opts = { - help = "h", counters = "c" + help = "h", counters = "c", yang = "y" } function clearterm () io.write('\027[2J') end @@ -20,15 +22,18 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local object = nil + local yang = false function opt.h (arg) print(usage) main.exit(1) end function opt.c (arg) object = arg end - args = lib.dogetopt(args, opt, "hc:", long_opts) + function opt.y () yang = true end + args = lib.dogetopt(args, opt, "hc:y", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) - if object then list_counters(target_pid, object) - else top(target_pid) end + if object then list_counters(target_pid, object) + elseif yang then dump_yang(target_pid) + else top(target_pid) end ordered_exit(0) end @@ -55,18 +60,106 @@ function ordered_exit (value) os.exit(value) end +function read_counter (name, path) + if path then name = path.."/"..name end + local value = counter.read(counter.open(name, 'readonly')) + counter.delete(name) + return value +end + function list_counters (pid, object) local path = "//"..pid.."/counters/"..object local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do - local cpath = path.."/"..cname - local value = counter.read(counter.open(cpath, 'readonly')) - print_row({30, 30}, {cname, lib.comma_value(value)}) - counter.delete(cpath) + print_row({30, 30}, {cname, lib.comma_value(read_counter(cname, path))}) end end +function dump_yang (instance_pid) + local instance_tree = "//"..instance_pid + local interface_state = {} + local types = { [0x1000] = 'hardware', + [0x1001] = 'virtual', + [0x1002] = 'link' } + + for _, link in ipairs(shm.children(instance_tree.."/links")) do + local counters = instance_tree.."/counters/"..link + local statistics = {} + statistics['discontinuity-time'] = + totime(read_counter('discontinuity-time', counters)) + statistics['in-octets'] = + tohex64(read_counter('rxbytes', counters)) + statistics['out-octets'] = + tohex64(read_counter('txbytes', counters)) + statistics['out-discards'] = + truncate32(tonumber(read_counter('txdrop', counters))) + table.insert(interface_state, { name = link, + type = types[0x1002], + statistics = statistics }) + end + + for _, name in ipairs(shm.children(instance_tree.."/counters")) do + local counters = instance_tree.."/counters/"..name + local exists = {} + for _, c in ipairs(shm.children(counters)) do exists[c] = true end + local type = nil + if exists['type'] then + type = types[tonumber(read_counter('type', counters))] + end + if type then + local statistics = {} + statistics['discontinuity-time'] = + totime(read_counter('discontinuity-time', counters)) + for _, c in ipairs({'in-octets', 'in-unicast', + 'in-broadcast', 'in-multicast', + 'out-octets', 'out-unicast', + 'out-broadcast', 'out-multicast'}) do + if exists[c] then + statistics[c] = tohex64(read_counter(c, counters)) + end + end + for _, c in ipairs({'in-discards', 'out-discards'}) do + if exists[c] then + statistics[c] = truncate32(read_counter(c, counters)) + end + end + local interface = { name = name, type = type, statistics = statistics} + if exists['phys-address'] then + interface['phys-address'] = + tomac(read_counter('phys-address', counters)) + end + table.insert(interface_state, interface) + end + end + + print(json.encode({['interface-state'] = interface_state})) +end + +function tomac (n) + return tostring(macaddress:new(n)) +end + +function totime (n) + return os.date("!%FT%TZ", tonumber(n)) +end + +function truncate32 (n) + local box = ffi.new("union { uint64_t i64; uint32_t i32[2]; }") + box.i64 = n + if ffi.abi("le") then return tonumber(box.i32[0]) + elseif ffi.abi("be") then return tonumber(box.i32[1]) end +end + +function tohex64 (n) + local box = ffi.new("uint64_t[1]") + box[0] = n + local s = ffi.string(box, 8) + if ffi.abi("le") then s = s:reverse() end + return string.format("0x%02X%02X%02X%02X%02X%02X%02X%02X", s:byte(1, 8)) +end + + function top (instance_pid) local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) From 5d6baa48bd6da9fc80f492133c8b120bc23ab86a Mon Sep 17 00:00:00 2001 From: Pete Bristow Date: Wed, 27 Apr 2016 18:46:01 +0100 Subject: [PATCH 021/103] remove superfluous syscall.sysctl --- src/core/memory.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/memory.lua b/src/core/memory.lua index e76be61f37..046eaa3091 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -78,7 +78,6 @@ function reserve_new_page () local have = tonumber(lib.firstline("/proc/sys/vm/nr_hugepages")) local want = have + 1 lib.writefile("/proc/sys/vm/nr_hugepages", tostring(want)) - syscall.sysctl("vm.nr_hugepages", tostring(want)) io.write("[memory: Provisioned a huge page: sysctl vm.nr_hugepages ", have, " -> ", want, "]\n") end end From e457300569604dd9a9ab3163849aaa8dae753f37 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 17:29:52 +0200 Subject: [PATCH 022/103] Update CONTRIBUTING.md. --- CONTRIBUTING.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77914f29b2..bd4a20d245 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,18 +1,30 @@ -Whenever you create a Pull Request (PR), a maintainer will self-assign +# Hints for Contributors + +* Whenever you create a Pull Request (PR), a maintainer will self-assign themselves as the *upstream*. The upstream decides if your PR is accepted and might require you to amend additional changes before merging. You can expect the upstream to communicate clearly if there are any issues preventing your PR from being merged, and how they can be fixed. Once merged, the upstream will -add the `merged` label to your PR. +add the `merged` label to your PR. More on [our Git workflow](https://github.com/snabbco/snabb/blob/master/src/doc/git-workflow.md). + +* Target your PR against the branch you would like it to be merged into. Refer +to the [list of subsystem branches](https://github.com/snabbco/snabb/blob/master/src/doc/branches.md). +When in doubt use `master`. -You are also welcome to submit PRs you would like to receive feedback on, but +* Feel free to @ping the assignee if you feel like your PR has been overlooked, +and are waiting for a response. + +* You are also welcome to submit PRs you would like to receive feedback on, but which are not ready to be merged: include the labels `[wip]` in the title of PRs that require further work, and `[sketch]` for PRs that are not meant to be merged at all. -Our [Documentation Guide](https://github.com/SnabbCo/snabbswitch/blob/master/src/doc/documentation-guide.md) +* Please make sure your editor is configured to not emit tabs and use three +spaces for indentation. + +* Our [Documentation Guide](https://github.com/SnabbCo/snabbswitch/blob/master/src/doc/documentation-guide.md) gives pointers on how to contribute to the project's documentation. -If you wish to record a copyright notice with your contribution then you can +* If you wish to record a copyright notice with your contribution then you can optionally do this in the file `src/COPYRIGHT`; copyright notices in other files will be rejected. From c3c69d240b1c9d33722584116c7a15e3c0995d5a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 29 Apr 2016 10:46:21 +0200 Subject: [PATCH 023/103] Add 'start' method to apps --- src/README.md | 8 ++++++++ src/core/app.lua | 1 + 2 files changed, 9 insertions(+) diff --git a/src/README.md b/src/README.md index ea12be3741..1d4dfcc9e4 100644 --- a/src/README.md +++ b/src/README.md @@ -126,6 +126,14 @@ implemented the app instance is discarded and a new instance is created. *Optional*. Print a report of the current app status. +— Method **myapp:start** + +*Optional*. Starts the app. + +At this point links and app name are set. The app has a chance to do +additional initialization if needed. + + — Method **myapp:stop** *Optional*. Stop the app and release associated external resources. diff --git a/src/core/app.lua b/src/core/app.lua index cf492a593f..e1048dc012 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -178,6 +178,7 @@ function apply_config_actions (actions, conf) table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array app.zone = zone + if app_table[name].start then app_table[name]:start() end end function ops.restart (name) ops.stop(name) From 9dabdd2fc34cca0904de18f68beb2b1b120e750a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 29 Apr 2016 11:37:34 +0200 Subject: [PATCH 024/103] Call app.start instead of app_table[name].start --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index e1048dc012..ccf163193d 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -178,7 +178,7 @@ function apply_config_actions (actions, conf) table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array app.zone = zone - if app_table[name].start then app_table[name]:start() end + if app.start then app:start() end end function ops.restart (name) ops.stop(name) From 6c46b5d5d4967f9c20e3f057d5f27a69ea22eef8 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 2 May 2016 04:30:39 +0000 Subject: [PATCH 025/103] doc: Remove stale .images/Branches.png file Ditaa images are not kept in tree anymore... --- src/doc/.images/Branches.png | Bin 9218 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/doc/.images/Branches.png diff --git a/src/doc/.images/Branches.png b/src/doc/.images/Branches.png deleted file mode 100644 index 7793578f6932b5113ec39ecebeef57d30daacc0c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9218 zcmaia2UJsA*DWf74NwsgA)ulnA|M??iHM36X-X$VrAjXeNLTQJw4flp1rShrF9}7Z z2}lUNgMdIF^w8_u!F#`Z-+TA}{xKLCC+F<5v-jGo%sChSYAW({CoZ0#qN1Wxcz92P zit6wNxNbjo7<}dtIa{cxI2#r2$v%16KR4v2{KRap!Jg?CeZZ6QnC_xronnLj3>=?% z{-IQhN4Eph=F+bBJ7vOiV?}IQY~ZPu!OR~mG?9_r7ES_>Wbb9%^0(sUMTS50*0!?L z4eh7Nc*zsFxm(q|9!Zvztchc}ap~vwDsA*soRFDm!qli#qL8C$!lC^mR8$02It*ov zvb0oGm{uAHD;3q9>N|u(R8-UuaNC>y&u6bNVDQ2HOT_^-j6c)7!+Lo%OuC3 zl$(D~b@0kxgNYA1(jKwie)`3SL8jN{3}sp15%?pJd{lIGbv->j?d|PbVPONMwjFGg z$5;VkFB1jxLIJ^u*-=fj@=4PRVioCqr+}zwZ zZhRjYaNF5f!a2=3xEmbT%+JqHNJyY%VE-sV+3<=Qq1%Hty< zOoug=gZ=#EpFH`H6JCvQU;4x5U3|PWCO9|O*3K?(*s$Cob#s1iZ?8a08zOG^YVu&! za0(&AMNkdshrOJhD zYH3MoPgP14Hz#14Bl%+&WIVQ3PBQa#c6Pe&uJ#Di0^_}6KLPE@frlAfsoU7tVB*m- zHZwz1ylk6C%gD%_eT8i7{QTHN;o-x#gM55^-E2apKcZe(kByBX8>>9FgK!LN;&~+{ z5n7Z*E_x-Wq|hQGjL)q5pnJ>gQz9cHT^K@N{?PaJ^JB%uNIK`(I8Nc?<^1R++~{LP ztp(?0p4T3`cn)~Nm;Eqz!XIvqo2$zU)%!9q7~*Ven5xWYh18rJd~bV42f03&6+6MG zD)p=)#c{0m_>j1eP)B^j(St3}LFwq|^h<5JbU2;n2S4|vVzK?E*zc3YJ%*Xt*&=+> zCAbG>ouAZQ)m0upehd!Y!Xjff0&@ho%ujY&h`)FY{P5w!D;$u+f}g$1%*bAe&RLGZkxffrFJF$AqLrWELJb~qi(4U9Ci{4v9J%s zt@NIZq^&Hi%o)Pxd2FxAv%B2$LSB-VuFU12-Fj}wMWnI|3^H^*UF|Rtuo{t?n#v%( zqQ>vSk-APD?>RWu?K~Zwq%PPky?{c;kCIzqVmYcD(ym2IDT>q(Z~9j<60;&ABH4zO zdS;@Rh^LWHoK+Y3Ds8$lG%J&OB%FLJ-PUJ9jT+FaA%?PqLr-G`O#;}DsU%3TFfxvl zNU@=z8v6SBnwsJ7-u;}KVrFL6*3+}F;`jU37{*0%?{VlCvmdLCi(`Xps;Ob*8-s#Q zahDVoDaQya1iXL$e$F%IDVvYX-p2UG#!C*4MEmi2CMKrnQhQNY$*R$`>OgcbM6!G7 z1_vvvo(`yu>5Z?6E%5&kehzP1T z?^*j;yH0NP9+ofiX87(yh(&%yg~56SFd-hr8|&7QR63v^S6p`A8Rf*>kkpp!IXGQo zYSx-CS7kWm@XdvUzm#pHec85p*Qu0UVDR&u$e|xCyNt+J+tNg88k#17Z{lAXzuu*1 zdmwQ2>c{+i^-{0B%_-E0s}csTFJBUAINTRs=fLxxzE&}VKYkn*9sSudgk9Rg-XP+u z+#083XYj=7ogwFuX_{`bk-k(!GbSuN+(hd6K0VZHkDS<&tL}{{40~BPmEfk*(-f5i zi!d7YaiI=XZ-ElaMv4cc>4$V}?Yqa$4|TO*jNmZ7?YzG3Y<*mDPkEf(Fjfe8gNW=p zQ)6-zJ8L>Wn@EX;J)^ed`{Ci?jMtS3$#HizpFFvxTW~MX0c~Cp?l3&Pr>$Gmd5TICD1|!K8E_8Vo7$?Ydn30&LYU^k3ar! zaHeAmm7-^t(m@~)aQJSwzHPMpCI6G%vJEXc6MIPF+}e3FUYj8}+%N@fZ(fQgrAM0l zLuhD-l*JY2+{a>-hG^JZ=}>i>d^=ij815wGyrxPxbPT#LF^BWq#h#n6OKEsd$)V5g zGsKV058)gls$VXRt*opR6$$U&hf72YIq5ukV$4=Mv5|}`7UyGNm)Zap6zyO-!YXQ& zS1v$t!D30@#z5DfuW9329`u3z#Wj(Xw&!fUGL>X@7Ujsjrpmgj^{lo;8WR%}tnudZ z!~@2MlXipUu?O3jL=y3?fraNw=X3ac(5T%qa#m5#3tv(+X58wVOJsZ!Nc)-S3w%5T z6ER1#eQF3MWV*jEdo_>g%$cM*5Gz9(_^y?YYJm=ECYCs~AUAiTeW^pY<*b@YbC6k? zY0JgP1`~-R9nT*<3LaMS<>1D~aSUwb_d{xaq3iEsDkGNGwpDFMIOWmAW#?fR`N#`& z8Ah?{jSE%VIW~(eVk5**X2X|Xbu3TmotJHRNaghRs+DlO^94BUDQNfVIKHsmaB9O} zQ+szGx+fmy}j+7|}yPZLS}!Z*5%wg{!;4WIV|vPk>|G03Tw#tAm%phH49gTsXXg zR$&%Y85?mYKY~B6upe87#G5on=vJpNa&XvIZ8UzTMl*BbI*OrC2o`SpqO=a~)RVZ3 zMt6otwl)z_PCb{LMl8v!+7)A(y zVoB=BEiEb8TaK(oIt8oTzkeSNkGDV6(Uu|)$!#mQ8~q0A8fvQ0thHMS0SuwUqRXWc zxAE9pMLu25ne7Zjf6&C)x-?82_n;x)bC3brPtwZP>k{fA7Dn*#;BFvaCBK zo9;G{eW<+Afi>NwBc811&RxCF5;plGTF8CF4hDmb=Uwr`*Vf*HFd3@}qh({oY(Yq4 z>vnqiI7pz<3NUcxRxAe$r8GBU4NH4Ne9$c`_gPM2I84Tehp)~SwqEpO8sFJi9Ie4FRk)B>c3+XFTAi;Dd*fO|=I}<1qxv*V z5J`~Kv**M%f&(vXSrXb3RagX`zv=w3GTjk;&wLLO58U~o*s){B2na*nYR}ye*q%9i zY-+!cj}L%mw^qxI8_&Xr+uIAA=K9~hJtpB7VIiDjHHReRql}>|PHurlf%-^oGiQ6Y zi@Y=GDYEiz@oQc4+YpEuBOW(<3eJu+5PjWz0@>FBE%NfFw}a6@D)#rbw~~%)zGim> z=!3R1A!(tI3O+y6l?fjpQ9vtzz?hLMpp;TTBKSf9t_OFi01%>J zT3K2OHm0E8|2hCbE&xLS00I}kKf#39hPAl?4wqkN`1$!+LckeW|M;u~VEAd|mR~Du z-~CSPt5>gp_vPy3dt>B&Kkx@N+=3wCG#izlmz8x32CFVBi`u#j)^H5!p{c0Y^lE=? zZNPS5ZyOR1{2U(s{{7{;h=>SGI8+ ziV?~QQWCdstJ&N#C{xoG5f!Bea4X^}7uQIIOJQ*_YPvmDgH70+0Kz#lBLg`#F)>kK z@90Q2OZ13{js0yYg1$HU#?708!>jKndU6px6LocUSi2ul0v-Leb#>FB03v|xJvtjQ zhz|P*(1Bx_%#RvdzD4nRy;3tHE z7?BwV4hCec8d}eBzR_M2}x4l3nldmmFZT4X*48kw># zA>h?d7t?Y;(8c3#aC2|U#cpkF@r{(vl2aoMv9`1e^j+2?8JRCS(i-;wO8cskRSDZS z0cEGHcy6F(%Q^Nib^M+(y5~s00pM!Q4L*MCuAeOZ2CA5lgSi7T+S==mgz4I9{?n{5s8Ti1A`v|jF*VWVD86hL^b>6UyV^w7Z8Y%4QpK{ zd-hC2LqkpNU1{n1#?rX+T>pf4P@_7(sHhhG7|55x-3iieTIXZd`(kNr$BW%}mKv6p zmO#oLuMg${F=9KOb@RlD6Dvnejg24Xr>2^;e^k&23JS`~%`N>jAfu;!Ufr}VfN5)I z#FfNL?r)XZR>mfG>LZazL&JpFq2eKWQ;$f)t}Ne~nVEUZ5%xX#+ofeFCV|6;sN?CW zDk0FnI$$uH(S5jBv!(R-_^Ln zO{E-NUW|CG$J;!GGo3tti=YW;PjqxNc8%DU(tWugq&vJ2Q~0cg#$(yGYBSBn`YG#p zszSr|SRlWFleQV;#};2@x-K2SFGKxR(mNqfnXP#tvabi}zKUFl}CMRDR4{y4SnKKc%vNnXZSf`_!)63M#b}%3T6-VH1BvGFA^5x5X zI7=Ml#{_)dr7&h-{WUE+@%7qVqZhTlTMu0K$r*s0aI#!iuG|e|VPSE-^EoT)!&tPC zS|vpyY#|wPzNgx7wu5Er4bABvtX)ft$Ygw}mgO7I50+z>K!pR&)j&VF%(q^l+cIZq;5wLn^#k(RwqB18PPQTW zPxIG5t2+jDNV`_ok**r+ga7IEl($397QeXr;Ld&9tt*j%TU&!tJjmG)pYZE?a zvcHr!1NbaOJ|u$m(4j+sjiZ6UF>u6bfmk-1gNFQwXZLK4?Ha5C`GKTAX9+i&6Aie} zbO@v|dqQJ#b4Ri#!$f9o)$;nfnD1*}Utdt4EV9l>I?oF)SO6rgck9Lt0$mW#g$KbG zQzLJE-$ec?*ECJ;G}n~cju;kOXnLK1@@3vVZ=umB;DwMF0had_&4C?>iu`)}UaY2wNK{%v=AokYvAtUdqSee-^` zq{S`|_qA)+T*IHkjzPOM+xq)sqM|+}wmodS5F}BPkDt>u7y?&fB{>I+fFx#Wm#81o#La1LXGR|>NYnv zzAIR0X>}L2%2=K{;;^?hlbKszTT4w#>*nIp_qtlmqQ+VVb>pFUC3$qC0SXspVJfCN7^=tDIq_Ox9U2hy1!;Ev=799L$6;| z*u73Hdwd_%*xivNIlq-7hcm_!I+;X9+$bQqh#z2L7;<$+Gje1^R;-KZ$g^CBi6*9g zW&oR>Ub>fPi@-5_i#kn_1gRmQ;vW=!3M2qv_%zZSvCe`ZOSvo?D=P;9y;MY4GC3=S@C*nwK*Raxe}z>^ixbp6IE^v6bJz-u(T$xrIf6KsFT2cjU4; zA`ruP;lho^nhn8py5f2n*bp`%zilir*D%bjEW zrMno;^SpXb0J?qljheTg^Q&cR?tCSvUgNH9zO>SP6^Cj9cMMOq64?Wh< zK>B?9c9*py!v}NZ_G#}m)Onb21+{30&QUzha%XU&AXbB zKe3WOR4&!CVm)+qqxp2J+{^i%>7jsKXzs4K)Fb)ef^OwB@qoziUWC;d>gsaW37O%e zyaI+MRu%whZz(6i@lyNqxP=)SK$nC0Tt@$3Mjk&{!)fFzCvSADBfczdZ|DjUi3GnV z)GAq(QfSpD2>(_zb8re6vTVb_TCjQ3Ae?!23Axji+^Hy+jN&G?x8DS65g0#HxUlsl;E|X^Bu;7wPz$ zv-Nqoy1L4f=nj@W$SU}R!Q2r92#HroNr}Uim^8e@t}iCiF_q|*qj~Q1>D#U=L?GHD zKMjLWc{R+AY-(y+CX+!aU1-I0baocGu3D(7stzrCo`kxs&v!9<&i3Y`X=*{+gN!%@ zUU55(l-c-s1oBBGQHHk;aAh4(j151GjqI(7@g_h~P4?6x0S>wUlwVmSZLO{0K!FDE ztGXJjcME8_DBA);Z(3g7;&gf<{fQIHYipjf`3-`Cf>%{_LPqfHD(ev+K74dzg?aAe zz;Q?yS=rYzMYD(iz*|6S34LyCZ0zB&3-o<~pVbsicfKt%Gt zW%<_ulgtGS7UajtU%!YlAgE?Dh;yYQPFlA0jPI0hgqhxrIKMv@PI-!`6 zcM@73OKsXJy&hM$VZCJ6*Vl(7^z?}R#Y}tnbn@F>Gg6j|ho|{v zEjV>(XB7pco|_0)6b)DG=Dl_k#YcSD@z9#I3=qxYU#5R!Xuq6^*i~#Oe0_vgajB?tWJvFR(gAGlDto{&|EB6S1(^rN`@#$rK10grZ$a-4V@ z8}cG6E6`7`)GaP9E-x=*u}~vn-dnejqMnj&;(Y^zB)Qb6xHxz6pjP|6c~e_kJ6P^0 zUyb1OiM~%@k$RbDd{KYEPZC;T-J2&HKtY;CWx!d7K<{US4XF|*o>V6`{8CQrozzFA z2!6!8)sEAe9U$U*lpa)(C_n{R1 zDb(aI&=WJdcVc~{O}E4@YzEIYnepCTe?$AG3m&p9G8}s2AYMS;ppp2 z!`*Zj(67Fydpn!1y2!hC?qb)3afunC{-np7&K**PHC*1_F-F9bM)vKM(X<>No!X;CD9N-}hGjNnvkQ50y0iu0+gv5kbl_5dkP>P{I1~DA5+6Un?1PS-)7Q{A%gj7AJ}zX@ z`6(+a3kX=lTLxL|9K$IwaCh!4U&{? zm>d>?)>BVXkX!L1$246tU@#}v3qo;;lFs<&qT4`gr=g`Ksd7L-+N-Ln!eY0wONW!m zAjGKFjR^0bCUq17Dxfuf*TyDCqQRVcx*Ib-KHf1nn56_LWX8hPXtCR^l>Jr;NbZ8h z$n25}rSJ;A;b!4h^mE12==X_xVOI(Rbk zdT(Wl(H<1LYPD98La{AINS3lc*U`mDgeP}-ZItn_E`X}_4&!IzdJNsNOG-p3s*6(( zG0F!G-a_{f4a)4E0Xirw1{~uKA^!vvq~yQ)2OAeZ6k^8lK@)nQl> Date: Wed, 20 Apr 2016 13:43:36 +0200 Subject: [PATCH 026/103] snabb top --yang: Represent uint64_t as decimal string. --- src/program/top/top.lua | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 34ae80be6e..317c0a2a77 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -89,9 +89,9 @@ function dump_yang (instance_pid) statistics['discontinuity-time'] = totime(read_counter('discontinuity-time', counters)) statistics['in-octets'] = - tohex64(read_counter('rxbytes', counters)) + tostring64(read_counter('rxbytes', counters)) statistics['out-octets'] = - tohex64(read_counter('txbytes', counters)) + tostring64(read_counter('txbytes', counters)) statistics['out-discards'] = truncate32(tonumber(read_counter('txdrop', counters))) table.insert(interface_state, { name = link, @@ -116,7 +116,7 @@ function dump_yang (instance_pid) 'out-octets', 'out-unicast', 'out-broadcast', 'out-multicast'}) do if exists[c] then - statistics[c] = tohex64(read_counter(c, counters)) + statistics[c] = tostring64(read_counter(c, counters)) end end for _, c in ipairs({'in-discards', 'out-discards'}) do @@ -151,12 +151,10 @@ function truncate32 (n) elseif ffi.abi("be") then return tonumber(box.i32[1]) end end -function tohex64 (n) - local box = ffi.new("uint64_t[1]") - box[0] = n - local s = ffi.string(box, 8) - if ffi.abi("le") then s = s:reverse() end - return string.format("0x%02X%02X%02X%02X%02X%02X%02X%02X", s:byte(1, 8)) +function tostring64 (n) + assert(ffi.istype('uint64_t', n)) + local s = tostring(n) + return s:sub(1, #s - 3) end From ee00d167b2aa437861f5ee04804e80755053ef36 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 18:26:55 +0200 Subject: [PATCH 027/103] [core.lib] Generalize `timer' to optionally accept 'repeating' option and support injecting a function to determine the current time. --- src/README.md | 14 +++++++++++--- src/core/lib.lua | 21 +++++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/README.md b/src/README.md index b406eeabd9..4cfbb73dd7 100644 --- a/src/README.md +++ b/src/README.md @@ -543,10 +543,18 @@ Returns a table that acts as a bounds checked wrapper around a C array of ctype and the caller must ensure that the allocated memory region at *base*/*offset* is at least `sizeof(type)*size` bytes long. -— Function **lib.timer** *s* +— Function **lib.timer** *duration*, *mode*, *timefun* + +Returns a closure that will return `false` until *duration* has elapsed. If +*mode* is `'repeating'` the timer will reset itself after returning `true`, +thus implementing an interval timer. *Timefun* is used to get a monotonic time. +*Timefun* defaults to `C.get_time_ns`. + +The “deadline” for a given *duration* is computed by adding *duration* to the +result of calling *timefun*, and is saved in the resulting closure. A +*duration* has elapsed when its deadline is less than or equal the value +obtained using *timefun* when calling the closure. -Returns a function that accepts no parameters and acts as a predicate to -test if *ns* nanoseconds have elapsed. — Function **lib.waitfor** *condition* diff --git a/src/core/lib.lua b/src/core/lib.lua index b6223cc8e6..6e37a90964 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -246,10 +246,23 @@ function bounds_checked (type, base, offset, size) return wrap(ffi.cast(tptr, ffi.cast("uint8_t *", base) + offset)) end --- Return a function that will return false until NS nanoseconds have elapsed. -function timer (ns) - local deadline = C.get_time_ns() + ns - return function () return C.get_time_ns() >= deadline end +-- Return a function that will return false until duration has elapsed. +-- If mode is 'repeating' the timer will reset itself after returning true, +-- thus implementing an interval timer. Timefun defaults to `C.get_time_ns'. +function timer (duration, mode, timefun) + timefun = timefun or C.get_time_ns + local deadline = timefun() + duration + local function oneshot () + return timefun() >= deadline + end + local function repeating () + if timefun() >= deadline then + deadline = deadline + duration + return true + else return false end + end + if mode == 'repeating' then return repeating + else return oneshot end end -- Loop until the function `condition` returns true. From 45490b8e162c96e16f739672c951d9b92f896fd5 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 19:30:55 +0200 Subject: [PATCH 028/103] Revert "Intel_app: Add RFC 7223 app counters." This reverts commit 8bb3215bf759f6c53af07c5c13afd2ea60003334. --- src/apps/intel/intel_app.lua | 45 ++++-------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index a4224cf0d0..80cace3d9f 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,10 +6,8 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") -local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") -local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local freelist = require("core.freelist") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty @@ -38,7 +36,6 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) - local self = setmetatable({counters = {}}, Intel82599) if conf.vmdq then if devices[conf.pciaddr] == nil then @@ -48,26 +45,12 @@ function Intel82599:new (arg) local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - self.dev = vf:open(conf) + return setmetatable({dev=vf:open(conf)}, Intel82599) else - self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") - self.zone = "intel" + local dev = intel10g.new_sf(conf):open() + if not dev then return null end + return setmetatable({dev=dev, zone="intel"}, Intel82599) end - - self.counters['type'] = counter.open('type') - self.counters['discontinuity-time'] = counter.open('discontinuity-time') - self.counters['in-octets'] = counter.open('in-octets') - self.counters['in-discards'] = counter.open('in-discards') - self.counters['out-octets'] = counter.open('out-octets') - counter.set(self.counters['type'], 0x1000) -- Hardware interface - counter.set(self.counters['discontinuity-time'], C.get_unix_time()) - if conf.vmdq then - self.counters['phys-address'] = counter.open('phys-address') - counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr):int()) - end - - return self end function Intel82599:stop() @@ -88,9 +71,6 @@ function Intel82599:stop() if close_pf then close_pf:close() end - - -- delete counters - for name, _ in pairs(self.counters) do counter.delete(name) end end @@ -99,11 +79,6 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) - - if conf.vmdq then - counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr):int()) - end end -- Allocate receive buffers from the given freelist. @@ -121,13 +96,6 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() - if self.dev.rxstats then - counter.set(self.counters['in-octets'], - bit.lshift(self.dev.pf.qs.QBRC_H[self.dev.rxstats]()+0LL, 32) - + self.dev.pf.qs.QBRC_L[self.dev.rxstats]()) - counter.set(self.counters['in-discards'], - self.dev.pf.qs.QPRDC[self.dev.rxstats]()) - end end function Intel82599:add_receive_buffers () @@ -148,11 +116,6 @@ function Intel82599:push () end end self.dev:sync_transmit() - if self.dev.txstats then - counter.set(self.counters['out-octets'], - bit.lshift(self.dev.pf.qs.QBTC_H[self.dev.txstats]()+0LL, 32) - + self.dev.pf.qs.QBTC_L[self.dev.txstats]()) - end end -- Report on relevant status and statistics. From f0ed10b70781505e38809acd189744067993f516 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 20:34:23 +0200 Subject: [PATCH 029/103] intel_app: expose per-pciaddress statistics in `counters/'. --- src/apps/intel/intel_app.lua | 69 +++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 80cace3d9f..0fff2dfc5e 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,6 +6,8 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") +local shm = require("core.shm") +local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") local intel10g = require("apps.intel.intel10g") @@ -36,21 +38,49 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) + local self = {} if conf.vmdq then if devices[conf.pciaddr] == nil then - devices[conf.pciaddr] = {pf=intel10g.new_pf(conf):open(), vflist={}} + local pf = intel10g.new_pf(conf):open() + devices[conf.pciaddr] = {pf=pf, vflist={}, stats={register=pf.s}} end local dev = devices[conf.pciaddr] local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - return setmetatable({dev=vf:open(conf)}, Intel82599) + self.dev = vf:open(conf) + self.stats = devices[conf.pciaddr].stats else - local dev = intel10g.new_sf(conf):open() - if not dev then return null end - return setmetatable({dev=dev, zone="intel"}, Intel82599) + self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") + self.stats = { register = self.dev.s } + self.zone = "intel" end + if not self.stats.counters then + local counters = {} + local path = "/counters/"..conf.pciaddr.."/" + counters['type'] = counter.open(path..'type') + counters['discontinuity-time'] = counter.open(path..'discontinuity-time') + counters['in-octets'] = counter.open(path..'in-octets') + counters['in-multicast'] = counter.open(path..'in-multicast') + counters['in-broadcast'] = counter.open(path..'in-broadcast') + counters['in-discards'] = counter.open(path..'in-discards') + counters['out-octets'] = counter.open(path..'out-octets') + counters['out-multicast'] = counter.open(path..'out-multicast') + counters['out-broadcast'] = counter.open(path..'out-broadcast') + counters['out-discards'] = counter.open(path..'out-discards') + counter.set(counters['type'], 0x1000) -- Hardware interface + counter.set(counters['discontinuity-time'], C.get_unix_time()) + if not conf.vmdq and conf.macaddr then + counters['phys-address'] = counter.open(path..'phys-address') + counter.set(counters['phys-address'], + macaddress:new(conf.macaddr):int()) + end + self.stats.counters = counters + self.stats.path = path + self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) + end + return setmetatable(self, Intel82599) end function Intel82599:stop() @@ -71,6 +101,12 @@ function Intel82599:stop() if close_pf then close_pf:close() end + if not self.dev.pf or close_pf then + for name, _ in pairs(self.stats.counters) do + counter.delete(self.stats.path..name) + end + shm.unlink(self.stats.path) + end end @@ -79,6 +115,11 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) + + if not self.dev.pf and conf.macaddr then + counter.set(self.stats.counters['phys-address'], + macaddress:new(conf.macaddr):int()) + end end -- Allocate receive buffers from the given freelist. @@ -96,6 +137,9 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() + if self.stats.sync_timer() then + self:sync_stats() + end end function Intel82599:add_receive_buffers () @@ -105,6 +149,21 @@ function Intel82599:add_receive_buffers () end end +-- Synchronize self.stats.register a and self.stats.counters. +function Intel82599:sync_stats () + --- XXX - it is questionable if I choose the right register to counter + --- mapping + local counters, register = self.stats.counters, self.stats.register + counter.set(counters['in-octets'], register.GORC64()) + counter.set(counters['in-multicast'], register.MPRC()) + counter.set(counters['in-broadcast'], register.BPRC()) + counter.set(counters['in-discards'], register.TPR() - register.GPRC()) + counter.set(counters['out-octets'], register.GOTC64()) + counter.set(counters['out-multicast'], register.MPTC()) + counter.set(counters['out-broadcast'], register.BPTC()) + counter.set(counters['out-discards'], register.TPT() - register.GPTC()) +end + -- Push packets from our 'rx' link onto the network. function Intel82599:push () local l = self.input.rx From b85326857b768eecf90c2a73dcd17709b9100b85 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 5 May 2016 14:28:37 +0200 Subject: [PATCH 030/103] core.packet: document ctype and deprecate data and length functions. --- src/README.md | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/README.md b/src/README.md index ea12be3741..ff2107e9e1 100644 --- a/src/README.md +++ b/src/README.md @@ -303,23 +303,37 @@ Returns a structure holding ring statistics for the *link*: ## Packet (core.packet) - -A *packet* is a data structure describing one of the network packets that -is currently being processed. The packet is used to explicitly manage the -life cycle of the packet. Packets are explicitly allocated and freed by -using `packet.allocate` and `packet.free`. When a packet is received -using `link.receive` its ownership is acquired by the calling app. The -app must then ensure to either transfer the packet ownership to another -app by calling `link.transmit` on the packet or free the packet using -`packet.free`. Apps may only use packets they own, e.g. packets that have -not been transmitted or freed. The number of allocatable packets is -limited by the size of the underlying "freelist", e.g. a pool of unused -packet objects from and to which packets are allocated and freed. + +A *packet* is an FFI object of type `packet.packet_t` representing a network +packet that is currently being processed. The packet is used to explicitly +manage the life cycle of the packet. Packets are explicitly allocated and freed +by using `packet.allocate` and `packet.free`. When a packet is received using +`link.receive` its ownership is acquired by the calling app. The app must then +ensure to either transfer the packet ownership to another app by calling +`link.transmit` on the packet or free the packet using `packet.free`. Apps may +only use packets they own, e.g. packets that have not been transmitted or +freed. The number of allocatable packets is limited by the size of the +underlying “freelist”, e.g. a pool of unused packet objects from and to which +packets are allocated and freed. + +— Ctype **packet.packet_t** + +``` +struct packet { + uint8_t data[packet.max_payload]; + uint16_t length; +}; +``` + +— Constant **packet.max_payload** + +The maximum payload length of a packet. — Function **packet.allocate** -Returns a new empty packet. An an error is raised if there are no packets -left on the freelist. +Returns a new empty packet. An an error is raised if there are no packets left +on the freelist. Initially the `length` of the allocated is 0, and its `data` +is uninitialized garbage. — Function **packet.free** *packet* @@ -327,11 +341,13 @@ Frees *packet* and puts in back onto the freelist. — Function **packet.data** *packet* -Returns a pointer to the payload of *packet*. +Returns a pointer to the payload of *packet*. **Deprecated, please use the +`data` field instead.** — Function **packet.length** *packet* -Returns the payload length of *packet*. +Returns the payload length of *packet*. **Deprecated, please use the `length` +field instead.** — Function **packet.clone** *packet* From b11e375c3da919013e3e6e1b0cfa7d01cd67b6c7 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 5 May 2016 14:29:09 +0200 Subject: [PATCH 031/103] core.packet: properly document shiftleft and shiftight. --- src/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/README.md b/src/README.md index ff2107e9e1..011d53ea34 100644 --- a/src/README.md +++ b/src/README.md @@ -367,7 +367,14 @@ accomodate *length* additional bytes. — Function **packet.shiftleft** *packet*, *length* -Truncates *packet* by *length* bytes from the front. +Truncates *packet* by *length* bytes from the front. *Length* must be less than +or equal to `length` of *packet*. + +— Function **packet.shiftright** *packet*, *length* + +Move *packet* payload to the right by *length* bytes, growing *packet* by +*length*. The sum of *length* and `length` of *packet* must be less than or +equal to `packet.max_payload`. — Function **packet.from_pointer** *pointer*, *length* From d0cc3c6575f63249cc929a4c42f4463a7f68a7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 9 May 2016 15:16:37 +0100 Subject: [PATCH 032/103] default.nix: enable parallel builds This essentially passes -j {cores} to make: $ nix-build --cores 4 --- default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/default.nix b/default.nix index a3932d5eab..a71304ed3a 100644 --- a/default.nix +++ b/default.nix @@ -36,4 +36,7 @@ stdenv.mkDerivation rec { mkdir -p $out/bin cp src/snabb $out/bin ''; + + enableParallelBuilding = true; + } From c186591e312fb90c4c2d847be01255fb7dba0c8b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 25 Apr 2016 16:25:06 +0200 Subject: [PATCH 033/103] lib.protocol.ethernet: Add n_mcast, branch-free Multicast predicate. --- src/apps/vhost/vhost_user.lua | 16 ++++++---------- src/lib/protocol/README.md | 9 +++++++++ src/lib/protocol/ethernet.lua | 7 ++++++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 96e82780eb..16936709bf 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -102,20 +102,16 @@ end function VhostUser:tx_callback (p) counter.add(self.counters['out-octets'], packet.length(p)) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters['out-multicast']) - else - counter.add(self.counters['out-unicast']) - end + local mcast = ethernet:n_mcast(packet.data(p)) + counter.add(self.counters['out-multicast'], mcast) + counter.add(self.counters['out-unicast'], 1 - mcast) end function VhostUser:rx_callback (p) counter.add(self.counters['in-octets'], packet.length(p)) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters['in-multicast']) - else - counter.add(self.counters['in-unicast']) - end + local mcast = ethernet:n_mcast(packet.data(p)) + counter.add(self.counters['in-multicast'], mcast) + counter.add(self.counters['in-unicast'], 1 - mcast) end function VhostUser:rxdrop_callback (p) diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 6f239687d7..4a13483a4b 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -98,6 +98,15 @@ Returns the binary representation of MAC address denoted by *string*. Returns the string representation of *mac* address. +— Function **ethernet:is_mcast** *mac* + +Returns a true value if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet). + +— Function **ethernet:n_mcast** *mac* + +Returns 1 if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet) +and 0 otherwise. + — Function **ethernet:ipv6_mcast** *ip* Returns the MAC address for IPv6 multicast *ip* as defined by RFC2464, diff --git a/src/lib/protocol/ethernet.lua b/src/lib/protocol/ethernet.lua index 0a55c20f1b..b5abed8d31 100644 --- a/src/lib/protocol/ethernet.lua +++ b/src/lib/protocol/ethernet.lua @@ -76,9 +76,14 @@ function ethernet:ipv6_mcast(ip) return result end +-- Return 1 if MAC address has its group bit set and 0 otherwise +function ethernet:n_mcast (addr) + return band(addr[0], 0x01) +end + -- Check whether a MAC address has its group bit set function ethernet:is_mcast (addr) - return band(addr[0], 0x01) ~= 0 + return ethernet:n_mcast(addr) ~= 0 end -- Instance methods From b09e843465686fd1afad4c6c48a248f33de6b07e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 27 May 2016 15:24:11 +0200 Subject: [PATCH 034/103] Fix for f0ed10b: require macaddress module. --- src/apps/intel/intel_app.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index f5773d66f3..2645e2d22c 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -10,6 +10,7 @@ local shm = require("core.shm") local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") +local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty Intel82599 = {} From 4eb21b61325b8c2c255fa3390d46d1a0675f36f0 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sat, 28 May 2016 18:23:42 +0000 Subject: [PATCH 035/103] virtio/net_device: Fix bug with mergeable rx buffers Snabb could over-report the size of jumbo packets delivered to the VM under certain circumstances. --- src/lib/virtio/net_device.lua | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lib/virtio/net_device.lua b/src/lib/virtio/net_device.lua index 9c93e66fbb..be06069833 100644 --- a/src/lib/virtio/net_device.lua +++ b/src/lib/virtio/net_device.lua @@ -306,7 +306,16 @@ function VirtioNetDevice:tx_buffer_add_mrg_rxbuf(tx_p, addr, len) self.tx.finished = true end - return to_copy + -- XXX The "adjust" is needed to counter-balance an adjustment made + -- in virtq_device. If we don't make this adjustment then we break + -- chaining together multiple buffers in that we report the size of + -- each buffer (except for the first) to be 12 bytes more than it + -- really is. This causes the VM to see an inflated ethernet packet + -- size which may or may not be noticed by an application. + -- + -- This formulation is not optimal and it would be nice to make + -- this code more transparent. -luke + return to_copy - adjust end function VirtioNetDevice:tx_packet_end_mrg_rxbuf(header_id, total_size, tx_p) From 0a5bfac2fbd049d7387c1129f2467a9879b8681f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 30 May 2016 18:29:07 +0200 Subject: [PATCH 036/103] Remove unassigned initialization --- src/core/app.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index ccf163193d..2fcb85df96 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -148,8 +148,8 @@ end -- Update the active app network by applying the necessary actions. function apply_config_actions (actions, conf) -- The purpose of this function is to populate these tables: - local new_app_table, new_app_array = {}, {}, {} - local new_link_table, new_link_array = {}, {}, {} + local new_app_table, new_app_array = {}, {} + local new_link_table, new_link_array = {}, {} -- Temporary name->index table for use in link renumbering local app_name_to_index = {} -- Table of functions that execute config actions From 80a7ee7eaa384185baeee6fd075a17b80221a702 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 30 May 2016 18:30:05 +0200 Subject: [PATCH 037/103] Trigger start event for each app --- src/core/app.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 2fcb85df96..6260ebfb8f 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -178,7 +178,6 @@ function apply_config_actions (actions, conf) table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array app.zone = zone - if app.start then app:start() end end function ops.restart (name) ops.stop(name) @@ -226,9 +225,13 @@ function apply_config_actions (actions, conf) for linkspec, r in pairs(link_table) do if not new_link_table[linkspec] then link.free(r, linkspec) end end - -- commit changes + -- Commit changes. app_table, link_table = new_app_table, new_link_table app_array, link_array = new_app_array, new_link_array + -- Trigger start event for each app. + for _, app in ipairs(app_array) do + app:start() + end end -- Call this to "run snabb switch". From c86ec3e305f6c5f62c328caa76c7c623aaf67087 Mon Sep 17 00:00:00 2001 From: Diego Pino Date: Mon, 30 May 2016 16:53:08 +0000 Subject: [PATCH 038/103] Call app:start only if defined --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 6260ebfb8f..b04a2312c0 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -230,7 +230,7 @@ function apply_config_actions (actions, conf) app_array, link_array = new_app_array, new_link_array -- Trigger start event for each app. for _, app in ipairs(app_array) do - app:start() + if app.start then app:start() end end end From 62e2fbfe631a56631f7c115fd73b82550256fd83 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 1 Jun 2016 15:25:44 +0200 Subject: [PATCH 039/103] Revert "lib.json: Import JSON4Lua 1.0.0, include encode functionality." This reverts commit 924ff4e6e82880d296d6be9ddae4dda3aef4dff0. --- src/lib/json.lua | 658 +++++++++++++++++------------------------------ 1 file changed, 241 insertions(+), 417 deletions(-) diff --git a/src/lib/json.lua b/src/lib/json.lua index 9ab4abd38e..86cbf85e43 100644 --- a/src/lib/json.lua +++ b/src/lib/json.lua @@ -1,417 +1,241 @@ ------------------------------------------------------------------------------ --- JSON4Lua: JSON encoding / decoding support for the Lua language. --- json Module. --- Author: Craig Mason-Jones --- Homepage: http://github.com/craigmj/json4lua/ --- Version: 1.0.0 --- This module is released under the MIT License (MIT). --- Please see LICENCE.txt for details. --- --- USAGE: --- This module exposes two functions: --- json.encode(o) --- Returns the table / string / boolean / number / nil / json.null value as a JSON-encoded string. --- json.decode(json_string) --- Returns a Lua object populated with the data encoded in the JSON string json_string. --- --- REQUIREMENTS: --- compat-5.1 if using Lua 5.0 --- --- CHANGELOG --- 0.9.20 Introduction of local Lua functions for private functions (removed _ function prefix). --- Fixed Lua 5.1 compatibility issues. --- Introduced json.null to have null values in associative arrays. --- json.encode() performance improvement (more than 50%) through table.concat rather than .. --- Introduced decode ability to ignore /**/ comments in the JSON string. --- 0.9.10 Fix to array encoding / decoding to correctly manage nil/null values in arrays. ------------------------------------------------------------------------------ - ------------------------------------------------------------------------------ --- Imports and dependencies ------------------------------------------------------------------------------ -local math = require('math') -local string = require("string") -local table = require("table") - ------------------------------------------------------------------------------ --- Module declaration ------------------------------------------------------------------------------ -local json = {} -- Public namespace -local json_private = {} -- Private namespace - --- Public functions - --- Private functions -local decode_scanArray -local decode_scanComment -local decode_scanConstant -local decode_scanNumber -local decode_scanObject -local decode_scanString -local decode_scanWhitespace -local encodeString -local isArray -local isEncodable - ------------------------------------------------------------------------------ --- PUBLIC FUNCTIONS ------------------------------------------------------------------------------ ---- Encodes an arbitrary Lua object / variable. --- @param v The Lua object / variable to be JSON encoded. --- @return String containing the JSON encoding in internal Lua string format (i.e. not unicode) -function json.encode (v) - -- Handle nil values - if v==nil then - return "null" - end - - local vtype = type(v) - - -- Handle strings - if vtype=='string' then - return '"' .. json_private.encodeString(v) .. '"' -- Need to handle encoding in string - end - - -- Handle booleans - if vtype=='number' or vtype=='boolean' then - return tostring(v) - end - - -- Handle tables - if vtype=='table' then - local rval = {} - -- Consider arrays separately - local bArray, maxCount = isArray(v) - if bArray then - for i = 1,maxCount do - table.insert(rval, json.encode(v[i])) - end - else -- An object, not an array - for i,j in pairs(v) do - if isEncodable(i) and isEncodable(j) then - table.insert(rval, '"' .. json_private.encodeString(i) .. '":' .. json.encode(j)) - end - end - end - if bArray then - return '[' .. table.concat(rval,',') ..']' - else - return '{' .. table.concat(rval,',') .. '}' - end - end - - -- Handle null values - if vtype=='function' and v==null then - return 'null' - end - - assert(false,'encode attempt to encode unsupported type ' .. vtype .. ':' .. tostring(v)) -end - - ---- Decodes a JSON string and returns the decoded value as a Lua data structure / value. --- @param s The string to scan. --- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. --- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, --- and the position of the first character after --- the scanned JSON object. -function json.decode(s, startPos) - startPos = startPos and startPos or 1 - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') - local curChar = string.sub(s,startPos,startPos) - -- Object - if curChar=='{' then - return decode_scanObject(s,startPos) - end - -- Array - if curChar=='[' then - return decode_scanArray(s,startPos) - end - -- Number - if string.find("+-0123456789.e", curChar, 1, true) then - return decode_scanNumber(s,startPos) - end - -- String - if curChar==[["]] or curChar==[[']] then - return decode_scanString(s,startPos) - end - if string.sub(s,startPos,startPos+1)=='/*' then - return decode(s, decode_scanComment(s,startPos)) - end - -- Otherwise, it must be a constant - return decode_scanConstant(s,startPos) -end - ---- The null function allows one to specify a null value in an associative array (which is otherwise --- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } -function null() - return null -- so json.null() will also return null ;-) -end ------------------------------------------------------------------------------ --- Internal, PRIVATE functions. --- Following a Python-like convention, I have prefixed all these 'PRIVATE' --- functions with an underscore. ------------------------------------------------------------------------------ - ---- Scans an array from JSON into a Lua object --- startPos begins at the start of the array. --- Returns the array and the next starting position --- @param s The string being scanned. --- @param startPos The starting position for the scan. --- @return table, int The scanned array as a table, and the position of the next character to scan. -function decode_scanArray(s,startPos) - local array = {} -- The return value - local stringLen = string.len(s) - assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) - startPos = startPos + 1 - -- Infinite loop for array elements - repeat - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') - local curChar = string.sub(s,startPos,startPos) - if (curChar==']') then - return array, startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') - object, startPos = json.decode(s,startPos) - table.insert(array,object) - until false -end - ---- Scans a comment and discards the comment. --- Returns the position of the next character following the comment. --- @param string s The JSON string to scan. --- @param int startPos The starting position of the comment -function decode_scanComment(s, startPos) - assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) - local endPos = string.find(s,'*/',startPos+2) - assert(endPos~=nil, "Unterminated comment in string at " .. startPos) - return endPos+2 -end - ---- Scans for given constants: true, false or null --- Returns the appropriate Lua type, and the position of the next character to read. --- @param s The string being scanned. --- @param startPos The position in the string at which to start scanning. --- @return object, int The object (true, false or nil) and the position at which the next character should be --- scanned. -function decode_scanConstant(s, startPos) - local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } - local constNames = {"true","false","null"} - - for i,k in pairs(constNames) do - if string.sub(s,startPos, startPos + string.len(k) -1 )==k then - return consts[k], startPos + string.len(k) - end - end - assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) -end - ---- Scans a number from the JSON encoded string. --- (in fact, also is able to scan numeric +- eqns, which is not --- in the JSON spec.) --- Returns the number, and the position of the next character --- after the number. --- @param s The string being scanned. --- @param startPos The position at which to start scanning. --- @return number, int The extracted number and the position of the next character to scan. -function decode_scanNumber(s,startPos) - local endPos = startPos+1 - local stringLen = string.len(s) - local acceptableChars = "+-0123456789.e" - while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) - and endPos<=stringLen - ) do - endPos = endPos + 1 - end - local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) - local stringEval = loadstring(stringValue) - assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON object into a Lua object. --- startPos begins at the start of the object. --- Returns the object and the next starting position. --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return table, int The scanned object as a table and the position of the next character to scan. -function decode_scanObject(s,startPos) - local object = {} - local stringLen = string.len(s) - local key, value - assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) - startPos = startPos + 1 - repeat - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') - local curChar = string.sub(s,startPos,startPos) - if (curChar=='}') then - return object,startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') - -- Scan the key - key, startPos = json.decode(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) - startPos = decode_scanWhitespace(s,startPos+1) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - value, startPos = json.decode(s,startPos) - object[key]=value - until false -- infinite loop while key-value pairs are found -end - --- START SoniEx2 --- Initialize some things used by decode_scanString --- You know, for efficiency -local escapeSequences = { - ["\\t"] = "\t", - ["\\f"] = "\f", - ["\\r"] = "\r", - ["\\n"] = "\n", - ["\\b"] = "\b" -} -setmetatable(escapeSequences, {__index = function(t,k) - -- skip "\" aka strip escape - return string.sub(k,2) -end}) --- END SoniEx2 - ---- Scans a JSON string from the opening inverted comma or single quote to the --- end of the string. --- Returns the string extracted as a Lua string, --- and the position of the next non-string character --- (after the closing inverted comma or single quote). --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return string, int The extracted string as a Lua string, and the next character to parse. -function decode_scanString(s,startPos) - assert(startPos, 'decode_scanString(..) called without start position') - local startChar = string.sub(s,startPos,startPos) - -- START SoniEx2 - -- PS: I don't think single quotes are valid JSON - assert(startChar == [["]] or startChar == [[']],'decode_scanString called for a non-string') - --assert(startPos, "String decoding failed: missing closing " .. startChar .. " for string at position " .. oldStart) - local t = {} - local i,j = startPos,startPos - while string.find(s, startChar, j+1) ~= j+1 do - local oldj = j - i,j = string.find(s, "\\.", j+1) - local x,y = string.find(s, startChar, oldj+1) - if not i or x < i then - i,j = x,y-1 - end - table.insert(t, string.sub(s, oldj+1, i-1)) - if string.sub(s, i, j) == "\\u" then - local a = string.sub(s,j+1,j+4) - j = j + 4 - local n = tonumber(a, 16) - assert(n, "String decoding failed: bad Unicode escape " .. a .. " at position " .. i .. " : " .. j) - -- math.floor(x/2^y) == lazy right shift - -- a % 2^b == bitwise_and(a, (2^b)-1) - -- 64 = 2^6 - -- 4096 = 2^12 (or 2^6 * 2^6) - local x - if n < 0x80 then - x = string.char(n % 0x80) - elseif n < 0x800 then - -- [110x xxxx] [10xx xxxx] - x = string.char(0xC0 + (math.floor(n/64) % 0x20), 0x80 + (n % 0x40)) - else - -- [1110 xxxx] [10xx xxxx] [10xx xxxx] - x = string.char(0xE0 + (math.floor(n/4096) % 0x10), 0x80 + (math.floor(n/64) % 0x40), 0x80 + (n % 0x40)) - end - table.insert(t, x) - else - table.insert(t, escapeSequences[string.sub(s, i, j)]) - end - end - table.insert(t,string.sub(j, j+1)) - assert(string.find(s, startChar, j+1), "String decoding failed: missing closing " .. startChar .. " at position " .. j .. "(for string at position " .. startPos .. ")") - return table.concat(t,""), j+2 - -- END SoniEx2 -end - ---- Scans a JSON string skipping all whitespace from the current start position. --- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. --- @param s The string being scanned --- @param startPos The starting position where we should begin removing whitespace. --- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string --- was reached. -function decode_scanWhitespace(s,startPos) - local whitespace=" \n\r\t" - local stringLen = string.len(s) - while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do - startPos = startPos + 1 - end - return startPos -end - ---- Encodes a string to be JSON-compatible. --- This just involves back-quoting inverted commas, back-quotes and newlines, I think ;-) --- @param s The string to return as a JSON encoded (i.e. backquoted string) --- @return The string appropriately escaped. - -local escapeList = { - ['"'] = '\\"', - ['\\'] = '\\\\', - ['/'] = '\\/', - ['\b'] = '\\b', - ['\f'] = '\\f', - ['\n'] = '\\n', - ['\r'] = '\\r', - ['\t'] = '\\t' -} - -function json_private.encodeString(s) - local s = tostring(s) - return s:gsub(".", function(c) return escapeList[c] end) -- SoniEx2: 5.0 compat -end - --- Determines whether the given Lua type is an array or a table / dictionary. --- We consider any table an array if it has indexes 1..n for its n items, and no --- other data in the table. --- I think this method is currently a little 'flaky', but can't think of a good way around it yet... --- @param t The table to evaluate as an array --- @return boolean, number True if the table can be represented as an array, false otherwise. If true, --- the second returned value is the maximum --- number of indexed elements in the array. -function isArray(t) - -- Next we count all the elements, ensuring that any non-indexed elements are not-encodable - -- (with the possible exception of 'n') - local maxIndex = 0 - for k,v in pairs(t) do - if (type(k)=='number' and math.floor(k)==k and 1<=k) then -- k,v is an indexed pair - if (not isEncodable(v)) then return false end -- All array elements must be encodable - maxIndex = math.max(maxIndex,k) - else - if (k=='n') then - if v ~= table.getn(t) then return false end -- False if n does not hold the number of elements - else -- Else of (k=='n') - if isEncodable(v) then return false end - end -- End of (k~='n') - end -- End of k,v not an indexed pair - end -- End of loop across all pairs - return true, maxIndex -end - ---- Determines whether the given Lua object / table / variable can be JSON encoded. The only --- types that are JSON encodable are: string, boolean, number, nil, table and json.null. --- In this implementation, all other types are ignored. --- @param o The object to examine. --- @return boolean True if the object should be JSON encoded, false if it should be ignored. -function isEncodable(o) - local t = type(o) - return (t=='string' or t=='boolean' or t=='number' or t=='nil' or t=='table') or (t=='function' and o==null) -end - -return json \ No newline at end of file +-- JSON4Lua: JSON encoding / decoding support for the Lua language. +-- json Module. +-- Author: Craig Mason-Jones +-- Homepage: http://json.luaforge.net/ +-- Version: 0.9.40 +-- This module is released under the MIT License (MIT). +-- +-- NOTE: This is only the decode functionality ripped out from JSON4Lua. +-- See: https://github.com/craigmj/json4lua + +module(..., package.seeall) + +local math = require('math') +local string = require("string") +local table = require("table") + +local base = _G + +-- Private functions +local decode_scanArray +local decode_scanComment +local decode_scanConstant +local decode_scanNumber +local decode_scanObject +local decode_scanString +local decode_scanWhitespace + +--- Decodes a JSON string and returns the decoded value as a Lua data structure / value. +-- @param s The string to scan. +-- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. +-- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, +-- and the position of the first character after +-- the scanned JSON object. +function decode(s, startPos) + startPos = startPos and startPos or 1 + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') + local curChar = string.sub(s,startPos,startPos) + -- Object + if curChar=='{' then + return decode_scanObject(s,startPos) + end + -- Array + if curChar=='[' then + return decode_scanArray(s,startPos) + end + -- Number + if string.find("+-0123456789.e", curChar, 1, true) then + return decode_scanNumber(s,startPos) + end + -- String + if curChar==[["]] or curChar==[[']] then + return decode_scanString(s,startPos) + end + if string.sub(s,startPos,startPos+1)=='/*' then + return decode(s, decode_scanComment(s,startPos)) + end + -- Otherwise, it must be a constant + return decode_scanConstant(s,startPos) +end + +--- The null function allows one to specify a null value in an associative array (which is otherwise +-- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } +function null() + return null -- so json.null() will also return null ;-) +end +----------------------------------------------------------------------------- +-- Internal, PRIVATE functions. +-- Following a Python-like convention, I have prefixed all these 'PRIVATE' +-- functions with an underscore. +----------------------------------------------------------------------------- + +--- Scans an array from JSON into a Lua object +-- startPos begins at the start of the array. +-- Returns the array and the next starting position +-- @param s The string being scanned. +-- @param startPos The starting position for the scan. +-- @return table, int The scanned array as a table, and the position of the next character to scan. +function decode_scanArray(s,startPos) + local array = {} -- The return value + local stringLen = string.len(s) + base.assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) + startPos = startPos + 1 + -- Infinite loop for array elements + repeat + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') + local curChar = string.sub(s,startPos,startPos) + if (curChar==']') then + return array, startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + base.assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') + object, startPos = decode(s,startPos) + table.insert(array,object) + until false +end + +--- Scans a comment and discards the comment. +-- Returns the position of the next character following the comment. +-- @param string s The JSON string to scan. +-- @param int startPos The starting position of the comment +function decode_scanComment(s, startPos) + base.assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) + local endPos = string.find(s,'*/',startPos+2) + base.assert(endPos~=nil, "Unterminated comment in string at " .. startPos) + return endPos+2 +end + +--- Scans for given constants: true, false or null +-- Returns the appropriate Lua type, and the position of the next character to read. +-- @param s The string being scanned. +-- @param startPos The position in the string at which to start scanning. +-- @return object, int The object (true, false or nil) and the position at which the next character should be +-- scanned. +function decode_scanConstant(s, startPos) + local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } + local constNames = {"true","false","null"} + + for i,k in base.pairs(constNames) do + --print ("[" .. string.sub(s,startPos, startPos + string.len(k) -1) .."]", k) + if string.sub(s,startPos, startPos + string.len(k) -1 )==k then + return consts[k], startPos + string.len(k) + end + end + base.assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) +end + +--- Scans a number from the JSON encoded string. +-- (in fact, also is able to scan numeric +- eqns, which is not +-- in the JSON spec.) +-- Returns the number, and the position of the next character +-- after the number. +-- @param s The string being scanned. +-- @param startPos The position at which to start scanning. +-- @return number, int The extracted number and the position of the next character to scan. +function decode_scanNumber(s,startPos) + local endPos = startPos+1 + local stringLen = string.len(s) + local acceptableChars = "+-0123456789.e" + while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) + and endPos<=stringLen + ) do + endPos = endPos + 1 + end + local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) + local stringEval = base.loadstring(stringValue) + base.assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON object into a Lua object. +-- startPos begins at the start of the object. +-- Returns the object and the next starting position. +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return table, int The scanned object as a table and the position of the next character to scan. +function decode_scanObject(s,startPos) + local object = {} + local stringLen = string.len(s) + local key, value + base.assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) + startPos = startPos + 1 + repeat + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') + local curChar = string.sub(s,startPos,startPos) + if (curChar=='}') then + return object,startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') + -- Scan the key + key, startPos = decode(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + base.assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) + startPos = decode_scanWhitespace(s,startPos+1) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + value, startPos = decode(s,startPos) + object[key]=value + until false -- infinite loop while key-value pairs are found +end + +--- Scans a JSON string from the opening inverted comma or single quote to the +-- end of the string. +-- Returns the string extracted as a Lua string, +-- and the position of the next non-string character +-- (after the closing inverted comma or single quote). +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return string, int The extracted string as a Lua string, and the next character to parse. +function decode_scanString(s,startPos) + base.assert(startPos, 'decode_scanString(..) called without start position') + local startChar = string.sub(s,startPos,startPos) + base.assert(startChar==[[']] or startChar==[["]],'decode_scanString called for a non-string') + local escaped = false + local endPos = startPos + 1 + local bEnded = false + local stringLen = string.len(s) + repeat + local curChar = string.sub(s,endPos,endPos) + -- Character escaping is only used to escape the string delimiters + if not escaped then + if curChar==[[\]] then + escaped = true + else + bEnded = curChar==startChar + end + else + -- If we're escaped, we accept the current character come what may + escaped = false + end + endPos = endPos + 1 + base.assert(endPos <= stringLen+1, "String decoding failed: unterminated string at position " .. endPos) + until bEnded + local stringValue = 'return ' .. string.sub(s, startPos, endPos-1) + local stringEval = base.loadstring(stringValue) + base.assert(stringEval, 'Failed to load string [ ' .. stringValue .. '] in JSON4Lua.decode_scanString at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON string skipping all whitespace from the current start position. +-- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. +-- @param s The string being scanned +-- @param startPos The starting position where we should begin removing whitespace. +-- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string +-- was reached. +function decode_scanWhitespace(s,startPos) + local whitespace=" \n\r\t" + local stringLen = string.len(s) + while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do + startPos = startPos + 1 + end + return startPos +end From f4834a515b5eef30de76dca2e572b77b28fd625f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 1 Jun 2016 15:26:23 +0200 Subject: [PATCH 040/103] snabb top: revert --yang dump. --- src/program/top/README | 2 - src/program/top/top.lua | 91 +---------------------------------------- 2 files changed, 2 insertions(+), 91 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 9fa3500804..ce815d6f89 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -5,8 +5,6 @@ Usage: Print usage information. -c, --counters Print counters of object by and exit. - -y, --yang - Print YANG model encoded in JSON and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 317c0a2a77..8334d76be3 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -9,12 +9,10 @@ local shm = require("core.shm") local counter = require("core.counter") local S = require("syscall") local histogram = require("core.histogram") -local json = require("lib.json") -local macaddress = require("lib.macaddress") local usage = require("program.top.README_inc") local long_opts = { - help = "h", counters = "c", yang = "y" + help = "h", counters = "c" } function clearterm () io.write('\027[2J') end @@ -22,17 +20,14 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local object = nil - local yang = false function opt.h (arg) print(usage) main.exit(1) end function opt.c (arg) object = arg end - function opt.y () yang = true end - args = lib.dogetopt(args, opt, "hc:y", long_opts) + args = lib.dogetopt(args, opt, "hc:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) if object then list_counters(target_pid, object) - elseif yang then dump_yang(target_pid) else top(target_pid) end ordered_exit(0) end @@ -76,88 +71,6 @@ function list_counters (pid, object) end end -function dump_yang (instance_pid) - local instance_tree = "//"..instance_pid - local interface_state = {} - local types = { [0x1000] = 'hardware', - [0x1001] = 'virtual', - [0x1002] = 'link' } - - for _, link in ipairs(shm.children(instance_tree.."/links")) do - local counters = instance_tree.."/counters/"..link - local statistics = {} - statistics['discontinuity-time'] = - totime(read_counter('discontinuity-time', counters)) - statistics['in-octets'] = - tostring64(read_counter('rxbytes', counters)) - statistics['out-octets'] = - tostring64(read_counter('txbytes', counters)) - statistics['out-discards'] = - truncate32(tonumber(read_counter('txdrop', counters))) - table.insert(interface_state, { name = link, - type = types[0x1002], - statistics = statistics }) - end - - for _, name in ipairs(shm.children(instance_tree.."/counters")) do - local counters = instance_tree.."/counters/"..name - local exists = {} - for _, c in ipairs(shm.children(counters)) do exists[c] = true end - local type = nil - if exists['type'] then - type = types[tonumber(read_counter('type', counters))] - end - if type then - local statistics = {} - statistics['discontinuity-time'] = - totime(read_counter('discontinuity-time', counters)) - for _, c in ipairs({'in-octets', 'in-unicast', - 'in-broadcast', 'in-multicast', - 'out-octets', 'out-unicast', - 'out-broadcast', 'out-multicast'}) do - if exists[c] then - statistics[c] = tostring64(read_counter(c, counters)) - end - end - for _, c in ipairs({'in-discards', 'out-discards'}) do - if exists[c] then - statistics[c] = truncate32(read_counter(c, counters)) - end - end - local interface = { name = name, type = type, statistics = statistics} - if exists['phys-address'] then - interface['phys-address'] = - tomac(read_counter('phys-address', counters)) - end - table.insert(interface_state, interface) - end - end - - print(json.encode({['interface-state'] = interface_state})) -end - -function tomac (n) - return tostring(macaddress:new(n)) -end - -function totime (n) - return os.date("!%FT%TZ", tonumber(n)) -end - -function truncate32 (n) - local box = ffi.new("union { uint64_t i64; uint32_t i32[2]; }") - box.i64 = n - if ffi.abi("le") then return tonumber(box.i32[0]) - elseif ffi.abi("be") then return tonumber(box.i32[1]) end -end - -function tostring64 (n) - assert(ffi.istype('uint64_t', n)) - local s = tostring(n) - return s:sub(1, #s - 3) -end - - function top (instance_pid) local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) From 793c394322294d8eb444244a5954315156392a37 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:35:41 +0200 Subject: [PATCH 041/103] snabbify statistics model. --- model.txt | 41 ++++++++++++++++++++++++++ src/apps/intel/intel_app.lua | 54 ++++++++++++++++------------------- src/apps/vhost/vhost_user.lua | 30 ++++++++++--------- src/core/link.h | 2 +- src/core/link.lua | 15 +++++----- 5 files changed, 89 insertions(+), 53 deletions(-) create mode 100644 model.txt diff --git a/model.txt b/model.txt new file mode 100644 index 0000000000..79df1e9847 --- /dev/null +++ b/model.txt @@ -0,0 +1,41 @@ +RFC7223 Snabb +======= ====== +name +description? +type type (~= identity) +enabled? +link-up-down-trap-enable? +admin-status +oper-status status (= enum) +last-change? +if-index +phys-address? macaddr (uint64) +higher-layer-if* +lower-layer-if* +speed? = +discontinuity-time dtime (seconds since epoch) +in-octets? rxbytes +in-unicast-pkts? +in-broadcast-pkts? rxbcast +in-multicast-pkts? +in-discards? +in-errors? +in-unknown-protos? +out-octets? txbytes +out-unicast-pkts? +out-broadcast-pkts? txbcast +out-multicast-pkts? +out-discards? +out-errors? + + +snabb/type: + +0x1000 Hardware interface +0x1001 Virtual interface + +snabb/stats: + +(tx/rx) - bcast ⊆ mcast ⊆ packets + +(tx/rx) - drop (all dropped packets) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 2645e2d22c..7b30e66ac0 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -57,28 +57,22 @@ function Intel82599:new (arg) self.zone = "intel" end if not self.stats.counters then - local counters = {} - local path = "/counters/"..conf.pciaddr.."/" - counters['type'] = counter.open(path..'type') - counters['discontinuity-time'] = counter.open(path..'discontinuity-time') - counters['in-octets'] = counter.open(path..'in-octets') - counters['in-multicast'] = counter.open(path..'in-multicast') - counters['in-broadcast'] = counter.open(path..'in-broadcast') - counters['in-discards'] = counter.open(path..'in-discards') - counters['out-octets'] = counter.open(path..'out-octets') - counters['out-multicast'] = counter.open(path..'out-multicast') - counters['out-broadcast'] = counter.open(path..'out-broadcast') - counters['out-discards'] = counter.open(path..'out-discards') - counter.set(counters['type'], 0x1000) -- Hardware interface - counter.set(counters['discontinuity-time'], C.get_unix_time()) + self.stats.path = "/counters/"..conf.pciaddr.."/" + self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) + self.stats.counters = {} + for _, name in ipairs( + {'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', + 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop'}) do + self.stats.counters[name] = counter.open(self.stats.path..name) + end + counter.set(self.stats.counters.type, 0x1000) -- Hardware interface + counter.set(self.stats.counters.dtime, C.get_unix_time()) if not conf.vmdq and conf.macaddr then - counters['phys-address'] = counter.open(path..'phys-address') - counter.set(counters['phys-address'], + self.stats.counters.macaddr = counter.open(self.stats.path..'macaddr') + counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end - self.stats.counters = counters - self.stats.path = path - self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) end return setmetatable(self, Intel82599) end @@ -117,7 +111,7 @@ function Intel82599:reconfig(arg) self.dev:reconfig(conf) if not self.dev.pf and conf.macaddr then - counter.set(self.stats.counters['phys-address'], + counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end end @@ -151,17 +145,17 @@ end -- Synchronize self.stats.register a and self.stats.counters. function Intel82599:sync_stats () - --- XXX - it is questionable if I choose the right register to counter - --- mapping local counters, register = self.stats.counters, self.stats.register - counter.set(counters['in-octets'], register.GORC64()) - counter.set(counters['in-multicast'], register.MPRC()) - counter.set(counters['in-broadcast'], register.BPRC()) - counter.set(counters['in-discards'], register.TPR() - register.GPRC()) - counter.set(counters['out-octets'], register.GOTC64()) - counter.set(counters['out-multicast'], register.MPTC()) - counter.set(counters['out-broadcast'], register.BPTC()) - counter.set(counters['out-discards'], register.TPT() - register.GPTC()) + counter.set(counters.rxbytes, register.GORC64()) + counter.set(counters.rxpackets, register.GPRC()) + counter.set(counters.rxmcast, register.MPRC()) + counter.set(counters.rxbcast, register.BPRC()) + counter.set(counters.rxdrop, register.TPR() - register.GPRC()) + counter.set(counters.txbytes, register.GOTC64()) + counter.set(counters.txpackets, register.GPTC()) + counter.set(counters.txmcast, register.MPTC()) + counter.set(counters.txbcast, register.BPTC()) + counter.set(counters.txdrop, register.TPT() - register.GPTC()) end -- Push packets from our 'rx' link onto the network. diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 16936709bf..fe2711769e 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -29,6 +29,12 @@ assert(ffi.sizeof("struct vhost_user_msg") == 276, "ABI error") VhostUser = {} +local provided_counters = { + 'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxdrop', + 'txbytes', 'txpackets', 'txmcast' +} + function VhostUser:new (args) local o = { state = 'init', dev = nil, @@ -56,13 +62,11 @@ function VhostUser:new (args) end -- initialize counters self.counters = {} - for _, name in ipairs({'type', 'discontinuity-time', - 'in-octets', 'in-unicast', 'in-multicast', 'in-discards', - 'out-octets', 'out-unicast', 'out-multicast'}) do + for _, name in ipairs(provided_counters) do self.counters[name] = counter.open(name) end - counter.set(self.counters['type'], 0x1001) -- Virtual interface - counter.set(self.counters['discontinuity-time'], C.get_unix_time()) + counter.set(self.counters.type, 0x1001) -- Virtual interface + counter.set(self.counters.dtime, C.get_unix_time()) return self end @@ -101,21 +105,19 @@ function VhostUser:push () end function VhostUser:tx_callback (p) - counter.add(self.counters['out-octets'], packet.length(p)) - local mcast = ethernet:n_mcast(packet.data(p)) - counter.add(self.counters['out-multicast'], mcast) - counter.add(self.counters['out-unicast'], 1 - mcast) + counter.add(self.counters.txbytes, packet.length(p)) + counter.add(self.counters.txpackets) + counter.add(self.counters.txmcast, ethernet:n_mcast(packet.data(p))) end function VhostUser:rx_callback (p) - counter.add(self.counters['in-octets'], packet.length(p)) - local mcast = ethernet:n_mcast(packet.data(p)) - counter.add(self.counters['in-multicast'], mcast) - counter.add(self.counters['in-unicast'], 1 - mcast) + counter.add(self.counters.rxbytes, packet.length(p)) + counter.add(self.counters.rxpackets) + counter.add(self.counters.rxmcast, ethernet:n_mcast(packet.data(p))) end function VhostUser:rxdrop_callback (p) - counter.add(self.counters['in-discards']) + counter.add(self.counters.rxdrop) end -- Try to connect to QEMU. diff --git a/src/core/link.h b/src/core/link.h index c4ca2b45a5..f4caadaf91 100644 --- a/src/core/link.h +++ b/src/core/link.h @@ -9,7 +9,7 @@ struct link { // http://en.wikipedia.org/wiki/Circular_buffer struct packet *packets[LINK_RING_SIZE]; struct { - struct counter *txbytes, *rxbytes, *txpackets, *rxpackets, *txdrop; + struct counter *dtime, *txbytes, *rxbytes, *txpackets, *rxpackets, *txdrop; } stats; // Two cursors: // read: the next element to be read diff --git a/src/core/link.lua b/src/core/link.lua index 34e2ffc942..aacff1e741 100644 --- a/src/core/link.lua +++ b/src/core/link.lua @@ -21,23 +21,23 @@ local band = require("bit").band local size = C.LINK_RING_SIZE -- NB: Huge slow-down if this is not local max = C.LINK_MAX_PACKETS -local counternames = {"rxpackets", "txpackets", "rxbytes", "txbytes", "txdrop"} +local provided_counters = { + "dtime", "rxpackets", "rxbytes", "txpackets", "txbytes", "txdrop" +} function new (name) local r = shm.create("links/"..name, "struct link") - for _, c in ipairs(counternames) do + for _, c in ipairs(provided_counters) do r.stats[c] = counter.open("counters/"..name.."/"..c) end - counter.set(counter.open("counters/"..name.."/discontinuity-time"), - C.get_unix_time()) + counter.set(r.stats.dtime, C.get_unix_time()) return r end function free (r, name) - for _, c in ipairs(counternames) do + for _, c in ipairs(provided_counters) do counter.delete("counters/"..name.."/"..c) end - counter.delete("counters/"..name.."/discontinuity-time") shm.unmap(r) shm.unlink("links/"..name) end @@ -95,8 +95,7 @@ end function stats (r) local stats = {} - for _, c - in ipairs(counternames) do + for _, c in ipairs(provided_counters) do stats[c] = tonumber(counter.read(r.stats[c])) end return stats From 2b9ef2c82df610ec1470a5bb11615f5d8d0510bd Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:37:13 +0200 Subject: [PATCH 042/103] intel_app: add ifTable MIB related counters. --- model.txt | 47 +++++++++++++----------- src/apps/intel/intel_app.lua | 69 ++++++++++++++++++++++++++---------- 2 files changed, 77 insertions(+), 39 deletions(-) diff --git a/model.txt b/model.txt index 79df1e9847..8f4e128823 100644 --- a/model.txt +++ b/model.txt @@ -1,41 +1,48 @@ -RFC7223 Snabb -======= ====== +RFC7223 Snabb ifTable MIB +======= ===== =========== name description? type type (~= identity) enabled? link-up-down-trap-enable? admin-status -oper-status status (= enum) +oper-status status (= enum) ifOperStatus last-change? if-index -phys-address? macaddr (uint64) +phys-address? macaddr (uint64) ifPhysAddress higher-layer-if* lower-layer-if* -speed? = -discontinuity-time dtime (seconds since epoch) -in-octets? rxbytes +speed? speed ifSpeed +discontinuity-time dtime (seconds since epoch) ifCounterDiscontinuityTime +in-octets? rxbytes ifInOctets + rxpackets in-unicast-pkts? -in-broadcast-pkts? rxbcast -in-multicast-pkts? -in-discards? -in-errors? +in-broadcast-pkts? rxbcast ifInBroadcastPkts + rxmcast +in-multicast-pkts? ifInMulticastPkts +in-discards? rxdrop ifInDiscards +in-errors? rxerrors ifInErrors in-unknown-protos? -out-octets? txbytes +out-octets? txbytes ifOutOctets + txpackets out-unicast-pkts? -out-broadcast-pkts? txbcast -out-multicast-pkts? -out-discards? -out-errors? - +out-broadcast-pkts? txbcast ifOutBroadcastPkts + txmcast +out-multicast-pkts? ifOutMulticastPkts +out-discards? txdrop ifOutDiscards +out-errors? txerrors ifOutErrors + rxpackets + txpackets + rxmcast + txmcast + mtu ifMtu + promisc ifPromiscuousMode snabb/type: 0x1000 Hardware interface 0x1001 Virtual interface -snabb/stats: +snabb/bcast, mcast, packets: (tx/rx) - bcast ⊆ mcast ⊆ packets - -(tx/rx) - drop (all dropped packets) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 7b30e66ac0..623fb1a3a3 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -34,6 +34,11 @@ local function firsthole(t) end end +local provided_counters = { + 'type', 'dtime', 'mtu', 'speed', 'status', 'promisc', 'macaddr', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', 'rxerrors', + 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop', 'txerrors' +} -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) @@ -43,7 +48,9 @@ function Intel82599:new (arg) if conf.vmdq then if devices[conf.pciaddr] == nil then local pf = intel10g.new_pf(conf):open() - devices[conf.pciaddr] = {pf=pf, vflist={}, stats={register=pf.s}} + devices[conf.pciaddr] = { pf = pf, + vflist = {}, + stats = { s = pf.s, r = pf.r, qs = pf.qs } } end local dev = devices[conf.pciaddr] local poolnum = firsthole(dev.vflist)-1 @@ -53,23 +60,22 @@ function Intel82599:new (arg) self.stats = devices[conf.pciaddr].stats else self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") - self.stats = { register = self.dev.s } + self.stats = { s = self.dev.s, r = self.dev.r, qs = self.dev.qs } self.zone = "intel" end if not self.stats.counters then self.stats.path = "/counters/"..conf.pciaddr.."/" self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) self.stats.counters = {} - for _, name in ipairs( - {'type', 'dtime', - 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', - 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop'}) do + for _, name in ipairs(provided_counters) do self.stats.counters[name] = counter.open(self.stats.path..name) end counter.set(self.stats.counters.type, 0x1000) -- Hardware interface counter.set(self.stats.counters.dtime, C.get_unix_time()) + counter.set(self.stats.counters.mtu, self.dev.mtu) + counter.set(self.stats.counters.speed, 10000000) -- 10 Gbits + counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then - self.stats.counters.macaddr = counter.open(self.stats.path..'macaddr') counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end @@ -143,19 +149,37 @@ function Intel82599:add_receive_buffers () end end --- Synchronize self.stats.register a and self.stats.counters. +-- Synchronize self.stats.s/r a and self.stats.counters. +local link_up_mask = lib.bits{Link_up=30} +local promisc_mask = lib.bits{UPE=9} function Intel82599:sync_stats () - local counters, register = self.stats.counters, self.stats.register - counter.set(counters.rxbytes, register.GORC64()) - counter.set(counters.rxpackets, register.GPRC()) - counter.set(counters.rxmcast, register.MPRC()) - counter.set(counters.rxbcast, register.BPRC()) - counter.set(counters.rxdrop, register.TPR() - register.GPRC()) - counter.set(counters.txbytes, register.GOTC64()) - counter.set(counters.txpackets, register.GPTC()) - counter.set(counters.txmcast, register.MPTC()) - counter.set(counters.txbcast, register.BPTC()) - counter.set(counters.txdrop, register.TPT() - register.GPTC()) + local counters = self.stats.counters + local s, r, qs = self.stats.s, self.stats.r, self.stats.qs + counter.set(counters.rxbytes, s.GORC64()) + counter.set(counters.rxpackets, s.GPRC()) + local mprc, bprc = s.MPRC(), s.BPRC() + counter.set(counters.rxmcast, mprc + bprc) + counter.set(counters.rxbcast, bprc) + -- The RX receive drop counts are only available through the RX stats + -- register. We only read stats register #0 here. + counter.set(counters.rxdrop, qs.QPRDC[0]()) + counter.set(counters.rxerrors, s.TPR() - s.GPRC()) + counter.set(counters.txbytes, s.GOTC64()) + counter.set(counters.txpackets, s.GPTC()) + local mptc, bptc = s.MPTC(), s.BPTC() + counter.set(counters.txmcast, mptc + bptc) + counter.set(counters.txbcast, bptc) + counter.set(counters.txerrors, s.TPT() - s.GPTC()) + if bit.band(r.LINKS(), link_up_mask) == link_up_mask then + counter.set(counters.status, 1) -- Up + else + counter.set(counters.status, 2) -- Down + end + if bit.band(r.FCTRL(), promisc_mask) ~= 0ULL then + counter.set(counters.promisc, 1) -- True + else + counter.set(counters.promisc, 2) -- False + end end -- Push packets from our 'rx' link onto the network. @@ -163,6 +187,13 @@ function Intel82599:push () local l = self.input.rx if l == nil then return end while not empty(l) and self.dev:can_transmit() do + -- We must not send packets that are bigger than the MTU. This + -- check is currently disabled to satisfy some selftests until + -- agreement on this strategy is reached. + -- if p.length > self.dev.mtu then + -- counter.add(self.stats.counters.txdrop) + -- packet.free(p) + -- else do local p = receive(l) self.dev:transmit(p) --packet.deref(p) From e9f5dda5452ed2bcb1f1394cabf8b1f604357e15 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:37:34 +0200 Subject: [PATCH 043/103] lib.macaddress: add bytes method. --- src/lib/macaddress.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index e55bdcfb3c..0508df17d6 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -46,6 +46,10 @@ function mac_mt:int () return self.bits end +function mac_mt:bytes () + return self.bytes +end + function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From 6b12e9c3e3e45ce2f60bfac0cf399ba555a03f0f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 18:45:40 +0200 Subject: [PATCH 044/103] spin out SNMP code from apps.intel.intel10 to lib.ipc.shmem.iftable_mib. --- src/apps/intel/intel10g.lua | 188 +----------------------------- src/lib/ipc/shmem/iftable_mib.lua | 154 ++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 182 deletions(-) create mode 100644 src/lib/ipc/shmem/iftable_mib.lua diff --git a/src/apps/intel/intel10g.lua b/src/apps/intel/intel10g.lua index 51e2d640fa..b5ec3e3ef3 100644 --- a/src/apps/intel/intel10g.lua +++ b/src/apps/intel/intel10g.lua @@ -6,7 +6,8 @@ --- affordable (~$400) network cards made by Intel and others. --- --- You will need to familiarize yourself with the excellent [data ---- sheet]() to understand this module. +--- sheet](http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82599-10-gbe-controller-datasheet.pdf) +--- to understand this module. module(...,package.seeall) @@ -17,7 +18,6 @@ local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") local index_set = require("lib.index_set") local macaddress = require("lib.macaddress") -local mib = require("lib.ipc.shmem.mib") local timer = require("core.timer") local bits, bitset = lib.bits, lib.bitset @@ -49,10 +49,6 @@ local default = { -- The default MTU allows for an IP packet of a total size of 9000 -- bytes without VLAN tagging. mtu = 9014, - - snmp = { - status_timer = 5, -- Interval for IF status check and MIB update - } } local function pass (...) return ... end @@ -79,7 +75,6 @@ function new_sf (conf) rdh = 0, -- Cache of receive head (RDH) register rdt = 0, -- Cache of receive tail (RDT) register rxnext = 0, -- Index of next buffer to receive - snmp = conf.snmp, } return setmetatable(dev, M_sf) end @@ -118,9 +113,6 @@ end --- See data sheet section 4.6.3 "Initialization Sequence." function M_sf:init () - if self.snmp then - self:init_snmp() - end self:init_dma_memory() self.redos = 0 @@ -186,156 +178,6 @@ do end end -function M_sf:init_snmp () - -- Rudimentary population of a row in the ifTable MIB. Allocation - -- of the ifIndex is delegated to the SNMP agent via the name of - -- the interface in ifDescr (currently the PCI address). - local ifTable = mib:new({ directory = self.snmp.directory or nil, - filename = self.pciaddress }) - self.snmp.ifTable = ifTable - -- ifTable - ifTable:register('ifDescr', 'OctetStr', self.pciaddress) - ifTable:register('ifType', 'Integer32', 6) -- ethernetCsmacd - ifTable:register('ifMtu', 'Integer32', self.mtu) - ifTable:register('ifSpeed', 'Gauge32', 10000000) - - -- After a reset of the NIC, the "native" MAC address is copied to - -- the receive address register #0 from the EEPROM - local ral, rah = self.r.RAL[0](), self.r.RAH[0]() - assert(bit.band(rah, bits({ AV = 31 })) == bits({ AV = 31 }), - "MAC address on "..self.pciaddress.." is not valid ") - local mac = ffi.new("struct { uint32_t lo; uint16_t hi; }") - mac.lo = ral - mac.hi = bit.band(rah, 0xFFFF) - ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }, - ffi.string(mac, 6)) - ifTable:register('ifAdminStatus', 'Integer32', 1) -- up - ifTable:register('ifOperStatus', 'Integer32', 2) -- down - ifTable:register('ifLastChange', 'TimeTicks', 0) - ifTable:register('_X_ifLastChange_TicksBase', 'Counter64', C.get_unix_time()) - ifTable:register('ifInOctets', 'Counter32', 0) - ifTable:register('ifInUcastPkts', 'Counter32', 0) - ifTable:register('ifInDiscards', 'Counter32', 0) - ifTable:register('ifInErrors', 'Counter32', 0) -- TBD - ifTable:register('ifInUnknownProtos', 'Counter32', 0) -- TBD - ifTable:register('ifOutOctets', 'Counter32', 0) - ifTable:register('ifOutUcastPkts', 'Counter32', 0) - ifTable:register('ifOutDiscards', 'Counter32', 0) - ifTable:register('ifOutErrors', 'Counter32', 0) -- TBD - -- ifXTable - ifTable:register('ifName', { type = 'OctetStr', length = 255 }, - self.pciaddress) - ifTable:register('ifInMulticastPkts', 'Counter32', 0) - ifTable:register('ifInBroadcastPkts', 'Counter32', 0) - ifTable:register('ifOutMulticastPkts', 'Counter32', 0) - ifTable:register('ifOutBroadcastPkts', 'Counter32', 0) - ifTable:register('ifHCInOctets', 'Counter64', 0) - ifTable:register('ifHCInUcastPkts', 'Counter64', 0) - ifTable:register('ifHCInMulticastPkts', 'Counter64', 0) - ifTable:register('ifHCInBroadcastPkts', 'Counter64', 0) - ifTable:register('ifHCOutOctets', 'Counter64', 0) - ifTable:register('ifHCOutUcastPkts', 'Counter64', 0) - ifTable:register('ifHCOutMulticastPkts', 'Counter64', 0) - ifTable:register('ifHCOutBroadcastPkts', 'Counter64', 0) - ifTable:register('ifLinkUpDownTrapEnable', 'Integer32', 2) -- disabled - ifTable:register('ifHighSpeed', 'Gauge32', 10000) - ifTable:register('ifPromiscuousMode', 'Integer32', 2) -- false - ifTable:register('ifConnectorPresent', 'Integer32', 1) -- true - ifTable:register('ifAlias', { type = 'OctetStr', length = 64 }, - self.pciaddress) -- TBD add description - ifTable:register('ifCounterDiscontinuityTime', 'TimeTicks', 0) -- TBD - ifTable:register('_X_ifCounterDiscontinuityTime', 'Counter64', 0) -- TBD - - --- Create a timer to periodically check the interface status. - --- Static variables are used in the timer function to avoid - --- garbage. - local status = { [1] = 'up', [2] = 'down' } - local mask = bits{Link_up=30} - local promisc = bits({UPE = 9}) - -- Pre-allocate storage for the results of register-reads - local r = { - in_mcast_pkts = { r = self.s.MPRC }, - in_bcast_pkts = { r = self.s.BPRC }, - in_pkts = { r = self.s.GPRC }, - in_octets64 = { r = self.s.GORC64 }, - - out_mcast_pkts = { r = self.s.MPTC }, - out_bcast_pkts = { r = self.s.BPTC }, - out_pkts = { r = self.s.GPTC }, - out_octets64 = { r = self.s.GOTC64 }, - } - local r_keys = {} - for k, _ in pairs(r) do - table.insert(r_keys, k) - r[k].v = ffi.new("uint64_t [1]") - end - local function read_registers() - for _, k in ipairs(r_keys) do - r[k].v[0] = r[k].r() - end - end - self.logger = lib.logger_new({ module = 'intel10g' }) - local t = timer.new("Interface "..self.pciaddress.." status checker", - function(t) - local old = ifTable:get('ifOperStatus') - local new = 1 - if band(self.r.LINKS(), mask) ~= mask then - new = 2 - end - if old ~= new then - self.logger:log("Interface "..self.pciaddress.. - " status change: "..status[old].. - " => "..status[new]) - ifTable:set('ifOperStatus', new) - ifTable:set('ifLastChange', 0) - ifTable:set('_X_ifLastChange_TicksBase', - C.get_unix_time()) - end - - ifTable:set('ifPromiscuousMode', - (bit.band(self.r.FCTRL(), promisc) ~= 0ULL - and 1) or 2) - -- Update counters - read_registers() - ifTable:set('ifHCInMulticastPkts', r.in_mcast_pkts.v[0]) - ifTable:set('ifInMulticastPkts', r.in_mcast_pkts.v[0]) - ifTable:set('ifHCInBroadcastPkts', r.in_bcast_pkts.v[0]) - ifTable:set('ifInBroadcastPkts', r.in_bcast_pkts.v[0]) - local in_ucast_pkts = r.in_pkts.v[0] - r.in_bcast_pkts.v[0] - - r.in_mcast_pkts.v[0] - ifTable:set('ifHCInUcastPkts', in_ucast_pkts) - ifTable:set('ifInUcastPkts', in_ucast_pkts) - ifTable:set('ifHCInOctets', r.in_octets64.v[0]) - ifTable:set('ifInOctets', r.in_octets64.v[0]) - - ifTable:set('ifHCOutMulticastPkts', r.out_mcast_pkts.v[0]) - ifTable:set('ifOutMulticastPkts', r.out_mcast_pkts.v[0]) - ifTable:set('ifHCOutBroadcastPkts', r.out_bcast_pkts.v[0]) - ifTable:set('ifOutBroadcastPkts', r.out_bcast_pkts.v[0]) - local out_ucast_pkts = r.out_pkts.v[0] - r.out_bcast_pkts.v[0] - - r.out_mcast_pkts.v[0] - ifTable:set('ifHCOutUcastPkts', out_ucast_pkts) - ifTable:set('ifOutUcastPkts', out_ucast_pkts) - ifTable:set('ifHCOutOctets', r.out_octets64.v[0]) - ifTable:set('ifOutOctets', r.out_octets64.v[0]) - - -- The RX receive drop counts are only - -- available through the RX stats register. - -- We only read stats register #0 here. See comment - -- in init_statistics() - ifTable:set('ifInDiscards', self.qs.QPRDC[0]()) - - ifTable:set('ifInErrors', self.s.CRCERRS() + - self.s.ILLERRC() + self.s.ERRBC() + - self.s.RUC() + self.s.RFC() + - self.s.ROC() + self.s.RJC()) - end, - 1e9 * (self.snmp.status_timer or - default.snmp.status_timer), 'repeating') - timer.activate(t) - return self -end - function M_sf:global_reset () local reset = bits{LinkReset=3, DeviceReset=26} self.r.CTRL(reset) @@ -448,22 +290,10 @@ end local txdesc_flags = bits{ifcs=25, dext=29, dtyp0=20, dtyp1=21, eop=24} function M_sf:transmit (p) - -- We must not send packets that are bigger than the MTU. This - -- check is currently disabled to satisfy some selftests until - -- agreement on this strategy is reached. - -- if p.length > self.mtu then - -- if self.snmp then - -- local errors = self.snmp.ifTable:ptr('ifOutDiscards') - -- errors[0] = errors[0] + 1 - -- end - -- packet.free(p) - -- else - do - self.txdesc[self.tdt].address = memory.virtual_to_physical(p.data) - self.txdesc[self.tdt].options = bor(p.length, txdesc_flags, lshift(p.length+0ULL, 46)) - self.txpackets[self.tdt] = p - self.tdt = band(self.tdt + 1, num_descriptors - 1) - end + self.txdesc[self.tdt].address = memory.virtual_to_physical(p.data) + self.txdesc[self.tdt].options = bor(p.length, txdesc_flags, lshift(p.length+0ULL, 46)) + self.txpackets[self.tdt] = p + self.tdt = band(self.tdt + 1, num_descriptors - 1) end function M_sf:sync_transmit () @@ -649,7 +479,6 @@ function new_pf (conf) mac_set = index_set:new(127, "MAC address table"), vlan_set = index_set:new(64, "VLAN Filter table"), mirror_set = index_set:new(4, "Mirror pool table"), - snmp = conf.snmp, } return setmetatable(dev, M_pf) end @@ -675,9 +504,6 @@ function M_pf:close() end function M_pf:init () - if self.snmp then - self:init_snmp() - end self.redos = 0 local mask = bits{Link_up=30} for i = 1, 100 do @@ -703,7 +529,6 @@ function M_pf:init () return self end -M_pf.init_snmp = M_sf.init_snmp M_pf.global_reset = M_sf.global_reset M_pf.disable_interrupts = M_sf.disable_interrupts M_pf.set_receive_descriptors = pass @@ -784,7 +609,6 @@ function M_pf:new_vf (poolnum) base = self.base, -- mmap()ed register file s = self.s, -- Statistics registers mtu = self.mtu, - snmp = self.snmp, -- and others are our own r = {}, -- Configuration registers poolnum = poolnum, diff --git a/src/lib/ipc/shmem/iftable_mib.lua b/src/lib/ipc/shmem/iftable_mib.lua new file mode 100644 index 0000000000..0f09aef0aa --- /dev/null +++ b/src/lib/ipc/shmem/iftable_mib.lua @@ -0,0 +1,154 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) +local mib = require("lib.ipc.shmem.mib") +local counter = require("core.counter") +local macaddress = require("lib.macaddress") +local ffi = require("ffi") + +local iftypes = { + [0x1000] = 6, -- ethernetCsmacd + [0x1001] = 53, -- propVirtual +} + +function init_snmp (name, counters, directory, interval) + -- Rudimentary population of a row in the ifTable MIB. Allocation + -- of the ifIndex is delegated to the SNMP agent via the name of + -- the interface in ifDescr. + local ifTable = mib:new({ directory = directory or nil, + filename = name }) + -- ifTable + ifTable:register('ifDescr', 'OctetStr', name) + ifTable:register('ifType', 'Integer32') + if counters.type then + ifTable:set('ifType', iftypes[counter.read(counters.type)] or 1) -- other + end + ifTable:register('ifMtu', 'Integer32') + if counters.mtu then + ifTable:set('ifMtu', counter.read(counters.mtu)) + end + ifTable:register('ifSpeed', 'Gauge32') + ifTable:register('ifHighSpeed', 'Gauge32') + if counters.speed then + ifTable:set('ifSpeed', counter.read(counters.speed)) + ifTable:set('ifHighSpeed', counter.read(counters.speed) / 1000) + end + ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }) + if counters.macaddr then + local mac = macaddress:new(counter.read(counters.macaddr)) + ifTable:set('ifPhysAddress', ffi.string(mac:bytes(), 6)) + end + ifTable:register('ifAdminStatus', 'Integer32', 1) -- up + ifTable:register('ifOperStatus', 'Integer32', 2) -- down + ifTable:register('ifLastChange', 'TimeTicks', 0) + ifTable:register('_X_ifLastChange_TicksBase', 'Counter64', + C.get_unix_time()) + ifTable:register('ifInOctets', 'Counter32', 0) + ifTable:register('ifInUcastPkts', 'Counter32', 0) + ifTable:register('ifInDiscards', 'Counter32', 0) + ifTable:register('ifInErrors', 'Counter32', 0) -- TBD + ifTable:register('ifInUnknownProtos', 'Counter32', 0) -- TBD + ifTable:register('ifOutOctets', 'Counter32', 0) + ifTable:register('ifOutUcastPkts', 'Counter32', 0) + ifTable:register('ifOutDiscards', 'Counter32', 0) + ifTable:register('ifOutErrors', 'Counter32', 0) -- TBD + -- ifXTable + ifTable:register('ifName', { type = 'OctetStr', length = 255 }, name) + ifTable:register('ifInMulticastPkts', 'Counter32', 0) + ifTable:register('ifInBroadcastPkts', 'Counter32', 0) + ifTable:register('ifOutMulticastPkts', 'Counter32', 0) + ifTable:register('ifOutBroadcastPkts', 'Counter32', 0) + ifTable:register('ifHCInOctets', 'Counter64', 0) + ifTable:register('ifHCInUcastPkts', 'Counter64', 0) + ifTable:register('ifHCInMulticastPkts', 'Counter64', 0) + ifTable:register('ifHCInBroadcastPkts', 'Counter64', 0) + ifTable:register('ifHCOutOctets', 'Counter64', 0) + ifTable:register('ifHCOutUcastPkts', 'Counter64', 0) + ifTable:register('ifHCOutMulticastPkts', 'Counter64', 0) + ifTable:register('ifHCOutBroadcastPkts', 'Counter64', 0) + ifTable:register('ifLinkUpDownTrapEnable', 'Integer32', 2) -- disabled + ifTable:register('ifPromiscuousMode', 'Integer32', 2) -- false + ifTable:register('ifConnectorPresent', 'Integer32', 1) -- true + ifTable:register('ifAlias', { type = 'OctetStr', length = 64 }, + name) -- TBD add description + ifTable:register('ifCounterDiscontinuityTime', 'TimeTicks', 0) + ifTable:register('_X_ifCounterDiscontinuityTime', 'Counter64') + if counters.dtime then + ifTable:set('_X_ifCounterDiscontinuityTime', counter.read(counters.dtime)) + end + + local logger = lib.logger_new({ module = 'iftable_mib' }) + local function t () + local old, new + if counters.status then + old = ifTable:get('ifOperStatus') + new = counter.read(counters.status) + else + new = 1 + end + if old ~= new then + logger:log("Interface "..name.. + " status change: "..status[old].." => "..status[new]) + ifTable:set('ifOperStatus', new) + ifTable:set('ifLastChange', 0) + ifTable:set('_X_ifLastChange_TicksBase', C.get_unix_time()) + end + + if counters.promisc then + ifTable:set('ifPromiscuousMode', counter.read(counters.promisc)) + end + -- Update counters + if counters.rxpackets and counters.rxmcast and counters.rxbcast then + local rxbcast = counter.read(counters.rxbcast) + local rxmcast = counter.read(counters.rxmcast) + local rxpackets = counter.read(counters.rxpackets) + local inMcast = rxmcast - rxbcast + local inUcast = rxpackets - rxmcast + ifTable:set('ifHCInMulticastPkts', inMcast) + ifTable:set('ifInMulticastPkts', inMcast) + ifTable:set('ifHCInBroadcastPkts', rxbcast) + ifTable:set('ifInBroadcastPkts', rxbcast) + ifTable:set('ifHCInUcastPkts', inUcast) + ifTable:set('ifInUcastPkts', inUcast) + end + if counters.rxbytes then + local rxbytes = counter.read(counters.rxbytes) + ifTable:set('ifHCInOctets', rxbytes) + ifTable:set('ifInOctets', rxbytes) + end + if counters.rxdrop then + ifTable:set('ifInDiscards', counter.read(counters.rxdrop)) + end + if counters.rxerrors then + ifTable:set('ifInErrors', counter.read(counters.rxerrors)) + end + if counters.txpackets and counters.txmcast and counters.txbcast then + local txbcast = counter.read(counters.txbcast) + local txmcast = counter.read(counters.txmcast) + local txpackets = counter.read(counters.txpackets) + local outMcast = txmcast - txbcast + local outUcast = txpackets - txmcast + ifTable:set('ifHCOutMulticastPkts', outMcast) + ifTable:set('ifOutMulticastPkts', outMcast) + ifTable:set('ifHCOutBroadcastPkts', txbcast) + ifTable:set('ifOutBroadcastPkts', txbcast) + ifTable:set('ifHCOutUcastPkts', outUcast) + ifTable:set('ifOutUcastPkts', outUcast) + end + if counters.txbytes then + local txbytes = counter.read(counters.txbytes) + ifTable:set('ifHCOutOctets', txbytes) + ifTable:set('ifOutOctets', txbytes) + end + if counters.txdrop then + ifTable:set('ifOutDiscards', counter.read(counters.txdrop)) + end + if counters.txerrors then + ifTable:set('ifOutErrors', counter.read(counters.txerrors)) + end + end + local t = timer.new("Interface "..name.." status checker", + t, 1e9 * (interval or 5), 'repeating') + timer.activate(t) + return t +end From 7c697b6775ee8bae9d488710c806eb62a3871a1a Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 3 Jun 2016 18:07:55 +0200 Subject: [PATCH 045/103] main: fork into worker and supervisor; perform clean up. --- src/core/main.lua | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index eac20ec884..d8998eb701 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -10,10 +10,11 @@ local STP = require("lib.lua.StackTracePlus") local ffi = require("ffi") local zone = require("jit.zone") local lib = require("core.lib") +local shm = require("core.shm") local C = ffi.C -- Load ljsyscall early to help detect conflicts -- (e.g. FFI type name conflict between Snabb and ljsyscall) -require("syscall") +local S = require("syscall") require("lib.lua.strict") require("lib.lua.class") @@ -44,10 +45,30 @@ function main () print("unsupported program: "..program:gsub("_", "-")) usage(1) else - require(modulename(program)).run(args) + -- Fork into worker process and supervisor + local worker_pid = S.fork() + if worker_pid == 0 then + -- Worker + S.prctl("set_pdeathsig", "hup") + require(modulename(program)).run(args) + else + -- Supervisor + local exit_signals = "hup, int, quit, term, chld" + local signalfd = S.signalfd(exit_signals) + S.sigprocmask("block", exit_signals) + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + if not signals[1].chld then S.kill(worker_pid, "kill") end + shutdown(S.getpid()) + end end end +-- Cleanup after worker process. +function shutdown (pid) + shm.unlink("//"..pid) +end + -- Take the program name from the first argument, unless the first -- argument is "snabb", in which case pop it off, handle any options -- passed to snabb itself, and use the next argument. From be09fe5ca54ef27cc02d0b3504e247aa121a3d36 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 16:04:55 +0200 Subject: [PATCH 046/103] main: send HUP instead of KILL. --- src/core/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index d8998eb701..7e7d4e2e0a 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -58,7 +58,7 @@ function main () S.sigprocmask("block", exit_signals) local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) - if not signals[1].chld then S.kill(worker_pid, "kill") end + if not signals[1].chld then S.kill(worker_pid, "hup") end shutdown(S.getpid()) end end From 3db174bb05fd7b4947cf61186c898a9d669494fd Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 16:35:56 +0200 Subject: [PATCH 047/103] main: return correct exit code. --- src/core/main.lua | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index 7e7d4e2e0a..bde7574b39 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -58,8 +58,17 @@ function main () S.sigprocmask("block", exit_signals) local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) - if not signals[1].chld then S.kill(worker_pid, "hup") end + local exit_status + if signals[1].chld then + local _, _, worker = S.waitpid(worker_pid) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end + else + S.kill(worker_pid, "hup") + exit_status = 128 + signals[1].signo + end shutdown(S.getpid()) + os.exit(exit_status) end end end From ce2f2206872f881ee0a1e22c13a07261476a3803 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:00:30 +0200 Subject: [PATCH 048/103] main: supervise xpcall(main, hanlder). --- src/core/main.lua | 62 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index bde7574b39..cb312d2cb4 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -45,39 +45,10 @@ function main () print("unsupported program: "..program:gsub("_", "-")) usage(1) else - -- Fork into worker process and supervisor - local worker_pid = S.fork() - if worker_pid == 0 then - -- Worker - S.prctl("set_pdeathsig", "hup") - require(modulename(program)).run(args) - else - -- Supervisor - local exit_signals = "hup, int, quit, term, chld" - local signalfd = S.signalfd(exit_signals) - S.sigprocmask("block", exit_signals) - local signals, err = S.util.signalfd_read(signalfd) - assert(signals, tostring(err)) - local exit_status - if signals[1].chld then - local _, _, worker = S.waitpid(worker_pid) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end - else - S.kill(worker_pid, "hup") - exit_status = 128 + signals[1].signo - end - shutdown(S.getpid()) - os.exit(exit_status) - end + require(modulename(program)).run(args) end end --- Cleanup after worker process. -function shutdown (pid) - shm.unlink("//"..pid) -end - -- Take the program name from the first argument, unless the first -- argument is "snabb", in which case pop it off, handle any options -- passed to snabb itself, and use the next argument. @@ -160,6 +131,11 @@ function handler (reason) os.exit(1) end +-- Cleanup after Snabb process. +function shutdown (pid) + shm.unlink("//"..pid) +end + function selftest () print("selftest") assert(programname("/bin/snabb-1.0") == "snabb", @@ -184,4 +160,28 @@ function selftest () "Incorrect program name selected") end -xpcall(main, handler) +-- Fork into worker process and supervisor +local worker_pid = S.fork() +if worker_pid == 0 then + -- Worker + S.prctl("set_pdeathsig", "hup") + xpcall(main, handler) +else + -- Supervisor + local exit_signals = "hup, int, quit, term, chld" + local signalfd = S.signalfd(exit_signals) + S.sigprocmask("block", exit_signals) + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + local exit_status + if signals[1].chld then + local _, _, worker = S.waitpid(worker_pid) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end + else + S.kill(worker_pid, "hup") + exit_status = 128 + signals[1].signo + end + shutdown(S.getpid()) + os.exit(exit_status) +end From 4950cbb7d4beb3c69bb81a4f6412b9951bc583fb Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:03:27 +0200 Subject: [PATCH 049/103] main: check status of S.waitpid. --- src/core/main.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index cb312d2cb4..7369639993 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -175,7 +175,8 @@ else assert(signals, tostring(err)) local exit_status if signals[1].chld then - local _, _, worker = S.waitpid(worker_pid) + local status, err, worker = S.waitpid(worker_pid) + assert(status, tostring(err)) if worker.WIFEXITED then exit_status = worker.EXITSTATUS else exit_status = 128 + worker.WTERMSIG end else From 28c568c4526f1848f25078494dc694f67610aab1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:06:31 +0200 Subject: [PATCH 050/103] main: cleanup after worker_pid. --- src/core/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index 7369639993..0a27c076e5 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -183,6 +183,6 @@ else S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo end - shutdown(S.getpid()) + shutdown(worker_pid) os.exit(exit_status) end From 4af7091f01939ef462480b39567eaa366fbb236f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:19:41 +0200 Subject: [PATCH 051/103] Remove obsolete `snabb gc'. --- src/program/gc/README | 7 ------- src/program/gc/README.inc | 1 - src/program/gc/gc.lua | 29 ----------------------------- 3 files changed, 37 deletions(-) delete mode 100644 src/program/gc/README delete mode 120000 src/program/gc/README.inc delete mode 100644 src/program/gc/gc.lua diff --git a/src/program/gc/README b/src/program/gc/README deleted file mode 100644 index 703a1ef95b..0000000000 --- a/src/program/gc/README +++ /dev/null @@ -1,7 +0,0 @@ -Usage: - gc [OPTIONS] - - -h, --help - Print usage information. - -Remove stale runtime files created by Snabb instances. diff --git a/src/program/gc/README.inc b/src/program/gc/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/gc/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/gc/gc.lua b/src/program/gc/gc.lua deleted file mode 100644 index d3de0a12e1..0000000000 --- a/src/program/gc/gc.lua +++ /dev/null @@ -1,29 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - -module(..., package.seeall) - -local lib = require("core.lib") -local shm = require("core.shm") -local syscall = require("syscall") -local usage = require("program.gc.README_inc") - -local long_opts = { - help = "h" -} - -function run (args) - local opt = {} - function opt.h (arg) print(usage) main.exit(1) end - args = lib.dogetopt(args, opt, "h", long_opts) - - if #args > 0 then print(usage) main.exit(1) end - - -- Unlink stale snabb resources. - for _, pid in ipairs(shm.children("//")) do - if not syscall.kill(tonumber(pid), 0) then - shm.unlink("//"..pid) - end - end - -- Unlink own resource - shm.unlink("//"..syscall.getpid()) -end From 1c9f58585f7480df16ce3ffe532a1d99286ff1ea Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 15:47:49 +0200 Subject: [PATCH 052/103] Move model.txt to src/doc/statistics.md. --- model.txt | 48 ----------------------------------- src/doc/statistics.md | 58 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 48 deletions(-) delete mode 100644 model.txt create mode 100644 src/doc/statistics.md diff --git a/model.txt b/model.txt deleted file mode 100644 index 8f4e128823..0000000000 --- a/model.txt +++ /dev/null @@ -1,48 +0,0 @@ -RFC7223 Snabb ifTable MIB -======= ===== =========== -name -description? -type type (~= identity) -enabled? -link-up-down-trap-enable? -admin-status -oper-status status (= enum) ifOperStatus -last-change? -if-index -phys-address? macaddr (uint64) ifPhysAddress -higher-layer-if* -lower-layer-if* -speed? speed ifSpeed -discontinuity-time dtime (seconds since epoch) ifCounterDiscontinuityTime -in-octets? rxbytes ifInOctets - rxpackets -in-unicast-pkts? -in-broadcast-pkts? rxbcast ifInBroadcastPkts - rxmcast -in-multicast-pkts? ifInMulticastPkts -in-discards? rxdrop ifInDiscards -in-errors? rxerrors ifInErrors -in-unknown-protos? -out-octets? txbytes ifOutOctets - txpackets -out-unicast-pkts? -out-broadcast-pkts? txbcast ifOutBroadcastPkts - txmcast -out-multicast-pkts? ifOutMulticastPkts -out-discards? txdrop ifOutDiscards -out-errors? txerrors ifOutErrors - rxpackets - txpackets - rxmcast - txmcast - mtu ifMtu - promisc ifPromiscuousMode - -snabb/type: - -0x1000 Hardware interface -0x1001 Virtual interface - -snabb/bcast, mcast, packets: - -(tx/rx) - bcast ⊆ mcast ⊆ packets diff --git a/src/doc/statistics.md b/src/doc/statistics.md new file mode 100644 index 0000000000..8ba1566a60 --- /dev/null +++ b/src/doc/statistics.md @@ -0,0 +1,58 @@ +### Statistics counters in Snabb + +Below is list of statistics counters defined by Snabb, and their relation to +RFC 7223 and ifTable MIB. All counters are unsigned 64bit integers. Each Snabb +app can optionally implement any number of these counters. + +| Snabb | RFC 7223 | ifTable MIB +| ----- | -------- | ----------- +| | name | +| | description? | ifDescr +| type (~= identity) | type | ifType +| | enabled? | +| | link-up-down-trap-enable? | +| | admin-status | +| status (= enum) | oper-status | ifOperStatus +| | last-change? | ifLastChange +| | if-index | +| macaddr (uint64) | phys-address? | ifPhysAddress +| | higher-layer-if* | +| | lower-layer-if* | +| speed | speed? | ifSpeed +| dtime (seconds since epoch) | discontinuity-time | ifCounterDiscontinuityTime +| rxbytes | in-octets? | ifInOctets +| rxpackets | | +| | in-unicast-pkts? | +| rxbcast | in-broadcast-pkts? | ifInBroadcastPkts +| rxmcast | | +| | in-multicast-pkts? | ifInMulticastPkts +| rxdrop | in-discards? | ifInDiscards +| rxerrors | in-errors? | ifInErrors +| | in-unknown-protos? | +| txbytes | out-octets? | ifOutOctets +| txpackets | | +| | out-unicast-pkts? | +| txbcast | out-broadcast-pkts? | ifOutBroadcastPkts +| txmcast | | +| | out-multicast-pkts? | ifOutMulticastPkts +| txdrop | out-discards? | ifOutDiscards +| txerrors | out-errors? | ifOutErrors +| rxpackets | | +| txpackets | | +| rxmcast | | +| txmcast | | +| mtu | | ifMtu +| promisc | | ifPromiscuousMode + + +#### type + + - `0x1000` Hardware interface + - `0x1001` Virtual interface + +#### rxbcast, rxmcast, rxpackets, txbcast, txmcast, txpackets + +Snabb defines total packet counts, multicast packet counts (packets with group +bit set) and broadcast packet counts for both input and output. + + E.g.: (tx/rx)bcast ⊆ (tx/rx)mcast ⊆ (tx/rx)packets From 339f8f1951256bb63273e47089aa8230b432346d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 16:27:24 +0200 Subject: [PATCH 053/103] apps.intel_app: Revert rxerrors calculation, remove broken txerrors. --- src/apps/intel/intel_app.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 623fb1a3a3..0e463d1e32 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -163,13 +163,13 @@ function Intel82599:sync_stats () -- The RX receive drop counts are only available through the RX stats -- register. We only read stats register #0 here. counter.set(counters.rxdrop, qs.QPRDC[0]()) - counter.set(counters.rxerrors, s.TPR() - s.GPRC()) + counter.set(counters.rxerrors, s.CRCERRS() + s.ILLERRC() + s.ERRBC() + + s.RUC() + s.RFC() + s.ROC() + s.RJC()) counter.set(counters.txbytes, s.GOTC64()) counter.set(counters.txpackets, s.GPTC()) local mptc, bptc = s.MPTC(), s.BPTC() counter.set(counters.txmcast, mptc + bptc) counter.set(counters.txbcast, bptc) - counter.set(counters.txerrors, s.TPT() - s.GPTC()) if bit.band(r.LINKS(), link_up_mask) == link_up_mask then counter.set(counters.status, 1) -- Up else From 7ec7f807d15484f531ca31b56d671e0939b06fb4 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 16:30:57 +0200 Subject: [PATCH 054/103] Revert "lib.macaddress: add bytes method." and remove int method. This reverts commit e9f5dda5452ed2bcb1f1394cabf8b1f604357e15. --- src/apps/intel/intel_app.lua | 4 ++-- src/lib/ipc/shmem/iftable_mib.lua | 2 +- src/lib/macaddress.lua | 8 -------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 0e463d1e32..fc5829c723 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -77,7 +77,7 @@ function Intel82599:new (arg) counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then counter.set(self.stats.counters.macaddr, - macaddress:new(conf.macaddr):int()) + macaddress:new(conf.macaddr).bits) end end return setmetatable(self, Intel82599) @@ -118,7 +118,7 @@ function Intel82599:reconfig(arg) if not self.dev.pf and conf.macaddr then counter.set(self.stats.counters.macaddr, - macaddress:new(conf.macaddr):int()) + macaddress:new(conf.macaddr).bits) end end diff --git a/src/lib/ipc/shmem/iftable_mib.lua b/src/lib/ipc/shmem/iftable_mib.lua index 0f09aef0aa..8d96d29e67 100644 --- a/src/lib/ipc/shmem/iftable_mib.lua +++ b/src/lib/ipc/shmem/iftable_mib.lua @@ -36,7 +36,7 @@ function init_snmp (name, counters, directory, interval) ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }) if counters.macaddr then local mac = macaddress:new(counter.read(counters.macaddr)) - ifTable:set('ifPhysAddress', ffi.string(mac:bytes(), 6)) + ifTable:set('ifPhysAddress', ffi.string(mac.bytes, 6)) end ifTable:register('ifAdminStatus', 'Integer32', 1) -- up ifTable:register('ifOperStatus', 'Integer32', 2) -- down diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index 0508df17d6..a973b1d7b3 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -42,14 +42,6 @@ function mac_mt.__eq (a, b) return a.bits == b.bits end -function mac_mt:int () - return self.bits -end - -function mac_mt:bytes () - return self.bytes -end - function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From e5f23a953d102d5754253da5a44c71bbc7c2f08a Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Wed, 8 Jun 2016 05:32:36 +0000 Subject: [PATCH 055/103] vhost_user: Fix & comment "feature cache" Added a long description of the "feature cache" and how it supports reconnecting to a running QEMU. Added a restriction so that the cache is only considered for the first feature negotiation performed by a vhost-user app instance. This is to avoid using the cached features when renegotiating with a guest e.g. when the guest has switched to a new driver. Not yet tested. --- src/apps/vhost/vhost_user.lua | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 7890b495f5..752dfb5142 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -176,13 +176,49 @@ function VhostUser:set_features (msg) self.dev:set_features(features) end +-- Feature cache: A kludge to be compatible with a "QEMU reconnect" patch. +-- +-- QEMU upstream (circa 2015) does not support the vhost-user device +-- (Snabb) reconnecting to QEMU. That is unfortunate because being +-- able to reconnect after a restart of either the Snabb process or +-- simply a vhost-user app is very practical. +-- +-- Reconnect support can however be enabled in QEMU with a small patch +-- [1]. Caveat: Feature negotiation does not work reliably on the new +-- connections and may provide an invalid feature list. Workaround: +-- Cache the most recently negotiated features for each vhost-user +-- socket and reuse those when available. +-- +-- This is far from perfect but it is better than nothing. +-- Reconnecting to QEMU VMs is very practical and enables faster +-- development, restart of the Snabb process for recovery or upgrade, +-- and stop/start of vhost-user app instances e.g. due to +-- configuration changes. +-- +-- QEMU upstream seem to be determined to solve the reconnect problem +-- by requiring changes to the guest drivers so that the device could +-- request a reset. However, this has the undesirable properties that +-- it will not be transparent to the guest and nor will it work on +-- existing guest drivers. +-- +-- And so for now we have this cache for people who want to patch +-- reconnect support into their QEMU... +-- +-- 1: QEMU patch: +-- https://github.com/SnabbCo/qemu/commit/f393aea2301734647fdf470724433f44702e3fb9.patch + +-- Consider using virtio-net feature cache to override negotiated features. function VhostUser:update_features (features) local stat = syscall.stat(self.socket_path) local mtime = ("%d.%d"):format(tonumber(stat.st_mtime), tonumber(stat.st_mtime_nsec)) local cachepath = "/tmp/vhost_features_"..string.gsub(self.socket_path, "/", "__") local f = io.open(cachepath, 'r') - if f then + -- Use cached features when: + -- Negotiating features for the first time for this app instance + -- Cache is populated + -- QEMU vhost-user socket file has same timestamp as cache + if not self.have_negotiated_features and f then local file_features, file_mtime = f:read('*a'):match("features:(.*) mtime:(.*)\n") f:close() if file_mtime == mtime then @@ -193,6 +229,12 @@ function VhostUser:update_features (features) print(("vhost_user: Skipped old feature cache in %s"):format(cachepath)) end end + -- Features are now negotiated for this app instance. If they are + -- negotiated again it will presumably be due to guest driver + -- restart and in that case we should trust the new features rather + -- than overriding them with the cache. + self.have_negotiated_features = true + -- Cache features after they are negotiated f = io.open(cachepath, 'w') if f then print(("vhost_user: Caching features (0x%s) in %s"):format( From b20cb17fc761db7e637503904b95398b1f368ab0 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 24 May 2016 08:29:44 +0000 Subject: [PATCH 056/103] core.memory: Add 'align' argument to dma_alloc() Now it is possible to request specific alignment for DMA memory. This is practical. For example, Mellanox ConnectX-4 requires specific alignments (e.g. 4KB). (cherry picked from commit f349c41973af27fbbab431736b7fb4ddc512aa8e) --- src/README.md | 5 ++++- src/core/memory.lua | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/README.md b/src/README.md index ea12be3741..16ef85d128 100644 --- a/src/README.md +++ b/src/README.md @@ -370,10 +370,13 @@ can be accessed directly by network cards. The important characteristic of DMA memory is being located in contiguous physical memory at a stable address. -— Function **memory.dma_alloc** *bytes* +— Function **memory.dma_alloc** *bytes*, *[alignment]* Returns a pointer to *bytes* of new DMA memory. +Optionally a specific *alignment* requirement can be provided (in +bytes). The default alignment is 128. + — Function **memory.virtual_to_physical** *pointer* Returns the physical address (`uint64_t`) the DMA memory at *pointer*. diff --git a/src/core/memory.lua b/src/core/memory.lua index 046eaa3091..ecfba556e1 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -23,13 +23,20 @@ chunks = {} -- Allocate DMA-friendly memory. -- Return virtual memory pointer, physical address, and actual size. -function dma_alloc (bytes) +function dma_alloc (bytes, align) + align = align or 128 assert(bytes <= huge_page_size) - bytes = lib.align(bytes, 128) - if #chunks == 0 or bytes + chunks[#chunks].used > chunks[#chunks].size then + -- Get current chunk of memory to allocate from + if #chunks == 0 then allocate_next_chunk() end + local chunk = chunks[#chunks] + -- Skip allocation forward pointer to suit alignment + chunk.used = lib.align(chunk.used, align) + -- Need a new chunk to service this allocation? + if chunk.used + bytes > chunk.size then allocate_next_chunk() + chunk = chunks[#chunks] end - local chunk = chunks[#chunks] + -- Slice out the memory we need local where = chunk.used chunk.used = chunk.used + bytes return chunk.pointer + where, chunk.physical + where, bytes From 327804f56d2512cce7967b26fe8bbab726178aaf Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 9 Jun 2016 17:09:14 +0200 Subject: [PATCH 057/103] doc/statistics.md: remove duplicates. --- src/doc/statistics.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/doc/statistics.md b/src/doc/statistics.md index 8ba1566a60..852d77829f 100644 --- a/src/doc/statistics.md +++ b/src/doc/statistics.md @@ -37,10 +37,6 @@ app can optionally implement any number of these counters. | | out-multicast-pkts? | ifOutMulticastPkts | txdrop | out-discards? | ifOutDiscards | txerrors | out-errors? | ifOutErrors -| rxpackets | | -| txpackets | | -| rxmcast | | -| txmcast | | | mtu | | ifMtu | promisc | | ifPromiscuousMode From 90fc893bef4caa2260793b672ad8ef29f1d7591c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 13:00:02 +0200 Subject: [PATCH 058/103] core.app: fix bug in #766 introduced while merging 6cac870a. --- src/core/app.lua | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index dbcc7f7fa9..94d08a0e23 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -72,20 +72,21 @@ end function with_restart (app, method) local oldshm = shm.path shm.path = app.shmpath + local status, result if use_restart then -- Run fn in protected mode using pcall. - local status, result_or_error = pcall(method, app) + status, result = pcall(method, app) -- If pcall caught an error mark app as "dead" (record time and cause -- of death). if not status then - app.dead = { error = result_or_error, time = now() } + app.dead = { error = result, time = now() } end - return status, result_or_error else - return true, method(app) + status, result = true, method(app) end shm.path = oldshm + return status, result_or_error end -- Restart dead apps. From ceb080f6dd40ea5d7173197d2e06e3fc6e16dcf1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 13:10:20 +0200 Subject: [PATCH 059/103] core.app: fix bug in #766, use app.shmpath during reconfig. --- src/core/app.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index 94d08a0e23..38a1973bf3 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -206,7 +206,10 @@ function apply_config_actions (actions, conf) if app_table[name].reconfig then local arg = conf.apps[name].arg local app = app_table[name] + local shmorig = shm.path + shm.path = app.shmpath app:reconfig(arg) + shm.path = shmorig new_app_table[name] = app table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array From 738ef3132bffd49a7f1d7375952ba139cabd27ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:09:49 +0100 Subject: [PATCH 060/103] Snabb version is now statically stored in .version --- .version | 1 + default.nix | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 .version diff --git a/.version b/.version new file mode 100644 index 0000000000..d1ff275cc5 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +2016.06 diff --git a/default.nix b/default.nix index a71304ed3a..674ccb44de 100644 --- a/default.nix +++ b/default.nix @@ -3,15 +3,15 @@ # ... and the files are produced in ./result/ { pkgs ? (import {}) +, src ? ./. +, version ? builtins.replaceStrings ["\n"] [""] (builtins.readFile ./.version) }: with pkgs; stdenv.mkDerivation rec { - # TODO: get the version from somewhere? - name = "snabb"; - - src = ./.; + name = "snabb-${version}"; + inherit src version; buildInputs = [ makeWrapper ]; From a119aebe2e1c7499d0a208397ffd81e5b77840cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:10:41 +0100 Subject: [PATCH 061/103] release.nix: don't rely on SNABB_PCI* defaults --- release.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/release.nix b/release.nix index 8501d4a488..1d249cbe28 100644 --- a/release.nix +++ b/release.nix @@ -13,9 +13,9 @@ let url = "http://lab1.snabb.co:2008/~max/assets/vm-ubuntu-trusty-14.04-dpdk-snabb.tar.gz"; sha256 = "0323591i925jhd6wv8h268wc3ildjpa6j57n4p9yg9d6ikwkw06j"; }; - optionalGetEnv = first: default: let - maybeEnv = builtins.getEnv first; - in if (maybeEnv != "") then maybeEnv else default; + requiredGetEnv = var: let + maybeEnv = builtins.getEnv var; + in if (maybeEnv != "") then maybeEnv else throw "Please export shell variable ${var}"; in rec { manual = import ./src/doc {}; snabb = import ./default.nix {}; @@ -45,10 +45,9 @@ in rec { doCheck = true; checkPhase = '' - export SNABB_PCI0=${ optionalGetEnv "SNABB_PCI0" "0000:01:00.0"} - export SNABB_PCI_INTEL0=${ optionalGetEnv "SNABB_PCI_INTEL0" "0000:01:00.0"} - export SNABB_PCI_INTEL1=${ optionalGetEnv "SNABB_PCI_INTEL1" "0000:01:00.1"} - export FAIL_ON_FIRST=true + export SNABB_PCI0=${ requiredGetEnv "SNABB_PCI0" } + export SNABB_PCI_INTEL0=${ requiredGetEnv "SNABB_PCI_INTEL0" } + export SNABB_PCI_INTEL1=${ requiredGetEnv "SNABB_PCI_INTEL1" } # run tests sudo -E make test -C src/ From 1feb2c9def6b378cebc8e22e9654f09d4e293095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:11:00 +0100 Subject: [PATCH 062/103] release.nix: by default use lugano servers --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 1d249cbe28..0371e72d08 100644 --- a/release.nix +++ b/release.nix @@ -26,7 +26,7 @@ in rec { # allow sudo __noChroot = true; - requiredSystemFeatures = [ "performance" ]; + requiredSystemFeatures = [ "lugano" ]; buildInputs = [ git telnet tmux numactl bc iproute which qemu ]; From 307b1e170b9a3a03320646919038c0fd8ec3cb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:18:00 +0100 Subject: [PATCH 063/103] default.nix: remove .git from source since it's not deterministic --- default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 674ccb44de..20bccef8c0 100644 --- a/default.nix +++ b/default.nix @@ -3,7 +3,7 @@ # ... and the files are produced in ./result/ { pkgs ? (import {}) -, src ? ./. +, source ? ./. , version ? builtins.replaceStrings ["\n"] [""] (builtins.readFile ./.version) }: @@ -11,7 +11,8 @@ with pkgs; stdenv.mkDerivation rec { name = "snabb-${version}"; - inherit src version; + inherit version; + src = lib.cleanSource source; buildInputs = [ makeWrapper ]; From 1d4e766fa2dccc62c0b35ebf904aebb8eae8540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:33:52 +0100 Subject: [PATCH 064/103] src/Makefile: silence grep Otherwise it prints misleading errors: grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 610186ce0b..467b88e9eb 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,7 +39,7 @@ EXE := bin/snabb $(patsubst %,bin/%,$(PROGRAM)) # core.memory core.lib ... # for each module that has a top-level selftest () function. TESTMODS = $(shell find . -regex '[^\#]*\.lua' -printf '%P ' | \ - xargs grep -l '^function selftest *[[:punct:]]' | \ + xargs grep -s -l '^function selftest *[[:punct:]]' | \ sed -e 's_\.lua__' -e 's_/_._g') # TESTSCRIPTS expands to: From d82729abf465c0e9a800e7de276d00f144c04b3a Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 14:41:58 +0200 Subject: [PATCH 065/103] core.app: fix typo in previous commit. --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 38a1973bf3..5d7fa0e8fd 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -86,7 +86,7 @@ function with_restart (app, method) status, result = true, method(app) end shm.path = oldshm - return status, result_or_error + return status, result end -- Restart dead apps. From 70b89675c74796750c21236c6a752bd7bd29fb34 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 13 Jun 2016 15:55:11 +0200 Subject: [PATCH 066/103] intel1g: canonicalize env var name, document it in testing.md, pass it through dock.sh. --- src/apps/intel/intel1g.lua | 6 +++--- src/doc/testing.md | 2 ++ src/scripts/dock.sh | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/apps/intel/intel1g.lua b/src/apps/intel/intel1g.lua index 74f116d0bf..4cd3fa7156 100644 --- a/src/apps/intel/intel1g.lua +++ b/src/apps/intel/intel1g.lua @@ -35,7 +35,7 @@ -- Note: section and page numbers in the comments below refer to the i210 data sheet -- run selftest() on APU2's second/middle NIC: --- sudo SNABB_SELFTEST_INTEL1G_0="0000:02:00.0" ./snabb snsh -t apps.intel.intel1g +-- sudo SNABB_PCI_INTEL1G0="0000:02:00.0" ./snabb snsh -t apps.intel.intel1g -- Note: rxqueue >0 not working yet! @@ -632,9 +632,9 @@ end -- function Intel1g:new() function selftest () print("selftest: Intel1g") - local pciaddr = os.getenv("SNABB_SELFTEST_INTEL1G_0") + local pciaddr = os.getenv("SNABB_PCI_INTEL1G0") if not pciaddr then - print("SNABB_SELFTEST_INTEL1G_0 not set") + print("SNABB_PCI_INTEL1G0 not set") os.exit(engine.test_skipped_code) end diff --git a/src/doc/testing.md b/src/doc/testing.md index cbb50eacb2..cd473c9951 100644 --- a/src/doc/testing.md +++ b/src/doc/testing.md @@ -75,6 +75,8 @@ the tests: `SNABB_PCI1` in Intel specific tests. Some Intel specific tests (namely packetblaster based benchmarks) will be skipped if these are not set. +* `SNABB_PCI_INTEL1G0`—Optional PCI address for use in Intel1G selftest. + * `SNABB_PCI_SOLARFLARE0`, `SNABB_PCI_SOLARFLARE1`—Optional PCI addresses of two wired Solarflare NICs. These are preferred over `SNABB_PCI0` and `SNABB_PCI1` in Solarflare specific tests. diff --git a/src/scripts/dock.sh b/src/scripts/dock.sh index a06a5b9ee2..8df7d9010a 100755 --- a/src/scripts/dock.sh +++ b/src/scripts/dock.sh @@ -9,6 +9,7 @@ docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ -e SNABB_PCI1=$SNABB_PCI1 \ -e SNABB_PCI_INTEL0=$SNABB_PCI_INTEL0 \ -e SNABB_PCI_INTEL1=$SNABB_PCI_INTEL1 \ + -e SNABB_PCI_INTEL1G0=$SNABB_PCI_INTEL1G0 \ -e SNABB_PCI_SOLARFLARE0=$SNABB_PCI_SOLARFLARE0 \ -e SNABB_PCI_SOLARFLARE1=$SNABB_PCI_SOLARFLARE1 \ -e SNABB_TELNET0=$SNABB_TELNET0 \ From 9f48b14adf9fbd4f78c3b360d1e0d02a6215e229 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 13 Jun 2016 17:31:37 +0200 Subject: [PATCH 067/103] apps.intel.intel1g: use pci.map_pci_memory_locked. --- src/apps/intel/intel1g.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/intel/intel1g.lua b/src/apps/intel/intel1g.lua index 4cd3fa7156..e1c125e14c 100644 --- a/src/apps/intel/intel1g.lua +++ b/src/apps/intel/intel1g.lua @@ -107,7 +107,7 @@ function Intel1g:new(conf) -- Setup device access pci.unbind_device_from_linux(pciaddress) pci.set_bus_master(pciaddress, true) - local regs, mmiofd = pci.map_pci_memory(pciaddress, 0) + local regs, mmiofd = pci.map_pci_memory_locked(pciaddress, 0) -- Common utilities, see snabb/src/lib/hardware/register.lua local function bitvalue (value) From ed687f19f8459413145fabc56a462d7b38513ba8 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:11:18 +0200 Subject: [PATCH 068/103] main: properly handle unsupported worker states (SIGSTOP/SIGCONT)- --- src/core/main.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 0a27c076e5..32a1b482e2 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -177,8 +177,13 @@ else if signals[1].chld then local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG + else + S.kill(worker_pid, "hup") + exit_status = 255 + print("Error: Unsupported worker state (stopped / continued).") + end else S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo From db9e343d2c86c8db0f1d1e8fca9be21d80425599 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:11:59 +0200 Subject: [PATCH 069/103] main: comment syscall usage. --- src/core/main.lua | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 32a1b482e2..6a100a9a97 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -163,31 +163,41 @@ end -- Fork into worker process and supervisor local worker_pid = S.fork() if worker_pid == 0 then - -- Worker + -- Worker: Use prctl to ensure we are killed (SIGHUP) when our parent quits + -- and run main. S.prctl("set_pdeathsig", "hup") xpcall(main, handler) else - -- Supervisor + -- Supervisor: Queue exit_signals using signalfd, prevent them from killing + -- us instantly using sigprocmask. local exit_signals = "hup, int, quit, term, chld" local signalfd = S.signalfd(exit_signals) S.sigprocmask("block", exit_signals) + -- Read signals from signalfd. Only process the first signal because any + -- signal causes shutdown. local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) local exit_status if signals[1].chld then + -- SIGCHILD means worker state changed: retrieve its status using waitpid + -- and set exit status accordingly. local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) if worker.WIFEXITED then exit_status = worker.EXITSTATUS elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG else + -- Stopping/continuing the worker is unsupported and causes shutdown. S.kill(worker_pid, "hup") exit_status = 255 print("Error: Unsupported worker state (stopped / continued).") end else + -- Supervisor received exit signal: kill worker by sending SIGHUP and + -- and set exit status accordingly. S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo end + -- Clean up and exit. shutdown(worker_pid) os.exit(exit_status) end From c3b8672860bd2075a65c874270a8c9e9aaba06b6 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:30:49 +0200 Subject: [PATCH 070/103] Revert 'ed687f19f8459413145fabc56a462d7b38513ba8'. Conflicts: - src/core/main.lua --- src/core/main.lua | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 6a100a9a97..29ed5a50f4 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -183,14 +183,8 @@ else -- and set exit status accordingly. local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG - else - -- Stopping/continuing the worker is unsupported and causes shutdown. - S.kill(worker_pid, "hup") - exit_status = 255 - print("Error: Unsupported worker state (stopped / continued).") - end + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end else -- Supervisor received exit signal: kill worker by sending SIGHUP and -- and set exit status accordingly. From 5eb0f697e3d0de565acf589b615d3d8a288c6d2c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 14:36:05 +0200 Subject: [PATCH 071/103] core.main: handle SIGSTOP and SIGCONT to worker. This reverts commit c3b8672860bd2075a65c874270a8c9e9aaba06b6. --- src/core/main.lua | 48 +++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 29ed5a50f4..0054890f77 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -173,25 +173,33 @@ else local exit_signals = "hup, int, quit, term, chld" local signalfd = S.signalfd(exit_signals) S.sigprocmask("block", exit_signals) - -- Read signals from signalfd. Only process the first signal because any - -- signal causes shutdown. - local signals, err = S.util.signalfd_read(signalfd) - assert(signals, tostring(err)) - local exit_status - if signals[1].chld then - -- SIGCHILD means worker state changed: retrieve its status using waitpid - -- and set exit status accordingly. - local status, err, worker = S.waitpid(worker_pid) - assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end - else - -- Supervisor received exit signal: kill worker by sending SIGHUP and - -- and set exit status accordingly. - S.kill(worker_pid, "hup") - exit_status = 128 + signals[1].signo + while true do + -- Read signals from signalfd. Only process the first signal because any + -- signal causes shutdown. + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + for i = 1, #signals do + local exit_status + if signals[i].chld then + -- SIGCHILD means worker state changed: retrieve its status using + -- waitpid and set exit status accordingly. + local status, err, worker = + S.waitpid(worker_pid, "stopped,continued") + assert(status, tostring(err)) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG + -- WIFSTOPPED and WIFCONTINUED are ignored. + else goto ignore_signal end + else + -- Supervisor received exit signal: kill worker by sending SIGHUP + -- and and set exit status accordingly. + S.kill(worker_pid, "hup") + exit_status = 128 + signals[i].signo + end + -- Run shutdown routine and exit. + shutdown(worker_pid) + os.exit(exit_status) + ::ignore_signal:: + end end - -- Clean up and exit. - shutdown(worker_pid) - os.exit(exit_status) end From 76a5d6e4391bc1d8560b82616b25cb69f0fdecb3 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 16:01:14 +0200 Subject: [PATCH 072/103] main: allow setting developer_debug/debug_on_error through environment. --- src/core/main.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 0054890f77..cc31d26285 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -29,8 +29,8 @@ ffi.cdef[[ extern char** argv; ]] -_G.developer_debug = false -debug_on_error = false +_G.developer_debug = lib.getenv("SNABB_DEBUG") and true +debug_on_error = _G.developer_debug function main () zone("startup") From 127d21fa6e6dccc0cd00ceb9b11dd24ba838b723 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 16:01:46 +0200 Subject: [PATCH 073/103] main: do not unlink runtime directory in developer_debug mode. --- src/core/main.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index cc31d26285..f736e25458 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -133,7 +133,9 @@ end -- Cleanup after Snabb process. function shutdown (pid) - shm.unlink("//"..pid) + if not _G.developer_debug then + shm.unlink("//"..pid) + end end function selftest () From 1befce438d9bbdae99ad57e1f274b19a9eaf1374 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 16:35:14 +0200 Subject: [PATCH 074/103] remove packet.length/data. --- src/README.md | 10 ---------- src/apps/bridge/learning.lua | 2 +- src/apps/ipv6/nd_light.lua | 6 +++--- src/apps/ipv6/ns_responder.lua | 2 +- src/apps/test/match.lua | 6 +++--- src/apps/vlan/vlan.lua | 4 ++-- src/lib/ipsec/esp.lua | 33 +++++++++++++++------------------ src/lib/protocol/README.md | 4 +--- src/lib/protocol/datagram.lua | 10 +++++----- 9 files changed, 31 insertions(+), 46 deletions(-) diff --git a/src/README.md b/src/README.md index d0f171f3ce..a5f84eeedd 100644 --- a/src/README.md +++ b/src/README.md @@ -339,16 +339,6 @@ is uninitialized garbage. Frees *packet* and puts in back onto the freelist. -— Function **packet.data** *packet* - -Returns a pointer to the payload of *packet*. **Deprecated, please use the -`data` field instead.** - -— Function **packet.length** *packet* - -Returns the payload length of *packet*. **Deprecated, please use the `length` -field instead.** - — Function **packet.clone** *packet* Returns an exact copy of *packet*. diff --git a/src/apps/bridge/learning.lua b/src/apps/bridge/learning.lua index a372b78137..0edb4a0365 100644 --- a/src/apps/bridge/learning.lua +++ b/src/apps/bridge/learning.lua @@ -203,7 +203,7 @@ function bridge:push() -- packet with one of the packet forwarding tables according -- to the result. local mac = self._mac - mac[0] = packet.data(p) + mac[0] = p.data mac_table:lookup_pft(mac, ip, ig, p, self._pft_C, self._flood_pl[ip]) -- Associate the source MAC address with the ingress port and -- group. Multicast addresses are forbidden to occur as diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index 2717338d68..152e55455e 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -256,7 +256,7 @@ local function na (self, dgram, eth, ipv6, icmp) end local function from_south (self, p) - if not self._filter:match(packet.data(p[0]), packet.length(p[0])) then + if not self._filter:match(p[0].data, p[0].length) then return false end local dgram = datagram:new(p[0], ethernet) @@ -314,8 +314,8 @@ function nd_light:push () else local p = cache.p p[0] = link.receive(l_in) - if packet.length(p[0]) >= self._eth_header:sizeof() then - self._eth_header:copy(packet.data(p[0])) + if p[0].length >= self._eth_header:sizeof() then + self._eth_header:copy(p[0].data) link.transmit(l_out, p[0]) else packet.free(p[0]) end end diff --git a/src/apps/ipv6/ns_responder.lua b/src/apps/ipv6/ns_responder.lua index a23a5151d6..e61b575a7d 100644 --- a/src/apps/ipv6/ns_responder.lua +++ b/src/apps/ipv6/ns_responder.lua @@ -36,7 +36,7 @@ function ns_responder:new(config) end local function process (self, p) - if not self._filter:match(packet.data(p), packet.length(p)) then + if not self._filter:match(p.data, p.length) then return false end local dgram = self._dgram:new(p, ethernet) diff --git a/src/apps/test/match.lua b/src/apps/test/match.lua index 48427d50bc..649bc4803f 100644 --- a/src/apps/test/match.lua +++ b/src/apps/test/match.lua @@ -5,7 +5,7 @@ local C = ffi.C local lib = require("core.lib") local function dump (p) - return lib.hexdump(ffi.string(packet.data(p), packet.length(p))) + return lib.hexdump(ffi.string(p.data, p.length)) end Match = {} @@ -24,8 +24,8 @@ function Match:push () local p = link.receive(self.input.rx) local cmp = link.front(self.input.comparator) if not cmp then - elseif packet.length(cmp) ~= packet.length(p) - or C.memcmp(packet.data(cmp), packet.data(p), packet.length(cmp)) ~= 0 then + elseif cmp.length ~= p.length + or C.memcmp(cmp.data, p.data, cmp.length) ~= 0 then if not self.fuzzy then table.insert(self.errs, "Mismatch:\n"..dump(cmp).."\n"..dump(p)) end diff --git a/src/apps/vlan/vlan.lua b/src/apps/vlan/vlan.lua index 763017beb8..3ec4b6b2d3 100644 --- a/src/apps/vlan/vlan.lua +++ b/src/apps/vlan/vlan.lua @@ -48,7 +48,7 @@ end -- packet is carrying a VLAN tag, if it's an untagged frame these bytes will be -- Ethernet payload function extract_tci(pkt) - return ntohs(cast("uint16_t*", packet.data(pkt) + o_ethernet_ethertype + 2)[0]) + return ntohs(cast("uint16_t*", pkt.data + o_ethernet_ethertype + 2)[0]) end -- extract VLAN id from TCI @@ -120,7 +120,7 @@ function VlanMux:push() if type(name) == "string" then for _ = 1, math.min(link.nreadable(l), maxoutput) do local p = receive(l) - local ethertype = cast("uint16_t*", packet.data(p) + o_ethernet_ethertype)[0] + local ethertype = cast("uint16_t*", p.data + o_ethernet_ethertype)[0] if name == "trunk" then -- trunk -- check for ethertype 0x8100 (802.1q VLAN tag) diff --git a/src/lib/ipsec/esp.lua b/src/lib/ipsec/esp.lua index 23ba7b4c6a..33e3beb270 100644 --- a/src/lib/ipsec/esp.lua +++ b/src/lib/ipsec/esp.lua @@ -86,16 +86,15 @@ local function padding (a, l) return (a - l%a) % a end -- 5. Write ESP header function esp_v6_encrypt:encapsulate (p) local gcm = self.aes_128_gcm - local data, length = packet.data(p), packet.length(p) - if length < PAYLOAD_OFFSET then return false end - local payload = data + PAYLOAD_OFFSET - local payload_length = length - PAYLOAD_OFFSET + if p.length < PAYLOAD_OFFSET then return false end + local payload = p.data + PAYLOAD_OFFSET + local payload_length = p.length - PAYLOAD_OFFSET -- Padding, see https://tools.ietf.org/html/rfc4303#section-2.4 local pad_length = padding(self.pad_to, payload_length + self.ESP_PAYLOAD_OVERHEAD) local overhead = self.ESP_OVERHEAD + pad_length - packet.resize(p, length + overhead) - self.ip:new_from_mem(data + ETHERNET_SIZE, IPV6_SIZE) - self.esp_tail:new_from_mem(data + length + pad_length, ESP_TAIL_SIZE) + packet.resize(p, p.length + overhead) + self.ip:new_from_mem(p.data + ETHERNET_SIZE, IPV6_SIZE) + self.esp_tail:new_from_mem(p.data + p.length + pad_length, ESP_TAIL_SIZE) self.esp_tail:next_header(self.ip:next_header()) self.esp_tail:pad_length(pad_length) self:next_seq_no() @@ -134,14 +133,13 @@ end -- 5. Shrink p by ESP overhead function esp_v6_decrypt:decapsulate (p) local gcm = self.aes_128_gcm - local data, length = packet.data(p), packet.length(p) - if length - PAYLOAD_OFFSET < self.MIN_SIZE then return false end - self.ip:new_from_mem(data + ETHERNET_SIZE, IPV6_SIZE) - local payload = data + PAYLOAD_OFFSET + if p.length - PAYLOAD_OFFSET < self.MIN_SIZE then return false end + self.ip:new_from_mem(p.data + ETHERNET_SIZE, IPV6_SIZE) + local payload = p.data + PAYLOAD_OFFSET self.esp:new_from_mem(payload, ESP_SIZE) local iv_start = payload + ESP_SIZE local ctext_start = payload + self.CTEXT_OFFSET - local ctext_length = length - self.PLAIN_OVERHEAD + local ctext_length = p.length - self.PLAIN_OVERHEAD local seq_low = self.esp:seq_no() local seq_high = C.track_seq_no(seq_low, self.seq:low(), self.seq:high(), self.window_size) if gcm:decrypt(ctext_start, seq_low, seq_high, iv_start, ctext_start, ctext_length) then @@ -176,20 +174,19 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ ) local d = datagram:new(payload) local ip = ipv6:new({}) - ip:payload_length(packet.length(payload)) + ip:payload_length(payload.length) d:push(ip) d:push(ethernet:new({type=0x86dd})) local p = d:packet() -- Check integrity - print("original", lib.hexdump(ffi.string(packet.data(p), packet.length(p)))) + print("original", lib.hexdump(ffi.string(p.data, p.length))) local p_enc = packet.clone(p) assert(enc:encapsulate(p_enc), "encapsulation failed") - print("encrypted", lib.hexdump(ffi.string(packet.data(p_enc), packet.length(p_enc)))) + print("encrypted", lib.hexdump(ffi.string(p_enc.data, p_enc.length))) local p2 = packet.clone(p_enc) assert(dec:decapsulate(p2), "decapsulation failed") - print("decrypted", lib.hexdump(ffi.string(packet.data(p2), packet.length(p2)))) - assert(packet.length(p2) == packet.length(p) - and C.memcmp(p, p2, packet.length(p)) == 0, + print("decrypted", lib.hexdump(ffi.string(p2.data, p2.length))) + assert(p2.length == p.length and C.memcmp(p, p2, p.length) == 0, "integrity check failed") -- Check invalid packets. local p_invalid = packet.from_string("invalid") diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 6f239687d7..41f0b89c77 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -804,9 +804,7 @@ If *pointer* and *length* are supplied then *length* bytes starting from — Method **datagram:data** -Returns the location and size of the buffer of the underlying packet. -This is a shortcut to *datagram:packet* followed by calls to -*packet.data* and *pakcet.length*. +Returns `data` and `length` of the underlying packet. - Method **datagram:commit** diff --git a/src/lib/protocol/datagram.lua b/src/lib/protocol/datagram.lua index 91d778fb71..f2b180e64c 100644 --- a/src/lib/protocol/datagram.lua +++ b/src/lib/protocol/datagram.lua @@ -180,8 +180,8 @@ function datagram:parse_match (class, check) local parse = self._parse local class = class or parse.ulp if not class then return nil end - local proto = class:new_from_mem(packet.data(self._packet[0]) + parse.offset, - packet.length(self._packet[0]) - parse.offset) + local proto = class:new_from_mem(self._packet[0].data + parse.offset, + self._packet[0].length - parse.offset) if proto == nil or (check and not check(proto)) then if proto then proto:free() end return nil @@ -295,14 +295,14 @@ end -- appended to the packet's payload first. function datagram:payload (mem, size) if mem then packet.append(self._packet[0], mem, size) end - return packet.data(self._packet[0]) + self._parse.offset, - packet.length(self._packet[0]) - self._parse.offset + return self._packet[0].data + self._parse.offset, + self._packet[0].length - self._parse.offset end -- Return the location and size of the entire packet buffer function datagram:data () local p = self._packet - return packet.data(p[0]), packet.length(p[0]) + return p[0].data, p[0].length end -- Commit the changes induced by previous calles to the push*() and From 694b1c4298b72b7aa865630bc673891fd06dd753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 16 Jun 2016 10:57:28 +0100 Subject: [PATCH 075/103] test_env.sh: output kernel console to stdout --- src/program/snabbnfv/test_env/test_env.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/program/snabbnfv/test_env/test_env.sh b/src/program/snabbnfv/test_env/test_env.sh index a3e253f53c..56bd6b7914 100644 --- a/src/program/snabbnfv/test_env/test_env.sh +++ b/src/program/snabbnfv/test_env/test_env.sh @@ -116,14 +116,15 @@ function launch_qemu { "numactl --cpunodebind=$(pci_node $1) --membind=$(pci_node $1) \ $QEMU $QEMU_ARGS \ -kernel $assets/$4 \ - -append \"earlyprintk root=/dev/vda $SNABB_KERNEL_PARAMS rw console=ttyS0 ip=$(ip $qemu_n)\" \ + -append \"earlyprintk root=/dev/vda $SNABB_KERNEL_PARAMS rw console=ttyS1 ip=$(ip $qemu_n)\" \ -m $GUEST_MEM -numa node,memdev=mem -object memory-backend-file,id=mem,size=${GUEST_MEM}M,mem-path=$HUGETLBFS,share=on \ -netdev type=vhost-user,id=net0,chardev=char0${mqueues} -chardev socket,id=char0,path=$2,server \ -device virtio-net-pci,netdev=net0,mac=$(mac $qemu_n),mq=$qemu_mq,vectors=$qemu_vectors \ -M pc -smp $qemu_smp -cpu host --enable-kvm \ -serial telnet:localhost:$3,server,nowait \ + -serial stdio \ -drive if=virtio,format=raw,file=$(qemu_image $5) \ - -nographic" \ + -display none" \ $(qemu_log) qemu_n=$(expr $qemu_n + 1) sockets="$sockets $2" From 751cc28332c3298a014429ea0b9aedde073ffc3a Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 16 Jun 2016 14:22:49 +0200 Subject: [PATCH 076/103] apps.bridge.base: use `app:start()' to run post configuration. --- src/apps/bridge/base.lua | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/apps/bridge/base.lua b/src/apps/bridge/base.lua index be320c8a2d..da3ab6f77e 100644 --- a/src/apps/bridge/base.lua +++ b/src/apps/bridge/base.lua @@ -26,12 +26,6 @@ -- can access the configuration via self._conf.config. If config is -- not set, it is initialiezed to an empty table. -- --- Note that it is necessary to call the method post_config() after --- the app has been configured with engine.configure() to complete the --- initialization. This step will add the ringbuffers associated with --- the ports to an internal data structure to save a lookup in the --- input and output tables during packet processing. --- -- To make processing in the fast path easier, each port and group is -- assigned a unique integer greater than zero to serve as a "handle". -- The group handle 0 is assigned to all free ports. @@ -149,7 +143,7 @@ end -- accessible via the keys l_in and l_out, respectively. This helps -- to speed up packet forwarding by eliminating a lookup in the input -- and output tables. -function bridge:post_config () +function bridge:start () assert(self.input and self.output) for _, port in ipairs(self._ports) do port.l_in = self.input[port.name] From 11890e6d04d5700d08dfa52c2574221fa9ef8505 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 16 Jun 2016 14:33:58 +0200 Subject: [PATCH 077/103] core.app: edit documentation of app:start(). --- src/README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/README.md b/src/README.md index 1d4dfcc9e4..57460ecea7 100644 --- a/src/README.md +++ b/src/README.md @@ -99,6 +99,18 @@ Tables of named input and output links. These tables are initialized by the engine for use in processing and are *read-only*. +— Field **myapp.appname** + +Name of the app. *Read-only*. + + +— Method **myapp:start** + +*Optional*. Start the app. At this the `input` and `output` link tables are +populated, and `appname` is set. The app can perform additional initialization +if required. + + — Method **myapp:pull** *Optional*. Pull packets into the network. @@ -126,14 +138,6 @@ implemented the app instance is discarded and a new instance is created. *Optional*. Print a report of the current app status. -— Method **myapp:start** - -*Optional*. Starts the app. - -At this point links and app name are set. The app has a chance to do -additional initialization if needed. - - — Method **myapp:stop** *Optional*. Stop the app and release associated external resources. From 5488c68cb5afb243e5006f000c2f33556cf44031 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 8 Jun 2016 17:44:10 +0200 Subject: [PATCH 078/103] lib.protocol.ethernet: add is_bcast and n_bcast. --- src/lib/protocol/README.md | 9 +++++++++ src/lib/protocol/ethernet.lua | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 4a13483a4b..a20093cf60 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -107,6 +107,15 @@ Returns a true value if *mac* address denotes a [Multicast address](https://en.w Returns 1 if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet) and 0 otherwise. +— Function **ethernet:is_bcast** *mac* + +Returns a true value if *mac* address denotes a [Broadcast address](https://en.wikipedia.org/wiki/Broadcast_address#Ethernet). + +— Function **ethernet:n_bcast** *mac* + +Returns 1 if *mac* address denotes a [Broadcast address](https://en.wikipedia.org/wiki/Broadcast_address#Ethernet) +and 0 otherwise. + — Function **ethernet:ipv6_mcast** *ip* Returns the MAC address for IPv6 multicast *ip* as defined by RFC2464, diff --git a/src/lib/protocol/ethernet.lua b/src/lib/protocol/ethernet.lua index b5abed8d31..2fcf3cf71d 100644 --- a/src/lib/protocol/ethernet.lua +++ b/src/lib/protocol/ethernet.lua @@ -86,6 +86,17 @@ function ethernet:is_mcast (addr) return ethernet:n_mcast(addr) ~= 0 end +-- Return 1 if MAC address is the broadcast address and 0 otherwise +local bcast_address = ethernet:pton("FF:FF:FF:FF:FF:FF") +function ethernet:n_bcast (addr) + return C.memcmp(addr, bcast_address, 6) == 0 +end + +-- Check whether a MAC address is the broadcast address +function ethernet:is_bcast (addr) + return ethernet:n_bcast(addr) ~= 0 +end + -- Instance methods function ethernet:src (a) From 9044c6f0a6f0cae2a2c5c46ff17e5dd1eeb33e94 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 8 Jun 2016 17:44:43 +0200 Subject: [PATCH 079/103] apps.vhost.vhost_user: count rxbcast/txbcast. --- src/apps/vhost/vhost_user.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index fe2711769e..85fd3198f0 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -31,8 +31,8 @@ VhostUser = {} local provided_counters = { 'type', 'dtime', - 'rxbytes', 'rxpackets', 'rxmcast', 'rxdrop', - 'txbytes', 'txpackets', 'txmcast' + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', + 'txbytes', 'txpackets', 'txmcast', 'txbcast' } function VhostUser:new (args) @@ -108,12 +108,14 @@ function VhostUser:tx_callback (p) counter.add(self.counters.txbytes, packet.length(p)) counter.add(self.counters.txpackets) counter.add(self.counters.txmcast, ethernet:n_mcast(packet.data(p))) + counter.add(self.counters.txbcast, ethernet:n_bcast(packet.data(p))) end function VhostUser:rx_callback (p) counter.add(self.counters.rxbytes, packet.length(p)) counter.add(self.counters.rxpackets) counter.add(self.counters.rxmcast, ethernet:n_mcast(packet.data(p))) + counter.add(self.counters.rxbcast, ethernet:n_bcast(packet.data(p))) end function VhostUser:rxdrop_callback (p) From 5a0161230e305e37f74149c098b5705b20ea315e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 8 Jun 2016 17:45:07 +0200 Subject: [PATCH 080/103] apps.tap.tap: add statistics counters. --- src/apps/tap/tap.lua | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/apps/tap/tap.lua b/src/apps/tap/tap.lua index 497399d11a..fe803a7b70 100644 --- a/src/apps/tap/tap.lua +++ b/src/apps/tap/tap.lua @@ -5,6 +5,8 @@ module(..., package.seeall) local S = require("syscall") local link = require("core.link") local packet = require("core.packet") +local counter = require("core.counter") +local ethernet = require("lib.protocol.ethernet") local ffi = require("ffi") local C = ffi.C local const = require("syscall.linux.constants") @@ -14,6 +16,12 @@ local t = S.types.t Tap = { } +local provided_counters = { + 'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', + 'txbytes', 'txpackets', 'txmcast', 'txbcast' +} + function Tap:new (name) assert(name, "missing tap interface name") @@ -27,8 +35,14 @@ function Tap:new (name) sock:close() error("Error opening /dev/net/tun: " .. tostring(err)) end - - return setmetatable({sock = sock, name = name}, {__index = Tap}) + local counters = {} + for _, name in ipairs(provided_counters) do + counters[name] = counter.open(name) + end + counter.set(counters.type, 0x1001) -- Virtual interface + counter.set(counters.dtime, C.get_unix_time()) + return setmetatable({sock = sock, name = name, counters = counters}, + {__index = Tap}) end function Tap:pull () @@ -49,6 +63,10 @@ function Tap:pull () end p.length = len link.transmit(l, p) + counter.add(self.counters.rxbytes, len) + counter.add(self.counters.rxpackets) + counter.add(self.counters.rxmcast, ethernet:n_mcast(p.data)) + counter.add(self.counters.rxbcast, ethernet:n_bcast(p.data)) end end @@ -66,6 +84,10 @@ function Tap:push () if len ~= p.length and err.errno == const.E.AGAIN then return end + counter.add(self.counters.txbytes, len) + counter.add(self.counters.txpackets) + counter.add(self.counters.txmcast, ethernet:n_mcast(p.data)) + counter.add(self.counters.txbcast, ethernet:n_bcast(p.data)) -- The write completed so dequeue it from the link and free the packet link.receive(l) packet.free(p) @@ -74,6 +96,8 @@ end function Tap:stop() self.sock:close() + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end function selftest() From 6233d1532e00ecd6c399b4db57da96146daa6c3b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 13 Jun 2016 18:21:55 +0200 Subject: [PATCH 081/103] core.packet: make max_payload public. --- src/README.md | 4 ++++ src/core/packet.lua | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/README.md b/src/README.md index 92ed940bed..1a5eb2a993 100644 --- a/src/README.md +++ b/src/README.md @@ -316,6 +316,10 @@ not been transmitted or freed. The number of allocatable packets is limited by the size of the underlying "freelist", e.g. a pool of unused packet objects from and to which packets are allocated and freed. +— Variable **packet.max_payload** + +Maximum payload length for packets. Read-only. + — Function **packet.allocate** Returns a new empty packet. An an error is raised if there are no packets diff --git a/src/core/packet.lua b/src/core/packet.lua index 6c4ed5311b..143bfdf920 100644 --- a/src/core/packet.lua +++ b/src/core/packet.lua @@ -17,7 +17,7 @@ local packet_t = ffi.typeof("struct packet") local packet_ptr_t = ffi.typeof("struct packet *") local packet_size = ffi.sizeof(packet_t) local header_size = 8 -local max_payload = tonumber(C.PACKET_PAYLOAD_SIZE) +max_payload = tonumber(C.PACKET_PAYLOAD_SIZE) -- Freelist containing empty packets ready for use. From 788438e8898156796289a23afc7a597362cea377 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 8 Jun 2016 17:45:18 +0200 Subject: [PATCH 082/103] apps.socket.raw: add statistics counters. --- src/apps/socket/raw.lua | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/apps/socket/raw.lua b/src/apps/socket/raw.lua index 24ad578d5d..cd0f9799fa 100644 --- a/src/apps/socket/raw.lua +++ b/src/apps/socket/raw.lua @@ -7,6 +7,8 @@ local h = require("syscall.helpers") local bit = require("bit") local link = require("core.link") local packet = require("core.packet") +local counter = require("core.counter") +local ethernet = require("lib.protocol.ethernet") local ffi = require("ffi") local C = ffi.C @@ -21,6 +23,12 @@ local c, t = S.c, S.types.t RawSocket = {} +local provided_counters = { + 'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', + 'txbytes', 'txpackets', 'txmcast', 'txbcast' +} + function RawSocket:new (ifname) assert(ifname) local index, err = S.util.if_nametoindex(ifname) @@ -40,7 +48,16 @@ function RawSocket:new (ifname) sock:close() error(err) end - return setmetatable({sock = sock}, {__index = RawSocket}) + local counters = {} + for _, name in ipairs(provided_counters) do + counters[name] = counter.open(name) + end + counter.set(counters.type, 0x1001) -- Virtual interface + counter.set(counters.dtime, C.get_unix_time()) + return setmetatable({sock = sock, + rx_p = packet.allocate(), + counters = counters}, + {__index = RawSocket}) end function RawSocket:pull () @@ -61,10 +78,14 @@ function RawSocket:can_receive () end function RawSocket:receive () - local buffer = ffi.new("uint8_t[?]", C.PACKET_PAYLOAD_SIZE) - local sz, err = S.read(self.sock, buffer, C.PACKET_PAYLOAD_SIZE) - assert(sz, err) - return packet.from_pointer(buffer, sz) + local p = self.rx_p + local sz = assert(S.read(self.sock, p.data, packet.max_payload)) + p.length = sz + counter.add(self.counters.rxbytes, sz) + counter.add(self.counters.rxpackets) + counter.add(self.counters.rxmcast, ethernet:n_mcast(p.data)) + counter.add(self.counters.rxbcast, ethernet:n_bcast(p.data)) + return packet.clone(p) end function RawSocket:push () @@ -73,6 +94,10 @@ function RawSocket:push () while not link.empty(l) and self:can_transmit() do local p = link.receive(l) self:transmit(p) + counter.add(self.counters.txbytes, p.length) + counter.add(self.counters.txpackets) + counter.add(self.counters.txmcast, ethernet:n_mcast(p.data)) + counter.add(self.counters.txbcast, ethernet:n_bcast(p.data)) packet.free(p) end end @@ -94,6 +119,9 @@ end function RawSocket:stop() self.sock:close() + packet.free(self.rx_p) + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end function selftest () From 7d5b9738e2156fd9e60121c33ace456284628da3 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 16 Jun 2016 18:23:51 +0200 Subject: [PATCH 083/103] Rename app:start from #905 to app:configure. --- src/README.md | 4 ++-- src/apps/bridge/base.lua | 2 +- src/core/app.lua | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/README.md b/src/README.md index 26a656335e..103cb390d9 100644 --- a/src/README.md +++ b/src/README.md @@ -104,9 +104,9 @@ the engine for use in processing and are *read-only*. Name of the app. *Read-only*. -— Method **myapp:start** +— Method **myapp:configure** -*Optional*. Start the app. At this the `input` and `output` link tables are +*Optional*. Configure the app. At this the `input` and `output` link tables are populated, and `appname` is set. The app can perform additional initialization if required. diff --git a/src/apps/bridge/base.lua b/src/apps/bridge/base.lua index da3ab6f77e..5a2355580f 100644 --- a/src/apps/bridge/base.lua +++ b/src/apps/bridge/base.lua @@ -143,7 +143,7 @@ end -- accessible via the keys l_in and l_out, respectively. This helps -- to speed up packet forwarding by eliminating a lookup in the input -- and output tables. -function bridge:start () +function bridge:configure () assert(self.input and self.output) for _, port in ipairs(self._ports) do port.l_in = self.input[port.name] diff --git a/src/core/app.lua b/src/core/app.lua index 3fe2a4e710..7d79349f44 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -232,9 +232,9 @@ function apply_config_actions (actions, conf) -- Commit changes. app_table, link_table = new_app_table, new_link_table app_array, link_array = new_app_array, new_link_array - -- Trigger start event for each app. + -- Trigger configure event for each app. for _, app in ipairs(app_array) do - if app.start then app:start() end + if app.configure then app:configure() end end end From 4dec991ca512e94071087b73d3fa9f77d78d6737 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 16 Jun 2016 18:42:26 +0200 Subject: [PATCH 084/103] scripts/dock.sh: also pass through SNABB_PCI_INTEL1G1. --- src/doc/testing.md | 3 ++- src/scripts/dock.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/doc/testing.md b/src/doc/testing.md index cd473c9951..b2b7beb761 100644 --- a/src/doc/testing.md +++ b/src/doc/testing.md @@ -75,7 +75,8 @@ the tests: `SNABB_PCI1` in Intel specific tests. Some Intel specific tests (namely packetblaster based benchmarks) will be skipped if these are not set. -* `SNABB_PCI_INTEL1G0`—Optional PCI address for use in Intel1G selftest. +* `SNABB_PCI_INTEL1G0`, `SNABB_PCI_INTEL1G1`—Optional PCI addresses for use in + Intel1G selftest. * `SNABB_PCI_SOLARFLARE0`, `SNABB_PCI_SOLARFLARE1`—Optional PCI addresses of two wired Solarflare NICs. These are preferred over `SNABB_PCI0` and diff --git a/src/scripts/dock.sh b/src/scripts/dock.sh index 8df7d9010a..7ae45005b9 100755 --- a/src/scripts/dock.sh +++ b/src/scripts/dock.sh @@ -10,6 +10,7 @@ docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ -e SNABB_PCI_INTEL0=$SNABB_PCI_INTEL0 \ -e SNABB_PCI_INTEL1=$SNABB_PCI_INTEL1 \ -e SNABB_PCI_INTEL1G0=$SNABB_PCI_INTEL1G0 \ + -e SNABB_PCI_INTEL1G1=$SNABB_PCI_INTEL1G1 \ -e SNABB_PCI_SOLARFLARE0=$SNABB_PCI_SOLARFLARE0 \ -e SNABB_PCI_SOLARFLARE1=$SNABB_PCI_SOLARFLARE1 \ -e SNABB_TELNET0=$SNABB_TELNET0 \ From a9f7011623ca1313abf5442363713ec5ae8cfc90 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 20 Jun 2016 16:16:26 +0200 Subject: [PATCH 085/103] lib.ipsec.esp: fix regression introduces by previous commit. --- src/lib/ipsec/esp.lua | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/lib/ipsec/esp.lua b/src/lib/ipsec/esp.lua index 33e3beb270..d7c668acad 100644 --- a/src/lib/ipsec/esp.lua +++ b/src/lib/ipsec/esp.lua @@ -86,15 +86,16 @@ local function padding (a, l) return (a - l%a) % a end -- 5. Write ESP header function esp_v6_encrypt:encapsulate (p) local gcm = self.aes_128_gcm - if p.length < PAYLOAD_OFFSET then return false end - local payload = p.data + PAYLOAD_OFFSET - local payload_length = p.length - PAYLOAD_OFFSET + local data, length = p.data, p.length + if length < PAYLOAD_OFFSET then return false end + local payload = data + PAYLOAD_OFFSET + local payload_length = length - PAYLOAD_OFFSET -- Padding, see https://tools.ietf.org/html/rfc4303#section-2.4 local pad_length = padding(self.pad_to, payload_length + self.ESP_PAYLOAD_OVERHEAD) local overhead = self.ESP_OVERHEAD + pad_length - packet.resize(p, p.length + overhead) - self.ip:new_from_mem(p.data + ETHERNET_SIZE, IPV6_SIZE) - self.esp_tail:new_from_mem(p.data + p.length + pad_length, ESP_TAIL_SIZE) + packet.resize(p, length + overhead) + self.ip:new_from_mem(data + ETHERNET_SIZE, IPV6_SIZE) + self.esp_tail:new_from_mem(data + length + pad_length, ESP_TAIL_SIZE) self.esp_tail:next_header(self.ip:next_header()) self.esp_tail:pad_length(pad_length) self:next_seq_no() @@ -133,13 +134,14 @@ end -- 5. Shrink p by ESP overhead function esp_v6_decrypt:decapsulate (p) local gcm = self.aes_128_gcm - if p.length - PAYLOAD_OFFSET < self.MIN_SIZE then return false end - self.ip:new_from_mem(p.data + ETHERNET_SIZE, IPV6_SIZE) - local payload = p.data + PAYLOAD_OFFSET + local data, length = p.data, p.length + if length - PAYLOAD_OFFSET < self.MIN_SIZE then return false end + self.ip:new_from_mem(data + ETHERNET_SIZE, IPV6_SIZE) + local payload = data + PAYLOAD_OFFSET self.esp:new_from_mem(payload, ESP_SIZE) local iv_start = payload + ESP_SIZE local ctext_start = payload + self.CTEXT_OFFSET - local ctext_length = p.length - self.PLAIN_OVERHEAD + local ctext_length = length - self.PLAIN_OVERHEAD local seq_low = self.esp:seq_no() local seq_high = C.track_seq_no(seq_low, self.seq:low(), self.seq:high(), self.window_size) if gcm:decrypt(ctext_start, seq_low, seq_high, iv_start, ctext_start, ctext_length) then From e60226c8df5b36ce957b164301c32992f38d6025 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 20 Jun 2016 16:43:04 +0200 Subject: [PATCH 086/103] Rename app:configure to app:link. --- src/README.md | 8 ++++---- src/apps/bridge/base.lua | 2 +- src/core/app.lua | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/README.md b/src/README.md index 103cb390d9..7ad5b88068 100644 --- a/src/README.md +++ b/src/README.md @@ -104,11 +104,11 @@ the engine for use in processing and are *read-only*. Name of the app. *Read-only*. -— Method **myapp:configure** +— Method **myapp:link** -*Optional*. Configure the app. At this the `input` and `output` link tables are -populated, and `appname` is set. The app can perform additional initialization -if required. +*Optional*. Called any time the app’s links may have been changed (including on +start-up). Guaranteed to be called before `pull` and `push` are called with new +links. — Method **myapp:pull** diff --git a/src/apps/bridge/base.lua b/src/apps/bridge/base.lua index 5a2355580f..284cef64c7 100644 --- a/src/apps/bridge/base.lua +++ b/src/apps/bridge/base.lua @@ -143,7 +143,7 @@ end -- accessible via the keys l_in and l_out, respectively. This helps -- to speed up packet forwarding by eliminating a lookup in the input -- and output tables. -function bridge:configure () +function bridge:link () assert(self.input and self.output) for _, port in ipairs(self._ports) do port.l_in = self.input[port.name] diff --git a/src/core/app.lua b/src/core/app.lua index 7d79349f44..619e4fd78e 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -232,9 +232,9 @@ function apply_config_actions (actions, conf) -- Commit changes. app_table, link_table = new_app_table, new_link_table app_array, link_array = new_app_array, new_link_array - -- Trigger configure event for each app. + -- Trigger link event for each app. for _, app in ipairs(app_array) do - if app.configure then app:configure() end + if app.link then app:link() end end end From 0f385055561b362a057f6e000ef93ccbf60e0086 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 18 May 2016 16:50:56 +0200 Subject: [PATCH 087/103] lib.ipsec.esp: Test minimum packet sizes. --- src/lib/ipsec/esp.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib/ipsec/esp.lua b/src/lib/ipsec/esp.lua index 23ba7b4c6a..5c6c0c07f7 100644 --- a/src/lib/ipsec/esp.lua +++ b/src/lib/ipsec/esp.lua @@ -196,6 +196,22 @@ ABCDEFGHIJKLMNOPQRSTUVWXYZ assert(not enc:encapsulate(p_invalid), "encapsulated invalid packet") local p_invalid = packet.from_string("invalid") assert(not dec:decapsulate(p_invalid), "decapsulated invalid packet") + -- Check minimum packet. + local p_min = packet.from_string("012345678901234567890123456789012345678901234567890123") + p_min.data[18] = 0 -- Set IPv6 payload length to zero + p_min.data[19] = 0 -- ... + assert(packet.length(p_min) == PAYLOAD_OFFSET) + print("original", lib.hexdump(ffi.string(packet.data(p_min), packet.length(p_min)))) + local e_min = packet.clone(p_min) + assert(enc:encapsulate(e_min)) + print("encrypted", lib.hexdump(ffi.string(packet.data(e_min), packet.length(e_min)))) + assert(packet.length(e_min) == dec.MIN_SIZE+PAYLOAD_OFFSET) + assert(dec:decapsulate(e_min)) + print("decrypted", lib.hexdump(ffi.string(packet.data(e_min), packet.length(e_min)))) + assert(packet.length(e_min) == PAYLOAD_OFFSET) + assert(packet.length(p_min) == packet.length(e_min) + and C.memcmp(p_min, e_min, packet.length(p_min)) == 0, + "integrity check failed") -- Check transmitted Sequence Number wrap around enc.seq:low(0) enc.seq:high(1) From b8cbd65eeafd1b44c75a7494ed12f8decd1d76b9 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 20 Jun 2016 18:14:30 +0200 Subject: [PATCH 088/103] =?UTF-8?q?SnabbNFV:=20Add=20=E2=80=9Ccrypto?= =?UTF-8?q?=E2=80=9D=20tunnel=20option=20using=20lib.ipsec.esp.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apps/ipsec/README.md | 44 ++++++++++++++++++ src/apps/ipsec/esp.lua | 45 +++++++++++++++++++ src/bench/snabbnfv-iperf-1500-crypto | 6 +++ src/bench/snabbnfv-iperf-1500-tunnel+crypto | 6 +++ src/program/snabbnfv/README.md | 14 +++++- src/program/snabbnfv/nfvconfig.lua | 8 ++++ src/program/snabbnfv/selftest.sh | 22 +++++++-- .../test_functions/crypto-tunnel.ports | 30 +++++++++++++ .../nfvconfig/test_functions/crypto.ports | 16 +++++++ src/program/snabbnfv/traffic/README | 10 +++++ 10 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 src/apps/ipsec/README.md create mode 100644 src/apps/ipsec/esp.lua create mode 100755 src/bench/snabbnfv-iperf-1500-crypto create mode 100755 src/bench/snabbnfv-iperf-1500-tunnel+crypto create mode 100644 src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto-tunnel.ports create mode 100644 src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports diff --git a/src/apps/ipsec/README.md b/src/apps/ipsec/README.md new file mode 100644 index 0000000000..c75eaf0708 --- /dev/null +++ b/src/apps/ipsec/README.md @@ -0,0 +1,44 @@ +# IPsec Apps + +## AES128gcm (apps.ipsec.esp) + +The `AES128gcm` implements an ESP transport tunnel using the AES-GCM-128 +cipher. It encrypts packets received on its `decapsulated` port and transmits +them on its `encapsulated` port, and vice-versa. Packets arriving on the +`decapsulated` port must have an IPv6 header, and packets arriving on the +`encapsulated` port must have an IPv6 header followed by an ESP header, +otherwise they will be discarded. + +References: + + - `lib.ipsec.esp` + + DIAGRAM: AES128gcm + +-----------+ + encapsulated | | + ---->* AES128gcm *<---- + <----* *----> + | | decapsulated + +-----------+ + + encapsulated + --------\ /---------- + <-------|---/ /-------> + \-----/ decapsulated + +### Configuration + +The `AES128gcm` app accepts a table as its configuration argument. The +following keys are defined: + +— Key **spi** + +*Required*. Security Parameter Index. A 32 bit integer. + +— Key **key** + +*Required*. 20 bytes in form of a hex encoded string. + +— Key **replay_window** + +*Optional*. Size of the “Anti-Replay Window”. Defaults to 128. diff --git a/src/apps/ipsec/esp.lua b/src/apps/ipsec/esp.lua new file mode 100644 index 0000000000..9c2121320a --- /dev/null +++ b/src/apps/ipsec/esp.lua @@ -0,0 +1,45 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +-- This app implements a point-to-point encryption tunnel using ESP with +-- AES-128-GCM. + +module(..., package.seeall) +local esp = require("lib.ipsec.esp") + +AES128gcm = {} + +function AES128gcm:new (arg) + local conf = arg and config.parse_app_arg(arg) or {} + local self = {} + self.encrypt = esp.esp_v6_encrypt:new{ + mode = "aes-128-gcm", + spi = conf.spi, + keymat = conf.key:sub(1, 32), + salt = conf.key:sub(33, 40)} + self.decrypt = esp.esp_v6_decrypt:new{ + mode = "aes-128-gcm", + spi = conf.spi, + keymat = conf.key:sub(1, 32), + salt = conf.key:sub(33, 40), + window_size = conf.replay_window} + return setmetatable(self, {__index = AES128gcm}) +end + +function AES128gcm:push () + -- Encapsulation path + local input = self.input.decapsulated + local output = self.output.encapsulated + for _=1,link.nreadable(input) do + local p = link.receive(input) + if self.encrypt:encapsulate(p) then link.transmit(output, p) + else packet.free(p) end + end + -- Decapsulation path + local input = self.input.encapsulated + local output = self.output.decapsulated + for _=1,link.nreadable(input) do + local p = link.receive(input) + if self.decrypt:decapsulate(p) then link.transmit(output, p) + else packet.free(p) end + end +end diff --git a/src/bench/snabbnfv-iperf-1500-crypto b/src/bench/snabbnfv-iperf-1500-crypto new file mode 100755 index 0000000000..7cbfb2fa24 --- /dev/null +++ b/src/bench/snabbnfv-iperf-1500-crypto @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e +out=$(program/snabbnfv/selftest.sh bench 1500 \ + program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports) +# Extract floating point Gbits number from output. +echo "$out" | grep IPERF-1500 | cut -d " " -f 2 diff --git a/src/bench/snabbnfv-iperf-1500-tunnel+crypto b/src/bench/snabbnfv-iperf-1500-tunnel+crypto new file mode 100755 index 0000000000..7967c28d26 --- /dev/null +++ b/src/bench/snabbnfv-iperf-1500-tunnel+crypto @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e +out=$(program/snabbnfv/selftest.sh bench 1500 \ + program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto-tunnel.ports) +# Extract floating point Gbits number from output. +echo "$out" | grep IPERF-1500 | cut -d " " -f 2 diff --git a/src/program/snabbnfv/README.md b/src/program/snabbnfv/README.md index 43e503ecfc..2b7d125ae0 100644 --- a/src/program/snabbnfv/README.md +++ b/src/program/snabbnfv/README.md @@ -50,6 +50,7 @@ port := { port_id = , -- A unique string ingress_filter = , -- A pcap-filter(7) expression egress_filter = , -- .. tunnel = , + crypto = , rx_police_gbps = , -- Allowed input rate in Gbps tx_police_gbps = } -- Allowed output rate in Gbps ``` @@ -64,9 +65,20 @@ tunnel := { type = "L2TPv3", -- The only type (for now) next_hop = , -- Gateway IP local_ip = , -- ~ `local_address' remote_ip = , -- ~ `remote_address' - session = <32bit-int> -- ~ `session_id' } + session = <32bit-int> } -- ~ `session_id' ``` +The `crypto` section allows configuration of traffic encryption based on +`apps.esp`: + +``` +crypto := { type = "esp-aes-128-gcm", -- The only type (for now) + spi = , -- Security Parameter Index + key = , -- 20 bytes as a hex encoded string + replay_window = } -- Replay window +``` + + ### snabbnfv traffic The `snabbnfv traffic` program loads and runs a NFV configuration using diff --git a/src/program/snabbnfv/nfvconfig.lua b/src/program/snabbnfv/nfvconfig.lua index c8948b1071..8977b6477d 100644 --- a/src/program/snabbnfv/nfvconfig.lua +++ b/src/program/snabbnfv/nfvconfig.lua @@ -7,6 +7,7 @@ local PcapFilter = require("apps.packet_filter.pcap_filter").PcapFilter local RateLimiter = require("apps.rate_limiter.rate_limiter").RateLimiter local nd_light = require("apps.ipv6.nd_light").nd_light local L2TPv3 = require("apps.keyed_ipv6_tunnel.tunnel").SimpleKeyedTunnel +local AES128gcm = require("apps.ipsec.esp").AES128gcm local pci = require("lib.hardware.pci") local ffi = require("ffi") local C = ffi.C @@ -94,6 +95,13 @@ function load (file, pciaddr, sockpath) config.link(c, Tunnel..".decapsulated -> "..VM_rx) VM_rx, VM_tx = ND..".south", ND..".south" end + if t.crypto and t.crypto.type == "esp-aes-128-gcm" then + local Crypto = name.."_Crypto" + config.app(c, Crypto, AES128gcm, t.crypto) + config.link(c, VM_tx.." -> "..Crypto..".decapsulated") + config.link(c, Crypto..".decapsulated -> "..VM_rx) + VM_rx, VM_tx = Crypto..".encapsulated", Crypto..".encapsulated" + end if t.rx_police_gbps then local RxLimit = name.."_RxLimit" local rate = t.rx_police_gbps * 1e9 / 8 diff --git a/src/program/snabbnfv/selftest.sh b/src/program/snabbnfv/selftest.sh index b836baf293..795f0d907d 100755 --- a/src/program/snabbnfv/selftest.sh +++ b/src/program/snabbnfv/selftest.sh @@ -288,10 +288,25 @@ function filter_tests { assert FILTER $? } -# Usage: iperf_bench [] +function crypto_tests { + load_config program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports + + test_ping $SNABB_TELNET0 "$(ip 1)%eth0" + test_iperf $SNABB_TELNET0 $SNABB_TELNET1 "$(ip 1)%eth0" + test_jumboping $SNABB_TELNET0 $SNABB_TELNET1 "$(ip 1)%eth0" + # Repeat iperf test now that jumbo frames are enabled + test_iperf $SNABB_TELNET0 $SNABB_TELNET1 "$(ip 1)%eth0" +} + +# Usage: iperf_bench [] [] # Run iperf benchmark. If is "jumbo", jumboframes will be enabled. +# defaults to same_vlan.ports. function iperf_bench { - load_config program/snabbnfv/test_fixtures/nfvconfig/test_functions/same_vlan.ports + if [ -z "$2" ]; then + load_config program/snabbnfv/test_fixtures/nfvconfig/test_functions/same_vlan.ports + else + load_config "$2" + fi if [ "$1" = "jumbo" ]; then test_jumboping $SNABB_TELNET0 $SNABB_TELNET1 "$(ip 1)%eth0" \ @@ -324,7 +339,7 @@ start_test_env # Decide which mode to run (`test', `bench' or `fuzz'). case $1 in bench) - iperf_bench "$2" + iperf_bench "$2" "$3" ;; fuzz) fuzz_tests "$2" @@ -334,6 +349,7 @@ case $1 in rate_limited_tests tunnel_tests filter_tests + crypto_tests esac exit 0 diff --git a/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto-tunnel.ports b/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto-tunnel.ports new file mode 100644 index 0000000000..e81323b981 --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto-tunnel.ports @@ -0,0 +1,30 @@ +return { + { vlan = 43, + mac_address = "52:54:00:00:00:00", + port_id = "A", + tunnel = { type = "L2TPv3", + remote_ip = "fe80:0:0:0:5054:ff:fe00:1", + local_ip = "fe80:0:0:0:5054:ff:fe00:0", + session = 16, + local_cookie = "deadbeef", + remote_cookie = "deadbeef", + next_hop = "fe80:0:0:0:5054:ff:fe00:1" }, + crypto = { type = "esp-aes-128-gcm", + spi = 0x42, + key = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", } + }, + { vlan = 43, + mac_address = "52:54:00:00:00:01", + port_id = "B", + tunnel = { type = "L2TPv3", + remote_ip = "fe80:0:0:0:5054:ff:fe00:0", + local_ip = "fe80:0:0:0:5054:ff:fe00:1", + session = 16, + local_cookie = "deadbeef", + remote_cookie = "deadbeef", + next_hop = "fe80:0:0:0:5054:ff:fe00:0" }, + crypto = { type = "esp-aes-128-gcm", + spi = 0x42, + key = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", } + }, +} diff --git a/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports b/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports new file mode 100644 index 0000000000..dbb9bafc0c --- /dev/null +++ b/src/program/snabbnfv/test_fixtures/nfvconfig/test_functions/crypto.ports @@ -0,0 +1,16 @@ +return { + { vlan = 43, + mac_address = "52:54:00:00:00:00", + port_id = "A", + crypto = { type = "esp-aes-128-gcm", + spi = 0x42, + key = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", } + }, + { vlan = 43, + mac_address = "52:54:00:00:00:01", + port_id = "B", + crypto = { type = "esp-aes-128-gcm", + spi = 0x42, + key = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", } + }, +} diff --git a/src/program/snabbnfv/traffic/README b/src/program/snabbnfv/traffic/README index 2447137d47..c961bf2aac 100644 --- a/src/program/snabbnfv/traffic/README +++ b/src/program/snabbnfv/traffic/README @@ -42,6 +42,7 @@ CONFIG FILE FORMAT: ingress_filter = , -- A pcap-filter(7) expression egress_filter = , -- .. tunnel = , + crypto = , rx_police_gbps = , -- Allowed input rate in Gbps tx_police_gbps = } -- Allowed output rate in Gbps @@ -55,3 +56,12 @@ CONFIG FILE FORMAT: local_ip = , -- ~ `local_address' remote_ip = , -- ~ `remote_address' session = <32bit-int> -- ~ `session_id' } + + The crypto section allows configuration of traffic encryption based on + apps.ipsec.esp: + + + crypto := { type = "esp-aes-128-gcm", -- The only type (for now) + spi = , -- Security Parameter Index + key = , -- 20 bytes as a hex encoded string + replay_window = } -- Replay window \ No newline at end of file From ab5d1af91582a3b0fdd72c2ab8703fc6c3e52134 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 20 Jun 2016 18:47:17 +0200 Subject: [PATCH 089/103] scripts/dock.sh: use eugeneia/snabb-nfv-test-vanilla by default. --- src/scripts/dock.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/dock.sh b/src/scripts/dock.sh index a06a5b9ee2..0486b57d6a 100755 --- a/src/scripts/dock.sh +++ b/src/scripts/dock.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export SNABB_TEST_IMAGE=${SNABB_TEST_IMAGE:=eugeneia/snabb-nfv-test} +export SNABB_TEST_IMAGE=${SNABB_TEST_IMAGE:=eugeneia/snabb-nfv-test-vanilla} # Snabb Docker environment From 626c6fc2d6da3029e743092caca9cc3876cec991 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 21 Jun 2016 16:07:51 +0200 Subject: [PATCH 090/103] apps.keyed_ipv6_tunnel: add diagnostics counters. --- src/apps/ipv6/README.md | 23 ++++++++++++++++++++++ src/apps/keyed_ipv6_tunnel/tunnel.lua | 28 ++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/apps/ipv6/README.md b/src/apps/ipv6/README.md index 2ecdec57f1..4cf0fd7598 100644 --- a/src/apps/ipv6/README.md +++ b/src/apps/ipv6/README.md @@ -115,3 +115,26 @@ the L2TPv3 header will be overwritten with this value. *Optional*. Destination MAC as a string. Not required if overwritten by an app such as `nd_light`. + + +### Special Counters + +— Key **length_errors** + +Ingress packets dropped due to invalid length (packet too short). + +— Key **protocol_errors** + +Ingress packets dropped due to unrecognized IPv6 protocol ID. + +— Key **cookie_errors** + +Ingress packets dropped due to wrong cookie value. + +— Key **remote_address_errors** + +Ingress packets dropped due to wrong remote IPv6 endpoint address. + +— Key **local_address_errors** + +Ingress packets dropped due to wrong local IPv6 endpoint address. diff --git a/src/apps/keyed_ipv6_tunnel/tunnel.lua b/src/apps/keyed_ipv6_tunnel/tunnel.lua index 9e394cfcfc..e29951fc10 100644 --- a/src/apps/keyed_ipv6_tunnel/tunnel.lua +++ b/src/apps/keyed_ipv6_tunnel/tunnel.lua @@ -16,6 +16,7 @@ local link = require("core.link") local lib = require("core.lib") local packet = require("core.packet") local config = require("core.config") +local counter = require("core.counter") local macaddress = require("lib.macaddress") @@ -102,6 +103,12 @@ end SimpleKeyedTunnel = {} +local provided_counters = { + 'type', 'dtime', 'rxerrors', + 'length_errors', 'protocol_errors', 'cookie_errors', + 'remote_address_errors', 'local_address_errors' +} + function SimpleKeyedTunnel:new (arg) local conf = arg and config.parse_app_arg(arg) or {} -- required fields: @@ -163,12 +170,20 @@ function SimpleKeyedTunnel:new (arg) header[HOP_LIMIT_OFFSET] = conf.hop_limit end + local counters = {} + for _, name in ipairs(provided_counters) do + counters[name] = counter.open(name) + end + counter.set(counters.type, 0x1001) -- Virtual interface + counter.set(counters.dtime, C.get_unix_time()) + local o = { header = header, remote_address = remote_address, local_address = local_address, - remote_cookie = remote_cookie[0] + remote_cookie = remote_cookie[0], + counters = counters } return setmetatable(o, {__index = SimpleKeyedTunnel}) @@ -198,15 +213,18 @@ function SimpleKeyedTunnel:push() local drop = true repeat if p.length < HEADER_SIZE then + counter.add(self.counters.length_errors) break end local next_header = ffi.cast(next_header_ctype, p.data + NEXT_HEADER_OFFSET) if next_header[0] ~= L2TPV3_NEXT_HEADER then + counter.add(self.counters.protocol_errors) break end local cookie = ffi.cast(pcookie_ctype, p.data + COOKIE_OFFSET) if cookie[0] ~= self.remote_cookie then + counter.add(self.counters.cookie_errors) break end @@ -214,6 +232,7 @@ function SimpleKeyedTunnel:push() if remote_address[0] ~= self.remote_address[0] or remote_address[1] ~= self.remote_address[1] then + counter.add(self.counters.remote_address_errors) break end @@ -221,6 +240,7 @@ function SimpleKeyedTunnel:push() if local_address[0] ~= self.local_address[0] or local_address[1] ~= self.local_address[1] then + counter.add(self.counters.local_address_errors) break end @@ -228,6 +248,7 @@ function SimpleKeyedTunnel:push() until true if drop then + counter.add(self.counters.rxerrors) -- discard packet packet.free(p) else @@ -237,6 +258,11 @@ function SimpleKeyedTunnel:push() end end +function SimpleKeyedTunnel:stop () + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end +end + -- prepare header template to be used by all apps prepare_header_template() From 1b60645552e80325471a76a546bca209207097b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 23 Jun 2016 11:05:22 +0200 Subject: [PATCH 091/103] default.nix: don't hardcode the version Nix expressions will always just set it according to source, but let's still provide a default for convenience. --- .version | 1 - default.nix | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 .version diff --git a/.version b/.version deleted file mode 100644 index d1ff275cc5..0000000000 --- a/.version +++ /dev/null @@ -1 +0,0 @@ -2016.06 diff --git a/default.nix b/default.nix index 20bccef8c0..ab7faf4cbf 100644 --- a/default.nix +++ b/default.nix @@ -1,10 +1,10 @@ # Run like this: # nix-build /path/to/this/directory -# ... and the files are produced in ./result/ +# ... and the files are produced in ./result/bin/snabb { pkgs ? (import {}) , source ? ./. -, version ? builtins.replaceStrings ["\n"] [""] (builtins.readFile ./.version) +, version ? "dev" }: with pkgs; @@ -39,5 +39,4 @@ stdenv.mkDerivation rec { ''; enableParallelBuilding = true; - } From c0f70765096ad11358e1b4d777da8917770a14d1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 23 Jun 2016 15:50:06 +0200 Subject: [PATCH 092/103] apps.ipsec.esp: add statistics counters. --- src/apps/ipsec/esp.lua | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/apps/ipsec/esp.lua b/src/apps/ipsec/esp.lua index 9c2121320a..f468b883ad 100644 --- a/src/apps/ipsec/esp.lua +++ b/src/apps/ipsec/esp.lua @@ -5,9 +5,15 @@ module(..., package.seeall) local esp = require("lib.ipsec.esp") +local counter = require("core.counter") +local C = require("ffi").C AES128gcm = {} +local provided_counters = { + 'type', 'dtime', 'txerrors', 'rxerrors' +} + function AES128gcm:new (arg) local conf = arg and config.parse_app_arg(arg) or {} local self = {} @@ -22,6 +28,12 @@ function AES128gcm:new (arg) keymat = conf.key:sub(1, 32), salt = conf.key:sub(33, 40), window_size = conf.replay_window} + self.counters = {} + for _, name in ipairs(provided_counters) do + self.counters[name] = counter.open(name) + end + counter.set(self.counters.type, 0x1001) -- Virtual interface + counter.set(self.counters.dtime, C.get_unix_time()) return setmetatable(self, {__index = AES128gcm}) end @@ -31,15 +43,28 @@ function AES128gcm:push () local output = self.output.encapsulated for _=1,link.nreadable(input) do local p = link.receive(input) - if self.encrypt:encapsulate(p) then link.transmit(output, p) - else packet.free(p) end + if self.encrypt:encapsulate(p) then + link.transmit(output, p) + else + packet.free(p) + counter.add(self.counters.txerrors) + end end -- Decapsulation path local input = self.input.encapsulated local output = self.output.decapsulated for _=1,link.nreadable(input) do local p = link.receive(input) - if self.decrypt:decapsulate(p) then link.transmit(output, p) - else packet.free(p) end + if self.decrypt:decapsulate(p) then + link.transmit(output, p) + else + packet.free(p) + counter.add(self.counters.rxerrors) + end end end + +function AES128gcm:stop () + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end +end From fc7363cef6b1396dd4312c20a16b025d28f31f11 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 23 Jun 2016 15:50:33 +0200 Subject: [PATCH 093/103] apps.ipv6.nd_light: add statistics counters. --- src/apps/ipv6/README.md | 24 +++++++++++++++++++++++ src/apps/ipv6/nd_light.lua | 40 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/apps/ipv6/README.md b/src/apps/ipv6/README.md index 4cf0fd7598..e34836e4d0 100644 --- a/src/apps/ipv6/README.md +++ b/src/apps/ipv6/README.md @@ -56,6 +56,30 @@ milliseconds. Default is 1,000ms. *Optional*. Number of neighbor solicitation retransmissions. Default is unlimited retransmissions. +### Special Counters + +— Key **ns_checksum_errors** + +Neighbor solicitation requests dropped due to invalid ICMP checksum. + +— Key **ns_target_address_errors** + +Neighbor solicitation requests dropped due to invalid target address. + +— Key **na_duplicate_errors** + +Neighbor advertisement requests dropped because next-hop is already resolved. + +— Key **na_target_address_errors** + +Neighbor advertisement requests dropped due to invalid target address. + +— Key **nd_protocol_errors** + +Neighbor discovery requests dropped due to protocol errors (invalid IPv6 +hop-limit or invalid neighbor solicitation request options). + + ## SimpleKeyedTunnel (apps.keyed_ipv6_tunnel.tunnel) The `SimpleKeyedTunnel` app implements "a simple L2 Ethernet over IPv6 diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index 2717338d68..1f1c484be6 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -36,6 +36,7 @@ local app = require("core.app") local link = require("core.link") local config = require("core.config") local packet = require("core.packet") +local counter = require("core.counter") local datagram = require("lib.protocol.datagram") local ethernet = require("lib.protocol.ethernet") local ipv6 = require("lib.protocol.ipv6") @@ -78,6 +79,13 @@ local function check_ip_address(ip, desc) return ip end +local provided_counters = { + 'type', 'dtime', 'status', 'rxerrors', 'rxdrop', + 'ns_checksum_errors', 'ns_target_address_errors', + 'na_duplicate_errors', 'na_target_address_errors', + 'nd_protocol_errors' +} + function nd_light:new (arg) local arg = arg and config.parse_app_arg(arg) or {} --copy the args to avoid changing the arg table so that it stays reusable. @@ -201,6 +209,16 @@ function nd_light:new (arg) mem = ffi.new("uint8_t *[1]") } o._logger = lib.logger_new({ module = 'nd_light' }) + + -- Create counters + o.counters = {} + for _, name in ipairs(provided_counters) do + o.counters[name] = counter.open(name) + end + counter.set(o.counters.type, 0x1001) -- Virtual interface + counter.set(o.counters.dtime, C.get_unix_time()) + counter.set(o.counters.status, 2) -- Link down + return o end @@ -209,13 +227,16 @@ local function ns (self, dgram, eth, ipv6, icmp) local mem, length = self._cache.mem mem[0], length = dgram:payload() if not icmp:checksum_check(mem[0], length, ipv6) then - self._logger:log("bad icmp checksum") + counter.add(self.counters.rxerrors) + counter.add(self.counters.ns_checksum_errors) return nil end -- Parse the neighbor solicitation and check if it contains our own -- address as target local ns = dgram:parse_match(nil, self._match_ns) if not ns then + counter.add(self.counters.ns_target_address_errors) + counter.add(self.counters.rxerrors) return nil end -- Ignore options as long as we don't implement a proper neighbor @@ -236,15 +257,21 @@ end -- Process neighbor advertisement local function na (self, dgram, eth, ipv6, icmp) if self._eth_header then + counter.add(self.counters.na_duplicate_errors) + counter.add(self.counters.rxerrors) return nil end local na = dgram:parse_match(nil, self._match_na) if not na then + counter.add(self.counters.na_target_address_errors) + counter.add(self.counters.rxerrors) return nil end local option = na:options(dgram:payload()) if not (#option == 1 and option[1]:type() == 2) then -- Invalid NS, ignore + counter.add(self.counters.nd_protocol_errors) + counter.add(self.counters.rxerrors) return nil end self._eth_header = ethernet:new({ src = self._config.local_mac, @@ -252,6 +279,7 @@ local function na (self, dgram, eth, ipv6, icmp) type = 0x86dd }) self._logger:log(string.format("Resolved next-hop %s to %s", ipv6:ntop(self._config.next_hop), ethernet:ntop(option[1]:option():addr()))) + counter.set(self.counters.status, 1) -- Link up return nil end @@ -265,6 +293,8 @@ local function from_south (self, p) local eth, ipv6, icmp = unpack(dgram:stack()) if ipv6:hop_limit() ~= 255 then -- Avoid off-link spoofing as per RFC + counter.add(self.counters.nd_protocol_errors) + counter.add(self.counters.rxerrors) return nil end local result @@ -311,13 +341,17 @@ function nd_light:push () -- Drop packets until ND for the next-hop -- has completed. packet.free(link.receive(l_in)) + counter.add(self.counters.rxdrop) else local p = cache.p p[0] = link.receive(l_in) if packet.length(p[0]) >= self._eth_header:sizeof() then self._eth_header:copy(packet.data(p[0])) link.transmit(l_out, p[0]) - else packet.free(p[0]) end + else + packet.free(p[0]) + counter.add(self.counters.rxerrors) + end end end end @@ -328,6 +362,8 @@ function nd_light:stop () self._next_hop.packet = nil packet.free(self._sna.packet) self._sna.packet = nil + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end function selftest () From 272a0d1b431bc1b02a7ec504a83977b20a398b85 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 23 Jun 2016 15:51:07 +0200 Subject: [PATCH 094/103] apps.packet_filter: add statistics counters. --- src/apps/packet_filter/README.md | 6 ++++++ src/apps/packet_filter/pcap_filter.lua | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/apps/packet_filter/README.md b/src/apps/packet_filter/README.md index 084b646b87..e94afcc4c7 100644 --- a/src/apps/packet_filter/README.md +++ b/src/apps/packet_filter/README.md @@ -33,3 +33,9 @@ expression. *rule* will be tracked in the specified state table and any packet that belongs to a tracked connection in the specified state table will be let pass. + +## Special Counters + +— Key **sessions_established** + +Total number of sessions established. diff --git a/src/apps/packet_filter/pcap_filter.lua b/src/apps/packet_filter/pcap_filter.lua index 6ee4eabb52..43e1a3917d 100644 --- a/src/apps/packet_filter/pcap_filter.lua +++ b/src/apps/packet_filter/pcap_filter.lua @@ -7,12 +7,18 @@ local link = require("core.link") local lib = require("core.lib") local packet = require("core.packet") local config = require("core.config") +local counter = require("core.counter") local conntrack = require("apps.packet_filter.conntrack") +local C = require("ffi").C local pf = require("pf") -- pflua PcapFilter = {} +local provided_counters = { + 'dtime', 'type', 'rxerrors', 'sessions_established' +} + -- PcapFilter is an app that drops all packets that don't match a -- specified filter expression. -- @@ -33,6 +39,13 @@ function PcapFilter:new (conf) state_table = conf.state_table or false } if conf.state_table then conntrack.define(conf.state_table) end + -- Create counters + o.counters = {} + for _, name in ipairs(provided_counters) do + o.counters[name] = counter.open(name) + end + counter.set(o.counters.type, 0x1001) -- Virtual interface + counter.set(o.counters.dtime, C.get_unix_time()) return setmetatable(o, { __index = PcapFilter }) end @@ -47,14 +60,23 @@ function PcapFilter:push () if spec and spec:check(self.state_table) then link.transmit(o, p) elseif self.accept_fn(p.data, p.length) then - if spec then spec:track(self.state_table) end + if spec then + spec:track(self.state_table) + counter.add(self.counters.sessions_established) + end link.transmit(o, p) else packet.free(p) + counter.add(self.counters.rxerrors) end end end +function PcapFilter:stop () + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end +end + -- Testing local pcap = require("apps.pcap.pcap") From 506c66b1fc042f793c6658c8c01d22e9afa017d2 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 23 Jun 2016 15:51:56 +0200 Subject: [PATCH 095/103] apps.rate_limiter: add statistics counters. --- src/apps/rate_limiter/rate_limiter.lua | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/apps/rate_limiter/rate_limiter.lua b/src/apps/rate_limiter/rate_limiter.lua index 98c1cd9347..421e5ec3e0 100644 --- a/src/apps/rate_limiter/rate_limiter.lua +++ b/src/apps/rate_limiter/rate_limiter.lua @@ -7,6 +7,7 @@ local link = require("core.link") local config = require("core.config") local packet = require("core.packet") local timer = require("core.timer") +local counter = require("core.counter") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local C = ffi.C @@ -22,6 +23,10 @@ local floor, min = math.floor, math.min RateLimiter = {} +local provided_counters = { + 'type', 'dtime', 'txdrop' +} + -- Source produces synthetic packets of such size local PACKET_SIZE = 60 @@ -30,11 +35,18 @@ function RateLimiter:new (arg) assert(conf.rate) assert(conf.bucket_capacity) conf.initial_capacity = conf.initial_capacity or conf.bucket_capacity + local counters = {} + for _, name in ipairs(provided_counters) do + counters[name] = counter.open(name) + end + counter.set(counters.type, 0x1001) -- Virtual interface + counter.set(counters.dtime, C.get_unix_time()) local o = { rate = conf.rate, bucket_capacity = conf.bucket_capacity, - bucket_content = conf.initial_capacity + bucket_content = conf.initial_capacity, + counters = counters } return setmetatable(o, {__index=RateLimiter}) end @@ -81,11 +93,17 @@ function RateLimiter:push () link.transmit(o, p) else -- discard packet + counter.add(self.counters.txdrop) packet.free(p) end end end +function RateLimiter:stop () + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end +end + local function compute_effective_rate (rl, rate, snapshot) local elapsed_time = (tonumber(C.get_time_ns()) - snapshot.time) / 1e9 From 398dd163dd55ee165c23e420b39ce1beac6bf29d Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 1 Jul 2016 09:15:22 +0000 Subject: [PATCH 096/103] core.main: Fix bug in SNABB_DEBUG env var handling Bug was to set _G.developer_debug to nil when debugging is disabled. In Lua you cannot use nil as a table value and should use false instead. Further discussion at: https://github.com/snabbco/snabb/pull/956#issuecomment-229893468 --- src/core/main.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index f736e25458..112cdd5491 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -29,7 +29,8 @@ ffi.cdef[[ extern char** argv; ]] -_G.developer_debug = lib.getenv("SNABB_DEBUG") and true +-- Enable developer-level debug if SNABB_DEBUG env variable is set. +_G.developer_debug = lib.getenv("SNABB_DEBUG") ~= nil debug_on_error = _G.developer_debug function main () From e81a192d2e9c465de40728778e3d59eca0855969 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 1 Jul 2016 09:22:55 +0000 Subject: [PATCH 097/103] core.timer: Inhibit debug messages Print debug messages only when enabled by editing the source file, not when _G.developer_debug is enabled. The debug mode prints a message every time a timer is executed. This could easily produce thousands of messages per second in reasonable applications. This seems appropriate when specifically debugging the timer module but not when enabling a more general debug mode. --- src/core/timer.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/timer.lua b/src/core/timer.lua index 71a3afd2e3..79236c28aa 100644 --- a/src/core/timer.lua +++ b/src/core/timer.lua @@ -7,7 +7,8 @@ local C = ffi.C local lib = require("core.lib") -debug = _G.developer_debug +-- Enable only when debugging this module. Can be very verbose. +local debug = false ticks = false -- current time, in ticks ns_per_tick = 1e6 -- tick resolution (millisecond) From 995f9d89d9ef0c35492a9597dc17a96868e38e35 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 8 Jun 2016 10:28:45 +0000 Subject: [PATCH 098/103] vhost-user: Support operation without MRG_RXBUF. MRG_RXBUF is enabled by default in the beginning, and QEMU will initially negotiate a feature set with Snabb NFV that includes MRG_RXBUF. This adds a field onto the virtio header, in legacy mode. However if we later negotiate to not have MRG_RXBUF, we need to re-set this value to not have the extra fields. (cherry picked from commit df8e83c6771a7b892324922f25d5123a69f651d9) --- src/lib/virtio/net_device.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/virtio/net_device.lua b/src/lib/virtio/net_device.lua index d259e92cff..14fa5c9b1e 100644 --- a/src/lib/virtio/net_device.lua +++ b/src/lib/virtio/net_device.lua @@ -392,6 +392,10 @@ function VirtioNetDevice:set_features(features) self.hdr_type = virtio_net_hdr_mrg_rxbuf_type self.hdr_size = virtio_net_hdr_mrg_rxbuf_size self.mrg_rxbuf = true + else + self.hdr_type = virtio_net_hdr_type + self.hdr_size = virtio_net_hdr_size + self.mrg_rxbuf = false end if band(self.features, C.VIRTIO_RING_F_INDIRECT_DESC) == C.VIRTIO_RING_F_INDIRECT_DESC then for i = 0, max_virtq_pairs-1 do From e3fcbbfbe5d3c6a2ec41fb13e8d02cae2b53afc7 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 4 Jul 2016 14:38:18 +0200 Subject: [PATCH 099/103] Revert "lib.protocol.ethernet: Add n_mcast, branch-free Multicast predicate." Reason: its actually slower than the initial naive version. This reverts commit c186591e312fb90c4c2d847be01255fb7dba0c8b. # Conflicts: # src/apps/vhost/vhost_user.lua --- src/apps/vhost/vhost_user.lua | 8 ++++++-- src/lib/protocol/README.md | 9 --------- src/lib/protocol/ethernet.lua | 7 +------ 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index fe2711769e..0bc822bcae 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -107,13 +107,17 @@ end function VhostUser:tx_callback (p) counter.add(self.counters.txbytes, packet.length(p)) counter.add(self.counters.txpackets) - counter.add(self.counters.txmcast, ethernet:n_mcast(packet.data(p))) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters.txmcast) + end end function VhostUser:rx_callback (p) counter.add(self.counters.rxbytes, packet.length(p)) counter.add(self.counters.rxpackets) - counter.add(self.counters.rxmcast, ethernet:n_mcast(packet.data(p))) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters.rxmcast) + end end function VhostUser:rxdrop_callback (p) diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 4a13483a4b..6f239687d7 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -98,15 +98,6 @@ Returns the binary representation of MAC address denoted by *string*. Returns the string representation of *mac* address. -— Function **ethernet:is_mcast** *mac* - -Returns a true value if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet). - -— Function **ethernet:n_mcast** *mac* - -Returns 1 if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet) -and 0 otherwise. - — Function **ethernet:ipv6_mcast** *ip* Returns the MAC address for IPv6 multicast *ip* as defined by RFC2464, diff --git a/src/lib/protocol/ethernet.lua b/src/lib/protocol/ethernet.lua index b5abed8d31..0a55c20f1b 100644 --- a/src/lib/protocol/ethernet.lua +++ b/src/lib/protocol/ethernet.lua @@ -76,14 +76,9 @@ function ethernet:ipv6_mcast(ip) return result end --- Return 1 if MAC address has its group bit set and 0 otherwise -function ethernet:n_mcast (addr) - return band(addr[0], 0x01) -end - -- Check whether a MAC address has its group bit set function ethernet:is_mcast (addr) - return ethernet:n_mcast(addr) ~= 0 + return band(addr[0], 0x01) ~= 0 end -- Instance methods From b418f8f7f6fe1a1ed993186fe98a276b46c729e0 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 4 Jul 2016 15:50:29 +0200 Subject: [PATCH 100/103] =?UTF-8?q?intel=5Fapp:=20fix=20wrong=20=E2=80=9Cs?= =?UTF-8?q?peed=E2=80=9D=20counter=20value.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apps/intel/intel_app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 92f97d00fd..c85c2e5530 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -73,7 +73,7 @@ function Intel82599:new (arg) counter.set(self.stats.counters.type, 0x1000) -- Hardware interface counter.set(self.stats.counters.dtime, C.get_unix_time()) counter.set(self.stats.counters.mtu, self.dev.mtu) - counter.set(self.stats.counters.speed, 10000000) -- 10 Gbits + counter.set(self.stats.counters.speed, 10000000000) -- 10 Gbits counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then counter.set(self.stats.counters.macaddr, From d52d89cd9936f2d2f396876d8f8786258f0d3651 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 4 Jul 2016 15:58:33 +0200 Subject: [PATCH 101/103] apps.ipv6.nd_light: revise counters, south = rx / north = tx. --- src/apps/ipv6/nd_light.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index 1f1c484be6..b01b85eb91 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -80,7 +80,7 @@ local function check_ip_address(ip, desc) end local provided_counters = { - 'type', 'dtime', 'status', 'rxerrors', 'rxdrop', + 'type', 'dtime', 'status', 'rxerrors', 'txerrors', 'txdrop', 'ns_checksum_errors', 'ns_target_address_errors', 'na_duplicate_errors', 'na_target_address_errors', 'nd_protocol_errors' @@ -227,8 +227,8 @@ local function ns (self, dgram, eth, ipv6, icmp) local mem, length = self._cache.mem mem[0], length = dgram:payload() if not icmp:checksum_check(mem[0], length, ipv6) then - counter.add(self.counters.rxerrors) counter.add(self.counters.ns_checksum_errors) + counter.add(self.counters.rxerrors) return nil end -- Parse the neighbor solicitation and check if it contains our own @@ -341,7 +341,7 @@ function nd_light:push () -- Drop packets until ND for the next-hop -- has completed. packet.free(link.receive(l_in)) - counter.add(self.counters.rxdrop) + counter.add(self.counters.txdrop) else local p = cache.p p[0] = link.receive(l_in) @@ -350,7 +350,7 @@ function nd_light:push () link.transmit(l_out, p[0]) else packet.free(p[0]) - counter.add(self.counters.rxerrors) + counter.add(self.counters.txerrors) end end end From 900ff8e82884a1df757fa6f83189ffe2c157ca89 Mon Sep 17 00:00:00 2001 From: Alexander Gall Date: Mon, 4 Jul 2016 17:36:19 +0200 Subject: [PATCH 102/103] Fix interface speed The counter value for the interface speed should be in units of bps. The ifSpeed SNMP object must obey RFC3635 sec. 3.2.8. --- src/apps/intel/intel_app.lua | 2 +- src/lib/ipc/shmem/iftable_mib.lua | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 92f97d00fd..c85c2e5530 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -73,7 +73,7 @@ function Intel82599:new (arg) counter.set(self.stats.counters.type, 0x1000) -- Hardware interface counter.set(self.stats.counters.dtime, C.get_unix_time()) counter.set(self.stats.counters.mtu, self.dev.mtu) - counter.set(self.stats.counters.speed, 10000000) -- 10 Gbits + counter.set(self.stats.counters.speed, 10000000000) -- 10 Gbits counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then counter.set(self.stats.counters.macaddr, diff --git a/src/lib/ipc/shmem/iftable_mib.lua b/src/lib/ipc/shmem/iftable_mib.lua index 8d96d29e67..c03f98c29a 100644 --- a/src/lib/ipc/shmem/iftable_mib.lua +++ b/src/lib/ipc/shmem/iftable_mib.lua @@ -30,8 +30,13 @@ function init_snmp (name, counters, directory, interval) ifTable:register('ifSpeed', 'Gauge32') ifTable:register('ifHighSpeed', 'Gauge32') if counters.speed then - ifTable:set('ifSpeed', counter.read(counters.speed)) - ifTable:set('ifHighSpeed', counter.read(counters.speed) / 1000) + speed = counters.read(counters.speed) + if speed > 1000000000 then + ifTable:set('ifSpeed', 4294967295) -- RFC3635 sec. 3.2.8 + else + ifTable:set('ifSpeed', speed) + end + ifTable:set('ifHighSpeed', speed / 1000000) end ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }) if counters.macaddr then From 905bf8dd6b12f2347780664c9ea5a83fff7a69c1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 8 Jul 2016 16:23:24 +0200 Subject: [PATCH 103/103] vhost/vhost_user: avoid callbacks. --- src/apps/vhost/vhost_user.lua | 21 --------------------- src/lib/virtio/net_device.lua | 25 +++++++++++++++++++++---- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 0bc822bcae..b81a974404 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -15,7 +15,6 @@ local main = require("core.main") local memory = require("core.memory") local counter = require("core.counter") local pci = require("lib.hardware.pci") -local ethernet = require("lib.protocol.ethernet") local net_device= require("lib.virtio.net_device") local timer = require("core.timer") local ffi = require("ffi") @@ -104,26 +103,6 @@ function VhostUser:push () end end -function VhostUser:tx_callback (p) - counter.add(self.counters.txbytes, packet.length(p)) - counter.add(self.counters.txpackets) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters.txmcast) - end -end - -function VhostUser:rx_callback (p) - counter.add(self.counters.rxbytes, packet.length(p)) - counter.add(self.counters.rxpackets) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters.rxmcast) - end -end - -function VhostUser:rxdrop_callback (p) - counter.add(self.counters.rxdrop) -end - -- Try to connect to QEMU. function VhostUser:client_connect () return C.vhost_user_connect(self.socket_path) diff --git a/src/lib/virtio/net_device.lua b/src/lib/virtio/net_device.lua index e01ac7f86b..cb9fca2863 100644 --- a/src/lib/virtio/net_device.lua +++ b/src/lib/virtio/net_device.lua @@ -10,6 +10,8 @@ local link = require("core.link") local memory = require("core.memory") local packet = require("core.packet") local timer = require("core.timer") +local counter = require("core.counter") +local ethernet = require("lib.protocol.ethernet") local vq = require("lib.virtio.virtq_device") local checksum = require("lib.checksum") local ffi = require("ffi") @@ -151,6 +153,7 @@ end function VirtioNetDevice:rx_packet_end(header_id, total_size, rx_p) local l = self.owner.output.tx + local counters = self.owner.counters if l then if band(self.rx_hdr_flags, C.VIO_NET_HDR_F_NEEDS_CSUM) ~= 0 and -- Bounds-check the checksum area @@ -162,11 +165,15 @@ function VirtioNetDevice:rx_packet_end(header_id, total_size, rx_p) rx_p.length - self.rx_hdr_csum_start, self.rx_hdr_csum_offset) end - self.owner:rx_callback(rx_p) + counter.add(counters.rxbytes, rx_p.length) + counter.add(counters.rxpackets) + if ethernet:is_mcast(rx_p.data) then + counter.add(counters.rxmcast) + end link.transmit(l, rx_p) else debug("droprx", "len", rx_p.length) - self.owner:rxdrop_callback(rx_p) + counter.add(counters.rxdrop) packet.free(rx_p) end self.virtq[self.ring_id]:put_buffer(header_id, total_size) @@ -254,7 +261,12 @@ function VirtioNetDevice:tx_buffer_add(tx_p, addr, len) end function VirtioNetDevice:tx_packet_end(header_id, total_size, tx_p) - self.owner:tx_callback(tx_p) + local counters = self.owner.counters + counter.add(counters.txbytes, tx_p.length) + counter.add(counters.txpackets) + if ethernet:is_mcast(tx_p.data) then + counter.add(counters.txmcast) + end packet.free(tx_p) self.virtq[self.ring_id]:put_buffer(header_id, total_size) end @@ -313,9 +325,14 @@ function VirtioNetDevice:tx_buffer_add_mrg_rxbuf(tx_p, addr, len) end function VirtioNetDevice:tx_packet_end_mrg_rxbuf(header_id, total_size, tx_p) + local counters = self.owner.counters -- free the packet only when all its data is processed if self.tx.finished then - self.owner:tx_callback(tx_p) + counter.add(counters.txbytes, tx_p.length) + counter.add(counters.txpackets) + if ethernet:is_mcast(tx_p.data) then + counter.add(counters.txmcast) + end packet.free(tx_p) self.tx.p = nil self.tx.data_sent = nil