From b34c3eee2bf5cb05a6f47c0ed9e318632c8dc534 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 22 Feb 2016 13:06:18 +0000 Subject: [PATCH 01/53] engine: Set shm path to "app/$name" Set the shared memory path (shm.path) to a private namespace for each app with prefix "app/$name". This means that apps can create shm objects such as counters and by default these will appear in a local namespace for that app. --- src/core/app.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index 35a5305631..099fc3125a 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -5,6 +5,7 @@ local lib = require("core.lib") local link = require("core.link") local config = require("core.config") local timer = require("core.timer") +local shm = require("core.shm") local counter = require("core.counter") local zone = require("jit.zone") local ffi = require("ffi") @@ -65,6 +66,8 @@ end -- Run app:methodname() in protected mode (pcall). If it throws an -- error app will be marked as dead and restarted eventually. local function with_restart (app, method) + local oldshm = shm.path + shm.path = app.shmpath if use_restart then -- Run fn in protected mode using pcall. local status, err = pcall(method, app) @@ -75,6 +78,7 @@ local function with_restart (app, method) else method(app) end + shm.path = oldshm end -- Restart dead apps. @@ -162,7 +166,11 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg + local shmpath, shmorig = "app/"..name, shm.path + shm.path = shmpath local app = class:new(arg) + shm.path = shmorig + local shmpath = "app/"..name if type(app) ~= 'table' then error(("bad return value from app '%s' start() method: %s"):format( name, tostring(app))) @@ -171,6 +179,7 @@ function apply_config_actions (actions, conf) app.appname = name app.output = {} app.input = {} + app.shmpath = shmpath new_app_table[name] = app table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array From 80614e1bd4b34d6c4ebf35289417cb00508fb745 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 18 Mar 2016 06:03:29 +0000 Subject: [PATCH 02/53] doc/git-workflow.md: Rewritten based on new experience I rewrote the git-workflow explanation with the goal of briefly but clearly explaining both how to submit a change to Snabb Switch and also how to be a subsystem maintainer. This is intended to be a chapter in the manual. --- src/doc/git-workflow.md | 110 +++++++++++++++------------------------- 1 file changed, 42 insertions(+), 68 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 353384cf3e..7560bd927a 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -1,86 +1,60 @@ -# Snabb Switch Git Workflow +## Git workflow -This document explains the Git workflows that we use to develop and -release Snabb Switch. +Snabb Switch is developed on Github using the distributed workflow +that was pioneered by the Linux kernel. -## Overview +The latest Snabb Switch release can always be found on the `master` +branch of the Github `SnabbCo/snabbswitch` repository. -Snabb Switch development follows a few well-known patterns: +### Being a contributor -- We follow the distributed development model [described by Linus - Torvald](https://www.youtube.com/watch?v=4XpnKHJAok8) and pioneered - by the Linux kernel. -- We use a [merge based workflow](https://www.atlassian.com/git/articles/git-team-workflows-merge-or-rebase/). -- We use the [fork and pull](https://help.github.com/articles/using-pull-requests/#fork--pull) - model of collaboration. -- We aim to support ad-hoc collaboration inspired by the - [DMZ Flow](https://gist.github.com/djspiewak/9f2f91085607a4859a66). +So you have decided to contribute an improvement to Snabb Switch. Great! This is what to do: -## HOWTO +1. Create your own "fork" of Snabb Switch on Github. Do this by clicking the "Fork" button on the Github UI. (You only have to do this the first time you are making a change.) +2. Create a topic branch based on `master`. Example: `git branch -b my-feature master`. +3. Develop your feature and push your branch to your Github fork. Example: `emacs`, `git commit`, `git push`. +4. Open a Github Pull Request from your branch to the SnabbCo/snabbswitch master branch. Clicking the Pull Request button on Github should do the right thing. -### Download and update the latest release +Once your Pull Request is open you can wait a short while, usually a day or two, for a maintainer to engage with you and take responsibility for merging your change "upstream". -1. Clone the [SnabbCo/snabbswitch](https://github.com/SnabbCo/snabbswitch) repository. -2. Check out and build the `master` branch. -3. Pull when you want to update to the latest stable release. +Tips for making your contribution smoothly: -### Develop and contribute an improvement +- Explain why your change is a good idea using the text area of the Pull Request. +- Don't rebase your branch after the pull request is open; make corrections by pushing new commits. +- If your change is a work in progress then prefix the Pull Request name with `[wip]`. +- If your change is a rough draft for early feedback then prefix the Pull Request name with `[sketch]`. -1. [Create your own fork](https://help.github.com/articles/fork-a-repo/) of Snabb Switch on Github. -2. Develop and debug your contribution on a new [topic branch](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows#Topic-Branches) based on the latest `master`. -3. Make a final cleanup of your code before review. (Last chance to rebase.) -4. Submit a Github [Pull Request](https://help.github.com/articles/using-pull-requests/#initiating-the-pull-request) - to the `master` branch. -5. Respond to feedback and correct problems by pushing additional commits. +### Being a maintainer -There are two milestones in the process of accepting your change: +So you have decided to help with the upstream maintenance of Snabb +Switch. Fantastic! You can do this by creating and maintaining a +"subsystem branch" where you take responsibility for reviewing and +merging some of the Pull Requests submitted to Snabb Switch. -1. Your change is merged onto a branch that feeds `master`, for - example `next`, `fixes`, `documentation-fixes`, or `nfv`. From this - point the owner of that branch will push your work upstream - together with other related changes. They might ask you for help - but otherwise your work is done. -2. Your change is merged onto `master`. This could happen in a series - of merge steps, for example `nfv->next->master`. Once this happens - your code has been officially released as part of Snabb Switch. +#### Registering a subsystem branch -### Develop and maintain a new program +The first step is to create and register your subsystem branch: -Snabb Switch includes programs like `snabbnfv`, `packetblaster`, and -`snsh`. Here is how you can create a new program and take charge of -its development. +1. Pick your technical area of interest. What kind of changes will you be responsible for reviewing and merging? Try to pick an area that is easy to identify, for example "the packetblaster program", "the Intel I350 device driver", or "the Git Workflow chapter of the manual". +2. Create a branch with a suitable name on your Github fork, for example `packetblaster`, `i350`, or `git-workflow`. This is where you will merge relevant changes. +3. Describe this branch in the file `src/doc/branches.md` and open a Github Pull Request. This will kick off two discussions: how to clearly identify the changes that you are responsible for, and to which "next-hop" upstream branch you should send the changes that you have accepted by merging. -1. [Fork](https://help.github.com/articles/fork-a-repo/) your own - repository on Github. -2. Create a [long-lived branch](branches.md) where new development of your program will be done. -3. Create a directory `src/program/myapplication/` and develop your program. -4. `git merge master` regularly to stay synchronized with the main line of development. -5. Optional: Send releases of your application to `master` with Pull Requests. +Once those details are worked out and your branch is registered then +you are a Snabb Switch maintainer. Congratulations! -The code in your `src/program/myapplication/` directory is developed -according to your own rules and tastes. If there are parts of this -code that you especially want to have reviewed (or do not want to have -reviewed) then please explain this in your Pull Request. The only -necessary review is to make sure that programs do not negatively -impact each other or change shared code without enough review. +#### Being "the upstream" for Pull Requests -Pull Requests that make changes to your application will be referred -to you for merge onto your branch. +Now as a maintainer your job is to watch for Pull Requests opened to +the Github repository and act as the responsible person (the +"upstream") when your branch most specifically matches a change. -Use the *Develop and contribute an improvement* workflow to make -changes to the core Snabb Switch code. Please do not bundle -substantial changes to the core software with updates to your program. - -If you do not want to include your program in the main Snabb Switch -release then this is no problem. You can simply pull from `master` to -receive updates and skip the step of pushing back. - -### To help maintain Snabb Switch - -Here are the best ways to help maintain Snabb Switch: - -1. Review Pull Requests to help people quickly improve them. -2. Test the `next` branch and help fix problems before releases. -3. Contribute new `selftest` cases to make our CI more effective. -4. Maintain a [branch](branches.md) where you accept Pull Requests and push them upstream. +Here is how to be the upstream for a change: +1. Set yourself as the *Assignee* of the Pull Request. This clearly signals to everybody that you are the one responsible for reviewing and merging the change. +2. Review the submitted changes: + 1. Does it all look good? If so then merge the changes onto your branch and add the label `merged` to the Pull Request. + 2. Do you see some serious problems? Tell the contributor exactly what they need to change in order for you to merge the changes. + 3. Do you see minor ways that the change could be improved? Suggest those and ask the contributor whether they want to do that before you merge the change. + 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. + 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. +3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. From 12d00096604974380d48ebf921344c392014f96f Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 18 Mar 2016 10:32:24 +0000 Subject: [PATCH 03/53] doc/git-workflow.md: Added section on upstreaming subsystems --- src/doc/git-workflow.md | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 7560bd927a..5a3835a30a 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -58,3 +58,56 @@ Here is how to be the upstream for a change: 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. 3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. + +#### Sending collected changes upstream to your next-hop + +So you merge some good changes onto your subsystem branch. What next? + +The next step is to open a Pull Request from your specific subsystem +branch to the more general "next hop" upstream branch. This is your +way to say "hey, I have collected some good changes here, please merge +them!" + +The upstreaming process is the same one described above, but now you +are the one submitting the changes and expecting clear feedback on +what actions you need to take for them to be accepted. + +#### Putting it all together + +Here is a complete example of how this can all fit together: + +You decide that you want to be the maintainer of the Intel I350 +ethernet driver. You create a branch called `i350` and open a Pull +Request to describe this branch in `src/doc/branches.md`. The other +maintainers are happy that you want to join in and gladly agree to +refer Pull Requests concerned the Intel I350 driver to you. You agree +that once you have good changes on your `i350` branch you will open a +Pull Request to the more general `drivers` branch as your "next hop" +upstream. + +People in the community start contributing improvements to the I350 +driver. You set yourself as the *Assignee* to these Pull Requests and +engage with the contributors to get the changes in good shape. You +merge the good changes onto your `i350` branch and periodically open a +Pull Request to the `drivers` branch to send this code upstream. The +`drivers` branch will in turn be merged to its next hop upstream and +step-by-step the changes will make their way towards release on the +`master` branch. + +The time scales involved are not written in stone but it is important +to find a rhythm that is comfortable for everybody involved. You might +aim to set yourself as the *Assignee* for relevant Pull Requests +within one day, to provide a review within a few days, and to send +Pull Requests to your next-hop upstream once or twice per week when +you have changes. + +#### Don't be shy! + +Becoming a Snabb Switch maintainer is a great service to the community +and you can learn all the skills that you need "on the job." If you are +tempted to give it a try then please do! + +The more subsystem maintainers we have the more capacity we have to +incorporate improvements into Snabb Switch. The Linux kernel has +more than one thousand registered subsystems. The sky is the limit! + From af8ae46df1f42e95ac7311cb1bd5e8e37acef952 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 21 Mar 2016 06:00:18 +0000 Subject: [PATCH 04/53] doc/git-workflow.md: Partly rewritten draft Contains "XXX rewrite" where old sections have been removed but replacements not written yet. --- src/doc/git-workflow.md | 190 +++++++++++++++++++--------------------- 1 file changed, 92 insertions(+), 98 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 5a3835a30a..16dae03e6e 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -1,113 +1,107 @@ ## Git workflow -Snabb Switch is developed on Github using the distributed workflow -that was pioneered by the Linux kernel. - -The latest Snabb Switch release can always be found on the `master` -branch of the Github `SnabbCo/snabbswitch` repository. - -### Being a contributor - -So you have decided to contribute an improvement to Snabb Switch. Great! This is what to do: - -1. Create your own "fork" of Snabb Switch on Github. Do this by clicking the "Fork" button on the Github UI. (You only have to do this the first time you are making a change.) -2. Create a topic branch based on `master`. Example: `git branch -b my-feature master`. -3. Develop your feature and push your branch to your Github fork. Example: `emacs`, `git commit`, `git push`. -4. Open a Github Pull Request from your branch to the SnabbCo/snabbswitch master branch. Clicking the Pull Request button on Github should do the right thing. - -Once your Pull Request is open you can wait a short while, usually a day or two, for a maintainer to engage with you and take responsibility for merging your change "upstream". - -Tips for making your contribution smoothly: - -- Explain why your change is a good idea using the text area of the Pull Request. -- Don't rebase your branch after the pull request is open; make corrections by pushing new commits. -- If your change is a work in progress then prefix the Pull Request name with `[wip]`. -- If your change is a rough draft for early feedback then prefix the Pull Request name with `[sketch]`. - -### Being a maintainer - -So you have decided to help with the upstream maintenance of Snabb -Switch. Fantastic! You can do this by creating and maintaining a -"subsystem branch" where you take responsibility for reviewing and -merging some of the Pull Requests submitted to Snabb Switch. +How do you engage with the Snabb Switch developer community? The answer depends on what you want to do: + +- Use the software, ask questions, report bugs. +- Contribute fixes and improvements. +- Maintain Snabb Switch by reviewing and merging pull requests. +- Create a new application to develop together with the community. + +### Using the software + +The recommended way to download Snabb Switch is with `git` directly +from from the `master` branch of the `snabbco` repository. This branch +always contains the latest release. + +``` +$ git checkout https://github.com/snabbco/snabbswitch +$ cd snabbswitch +$ make -j +``` + +The `master` branch is updated with a new release each month. You can +upgrade by pulling in the latest changes to your local copy: + +``` +$ git pull +$ make -j +``` + +This is safe to do at any time because the `master` branch is only +used for publishing the latest release to users. (The active software +development is done on different branches and only merged onto +`master` once it is ready for release.) + +You can also switch back and forth between different versions based on +their release tags: + +``` +$ git checkout v2016.01 # switch to a specific version +$ make -j +... +$ git checkout master # switch back to latest +$ make -j +``` + +### Contributing fixes and improvements + +The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. + +1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. +2. Push your proposed change to a branch on your repository. +3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. + +Here are some tips for making your contribution smoothly: + +- Use a dedicated "topic branch" for each feature or fix. +- Use the Pull Request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. + +### Becoming a maintainer + +Snabb Switch maintainers are the people who review and merge the Pull +Requests on Github. Each maintainer takes care of one or more specific +aspects of Snabb Switch. These aspects are called *subsystems*. + +Each subsystem has a dedicated branch and these branches are organized +as a network: + + DIAGRAM: Branches + +--lisper + +--max next<----+--documentation<--pdf manual + | + fixes | + | | + master<--+--next<--+--kbara next<--+--nix + | +--mellanox + | + | + +--wingo next<--+--lwaftr + +--multiproc + +Pull Requests are first merged onto the subsystem branch that most +specifically matches their subject matter. For example, a change to +the PDF formatting of the manual would first be reviewed and accepted +by the maintainer of the `pdf-manual` branch. Later, that whole +subsystem branch is merged onto its "next-hop upstream" branch. For +example, the maintainer of the `documentation` branch would merge the +entire `pdf-manual` branch at regular intervals. #### Registering a subsystem branch -The first step is to create and register your subsystem branch: - -1. Pick your technical area of interest. What kind of changes will you be responsible for reviewing and merging? Try to pick an area that is easy to identify, for example "the packetblaster program", "the Intel I350 device driver", or "the Git Workflow chapter of the manual". -2. Create a branch with a suitable name on your Github fork, for example `packetblaster`, `i350`, or `git-workflow`. This is where you will merge relevant changes. -3. Describe this branch in the file `src/doc/branches.md` and open a Github Pull Request. This will kick off two discussions: how to clearly identify the changes that you are responsible for, and to which "next-hop" upstream branch you should send the changes that you have accepted by merging. - -Once those details are worked out and your branch is registered then -you are a Snabb Switch maintainer. Congratulations! +XXX rewrite. #### Being "the upstream" for Pull Requests -Now as a maintainer your job is to watch for Pull Requests opened to -the Github repository and act as the responsible person (the -"upstream") when your branch most specifically matches a change. - -Here is how to be the upstream for a change: - -1. Set yourself as the *Assignee* of the Pull Request. This clearly signals to everybody that you are the one responsible for reviewing and merging the change. -2. Review the submitted changes: - 1. Does it all look good? If so then merge the changes onto your branch and add the label `merged` to the Pull Request. - 2. Do you see some serious problems? Tell the contributor exactly what they need to change in order for you to merge the changes. - 3. Do you see minor ways that the change could be improved? Suggest those and ask the contributor whether they want to do that before you merge the change. - 4. Is there somebody else who should also review the code? If so then ask them for help with a @mention. - 5. Do you see an obvious thing to fix that requires no discussion? You can simply do that yourself as part of your merge commit. -3. Manage the discussion. Everybody on Github is able to make comments on Pull Requests, often many people do, but as the upstream assignee you are the one who says what is necessary. Contributors can easily be overwhelmed by feedback from many sources so it is important for the upstream assignee to clearly explain what actions they have to take in order for their changes to be merged. +XXX rewrite. #### Sending collected changes upstream to your next-hop -So you merge some good changes onto your subsystem branch. What next? - -The next step is to open a Pull Request from your specific subsystem -branch to the more general "next hop" upstream branch. This is your -way to say "hey, I have collected some good changes here, please merge -them!" - -The upstreaming process is the same one described above, but now you -are the one submitting the changes and expecting clear feedback on -what actions you need to take for them to be accepted. +XXX rewrite. #### Putting it all together -Here is a complete example of how this can all fit together: - -You decide that you want to be the maintainer of the Intel I350 -ethernet driver. You create a branch called `i350` and open a Pull -Request to describe this branch in `src/doc/branches.md`. The other -maintainers are happy that you want to join in and gladly agree to -refer Pull Requests concerned the Intel I350 driver to you. You agree -that once you have good changes on your `i350` branch you will open a -Pull Request to the more general `drivers` branch as your "next hop" -upstream. - -People in the community start contributing improvements to the I350 -driver. You set yourself as the *Assignee* to these Pull Requests and -engage with the contributors to get the changes in good shape. You -merge the good changes onto your `i350` branch and periodically open a -Pull Request to the `drivers` branch to send this code upstream. The -`drivers` branch will in turn be merged to its next hop upstream and -step-by-step the changes will make their way towards release on the -`master` branch. - -The time scales involved are not written in stone but it is important -to find a rhythm that is comfortable for everybody involved. You might -aim to set yourself as the *Assignee* for relevant Pull Requests -within one day, to provide a review within a few days, and to send -Pull Requests to your next-hop upstream once or twice per week when -you have changes. - -#### Don't be shy! - -Becoming a Snabb Switch maintainer is a great service to the community -and you can learn all the skills that you need "on the job." If you are -tempted to give it a try then please do! - -The more subsystem maintainers we have the more capacity we have to -incorporate improvements into Snabb Switch. The Linux kernel has -more than one thousand registered subsystems. The sky is the limit! +XXX rewrite. From eccb61405a1c14c68c871f7a2b60b1ff9e74bdad Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 21 Mar 2016 06:04:03 +0000 Subject: [PATCH 05/53] doc/git-workflow.md: Created .src.md This file now contains a diagram so it needs to have separate .src.md and .md versions. This can be cleaned up when these changes eventually merge with the on-demand markdown (#829). --- src/doc/.images/Branches.png | Bin 0 -> 9218 bytes src/doc/git-workflow.md | 13 +---- src/doc/git-workflow.src.md | 107 +++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 src/doc/.images/Branches.png create mode 100644 src/doc/git-workflow.src.md diff --git a/src/doc/.images/Branches.png b/src/doc/.images/Branches.png new file mode 100644 index 0000000000000000000000000000000000000000..7793578f6932b5113ec39ecebeef57d30daacc0c GIT binary patch literal 9218 zcmaia2UJsA*DWf74NwsgA)ulnA|M??iHM36X-X$VrAjXeNLTQJw4flp1rShrF9}7Z z2}lUNgMdIF^w8_u!F#`Z-+TA}{xKLCC+F<5v-jGo%sChSYAW({CoZ0#qN1Wxcz92P zit6wNxNbjo7<}dtIa{cxI2#r2$v%16KR4v2{KRap!Jg?CeZZ6QnC_xronnLj3>=?% z{-IQhN4Eph=F+bBJ7vOiV?}IQY~ZPu!OR~mG?9_r7ES_>Wbb9%^0(sUMTS50*0!?L z4eh7Nc*zsFxm(q|9!Zvztchc}ap~vwDsA*soRFDm!qli#qL8C$!lC^mR8$02It*ov zvb0oGm{uAHD;3q9>N|u(R8-UuaNC>y&u6bNVDQ2HOT_^-j6c)7!+Lo%OuC3 zl$(D~b@0kxgNYA1(jKwie)`3SL8jN{3}sp15%?pJd{lIGbv->j?d|PbVPONMwjFGg z$5;VkFB1jxLIJ^u*-=fj@=4PRVioCqr+}zwZ zZhRjYaNF5f!a2=3xEmbT%+JqHNJyY%VE-sV+3<=Qq1%Hty< zOoug=gZ=#EpFH`H6JCvQU;4x5U3|PWCO9|O*3K?(*s$Cob#s1iZ?8a08zOG^YVu&! za0(&AMNkdshrOJhD zYH3MoPgP14Hz#14Bl%+&WIVQ3PBQa#c6Pe&uJ#Di0^_}6KLPE@frlAfsoU7tVB*m- zHZwz1ylk6C%gD%_eT8i7{QTHN;o-x#gM55^-E2apKcZe(kByBX8>>9FgK!LN;&~+{ z5n7Z*E_x-Wq|hQGjL)q5pnJ>gQz9cHT^K@N{?PaJ^JB%uNIK`(I8Nc?<^1R++~{LP ztp(?0p4T3`cn)~Nm;Eqz!XIvqo2$zU)%!9q7~*Ven5xWYh18rJd~bV42f03&6+6MG zD)p=)#c{0m_>j1eP)B^j(St3}LFwq|^h<5JbU2;n2S4|vVzK?E*zc3YJ%*Xt*&=+> zCAbG>ouAZQ)m0upehd!Y!Xjff0&@ho%ujY&h`)FY{P5w!D;$u+f}g$1%*bAe&RLGZkxffrFJF$AqLrWELJb~qi(4U9Ci{4v9J%s zt@NIZq^&Hi%o)Pxd2FxAv%B2$LSB-VuFU12-Fj}wMWnI|3^H^*UF|Rtuo{t?n#v%( zqQ>vSk-APD?>RWu?K~Zwq%PPky?{c;kCIzqVmYcD(ym2IDT>q(Z~9j<60;&ABH4zO zdS;@Rh^LWHoK+Y3Ds8$lG%J&OB%FLJ-PUJ9jT+FaA%?PqLr-G`O#;}DsU%3TFfxvl zNU@=z8v6SBnwsJ7-u;}KVrFL6*3+}F;`jU37{*0%?{VlCvmdLCi(`Xps;Ob*8-s#Q zahDVoDaQya1iXL$e$F%IDVvYX-p2UG#!C*4MEmi2CMKrnQhQNY$*R$`>OgcbM6!G7 z1_vvvo(`yu>5Z?6E%5&kehzP1T z?^*j;yH0NP9+ofiX87(yh(&%yg~56SFd-hr8|&7QR63v^S6p`A8Rf*>kkpp!IXGQo zYSx-CS7kWm@XdvUzm#pHec85p*Qu0UVDR&u$e|xCyNt+J+tNg88k#17Z{lAXzuu*1 zdmwQ2>c{+i^-{0B%_-E0s}csTFJBUAINTRs=fLxxzE&}VKYkn*9sSudgk9Rg-XP+u z+#083XYj=7ogwFuX_{`bk-k(!GbSuN+(hd6K0VZHkDS<&tL}{{40~BPmEfk*(-f5i zi!d7YaiI=XZ-ElaMv4cc>4$V}?Yqa$4|TO*jNmZ7?YzG3Y<*mDPkEf(Fjfe8gNW=p zQ)6-zJ8L>Wn@EX;J)^ed`{Ci?jMtS3$#HizpFFvxTW~MX0c~Cp?l3&Pr>$Gmd5TICD1|!K8E_8Vo7$?Ydn30&LYU^k3ar! zaHeAmm7-^t(m@~)aQJSwzHPMpCI6G%vJEXc6MIPF+}e3FUYj8}+%N@fZ(fQgrAM0l zLuhD-l*JY2+{a>-hG^JZ=}>i>d^=ij815wGyrxPxbPT#LF^BWq#h#n6OKEsd$)V5g zGsKV058)gls$VXRt*opR6$$U&hf72YIq5ukV$4=Mv5|}`7UyGNm)Zap6zyO-!YXQ& zS1v$t!D30@#z5DfuW9329`u3z#Wj(Xw&!fUGL>X@7Ujsjrpmgj^{lo;8WR%}tnudZ z!~@2MlXipUu?O3jL=y3?fraNw=X3ac(5T%qa#m5#3tv(+X58wVOJsZ!Nc)-S3w%5T z6ER1#eQF3MWV*jEdo_>g%$cM*5Gz9(_^y?YYJm=ECYCs~AUAiTeW^pY<*b@YbC6k? zY0JgP1`~-R9nT*<3LaMS<>1D~aSUwb_d{xaq3iEsDkGNGwpDFMIOWmAW#?fR`N#`& z8Ah?{jSE%VIW~(eVk5**X2X|Xbu3TmotJHRNaghRs+DlO^94BUDQNfVIKHsmaB9O} zQ+szGx+fmy}j+7|}yPZLS}!Z*5%wg{!;4WIV|vPk>|G03Tw#tAm%phH49gTsXXg zR$&%Y85?mYKY~B6upe87#G5on=vJpNa&XvIZ8UzTMl*BbI*OrC2o`SpqO=a~)RVZ3 zMt6otwl)z_PCb{LMl8v!+7)A(y zVoB=BEiEb8TaK(oIt8oTzkeSNkGDV6(Uu|)$!#mQ8~q0A8fvQ0thHMS0SuwUqRXWc zxAE9pMLu25ne7Zjf6&C)x-?82_n;x)bC3brPtwZP>k{fA7Dn*#;BFvaCBK zo9;G{eW<+Afi>NwBc811&RxCF5;plGTF8CF4hDmb=Uwr`*Vf*HFd3@}qh({oY(Yq4 z>vnqiI7pz<3NUcxRxAe$r8GBU4NH4Ne9$c`_gPM2I84Tehp)~SwqEpO8sFJi9Ie4FRk)B>c3+XFTAi;Dd*fO|=I}<1qxv*V z5J`~Kv**M%f&(vXSrXb3RagX`zv=w3GTjk;&wLLO58U~o*s){B2na*nYR}ye*q%9i zY-+!cj}L%mw^qxI8_&Xr+uIAA=K9~hJtpB7VIiDjHHReRql}>|PHurlf%-^oGiQ6Y zi@Y=GDYEiz@oQc4+YpEuBOW(<3eJu+5PjWz0@>FBE%NfFw}a6@D)#rbw~~%)zGim> z=!3R1A!(tI3O+y6l?fjpQ9vtzz?hLMpp;TTBKSf9t_OFi01%>J zT3K2OHm0E8|2hCbE&xLS00I}kKf#39hPAl?4wqkN`1$!+LckeW|M;u~VEAd|mR~Du z-~CSPt5>gp_vPy3dt>B&Kkx@N+=3wCG#izlmz8x32CFVBi`u#j)^H5!p{c0Y^lE=? zZNPS5ZyOR1{2U(s{{7{;h=>SGI8+ ziV?~QQWCdstJ&N#C{xoG5f!Bea4X^}7uQIIOJQ*_YPvmDgH70+0Kz#lBLg`#F)>kK z@90Q2OZ13{js0yYg1$HU#?708!>jKndU6px6LocUSi2ul0v-Leb#>FB03v|xJvtjQ zhz|P*(1Bx_%#RvdzD4nRy;3tHE z7?BwV4hCec8d}eBzR_M2}x4l3nldmmFZT4X*48kw># zA>h?d7t?Y;(8c3#aC2|U#cpkF@r{(vl2aoMv9`1e^j+2?8JRCS(i-;wO8cskRSDZS z0cEGHcy6F(%Q^Nib^M+(y5~s00pM!Q4L*MCuAeOZ2CA5lgSi7T+S==mgz4I9{?n{5s8Ti1A`v|jF*VWVD86hL^b>6UyV^w7Z8Y%4QpK{ zd-hC2LqkpNU1{n1#?rX+T>pf4P@_7(sHhhG7|55x-3iieTIXZd`(kNr$BW%}mKv6p zmO#oLuMg${F=9KOb@RlD6Dvnejg24Xr>2^;e^k&23JS`~%`N>jAfu;!Ufr}VfN5)I z#FfNL?r)XZR>mfG>LZazL&JpFq2eKWQ;$f)t}Ne~nVEUZ5%xX#+ofeFCV|6;sN?CW zDk0FnI$$uH(S5jBv!(R-_^Ln zO{E-NUW|CG$J;!GGo3tti=YW;PjqxNc8%DU(tWugq&vJ2Q~0cg#$(yGYBSBn`YG#p zszSr|SRlWFleQV;#};2@x-K2SFGKxR(mNqfnXP#tvabi}zKUFl}CMRDR4{y4SnKKc%vNnXZSf`_!)63M#b}%3T6-VH1BvGFA^5x5X zI7=Ml#{_)dr7&h-{WUE+@%7qVqZhTlTMu0K$r*s0aI#!iuG|e|VPSE-^EoT)!&tPC zS|vpyY#|wPzNgx7wu5Er4bABvtX)ft$Ygw}mgO7I50+z>K!pR&)j&VF%(q^l+cIZq;5wLn^#k(RwqB18PPQTW zPxIG5t2+jDNV`_ok**r+ga7IEl($397QeXr;Ld&9tt*j%TU&!tJjmG)pYZE?a zvcHr!1NbaOJ|u$m(4j+sjiZ6UF>u6bfmk-1gNFQwXZLK4?Ha5C`GKTAX9+i&6Aie} zbO@v|dqQJ#b4Ri#!$f9o)$;nfnD1*}Utdt4EV9l>I?oF)SO6rgck9Lt0$mW#g$KbG zQzLJE-$ec?*ECJ;G}n~cju;kOXnLK1@@3vVZ=umB;DwMF0had_&4C?>iu`)}UaY2wNK{%v=AokYvAtUdqSee-^` zq{S`|_qA)+T*IHkjzPOM+xq)sqM|+}wmodS5F}BPkDt>u7y?&fB{>I+fFx#Wm#81o#La1LXGR|>NYnv zzAIR0X>}L2%2=K{;;^?hlbKszTT4w#>*nIp_qtlmqQ+VVb>pFUC3$qC0SXspVJfCN7^=tDIq_Ox9U2hy1!;Ev=799L$6;| z*u73Hdwd_%*xivNIlq-7hcm_!I+;X9+$bQqh#z2L7;<$+Gje1^R;-KZ$g^CBi6*9g zW&oR>Ub>fPi@-5_i#kn_1gRmQ;vW=!3M2qv_%zZSvCe`ZOSvo?D=P;9y;MY4GC3=S@C*nwK*Raxe}z>^ixbp6IE^v6bJz-u(T$xrIf6KsFT2cjU4; zA`ruP;lho^nhn8py5f2n*bp`%zilir*D%bjEW zrMno;^SpXb0J?qljheTg^Q&cR?tCSvUgNH9zO>SP6^Cj9cMMOq64?Wh< zK>B?9c9*py!v}NZ_G#}m)Onb21+{30&QUzha%XU&AXbB zKe3WOR4&!CVm)+qqxp2J+{^i%>7jsKXzs4K)Fb)ef^OwB@qoziUWC;d>gsaW37O%e zyaI+MRu%whZz(6i@lyNqxP=)SK$nC0Tt@$3Mjk&{!)fFzCvSADBfczdZ|DjUi3GnV z)GAq(QfSpD2>(_zb8re6vTVb_TCjQ3Ae?!23Axji+^Hy+jN&G?x8DS65g0#HxUlsl;E|X^Bu;7wPz$ zv-Nqoy1L4f=nj@W$SU}R!Q2r92#HroNr}Uim^8e@t}iCiF_q|*qj~Q1>D#U=L?GHD zKMjLWc{R+AY-(y+CX+!aU1-I0baocGu3D(7stzrCo`kxs&v!9<&i3Y`X=*{+gN!%@ zUU55(l-c-s1oBBGQHHk;aAh4(j151GjqI(7@g_h~P4?6x0S>wUlwVmSZLO{0K!FDE ztGXJjcME8_DBA);Z(3g7;&gf<{fQIHYipjf`3-`Cf>%{_LPqfHD(ev+K74dzg?aAe zz;Q?yS=rYzMYD(iz*|6S34LyCZ0zB&3-o<~pVbsicfKt%Gt zW%<_ulgtGS7UajtU%!YlAgE?Dh;yYQPFlA0jPI0hgqhxrIKMv@PI-!`6 zcM@73OKsXJy&hM$VZCJ6*Vl(7^z?}R#Y}tnbn@F>Gg6j|ho|{v zEjV>(XB7pco|_0)6b)DG=Dl_k#YcSD@z9#I3=qxYU#5R!Xuq6^*i~#Oe0_vgajB?tWJvFR(gAGlDto{&|EB6S1(^rN`@#$rK10grZ$a-4V@ z8}cG6E6`7`)GaP9E-x=*u}~vn-dnejqMnj&;(Y^zB)Qb6xHxz6pjP|6c~e_kJ6P^0 zUyb1OiM~%@k$RbDd{KYEPZC;T-J2&HKtY;CWx!d7K<{US4XF|*o>V6`{8CQrozzFA z2!6!8)sEAe9U$U*lpa)(C_n{R1 zDb(aI&=WJdcVc~{O}E4@YzEIYnepCTe?$AG3m&p9G8}s2AYMS;ppp2 z!`*Zj(67Fydpn!1y2!hC?qb)3afunC{-np7&K**PHC*1_F-F9bM)vKM(X<>No!X;CD9N-}hGjNnvkQ50y0iu0+gv5kbl_5dkP>P{I1~DA5+6Un?1PS-)7Q{A%gj7AJ}zX@ z`6(+a3kX=lTLxL|9K$IwaCh!4U&{? zm>d>?)>BVXkX!L1$246tU@#}v3qo;;lFs<&qT4`gr=g`Ksd7L-+N-Ln!eY0wONW!m zAjGKFjR^0bCUq17Dxfuf*TyDCqQRVcx*Ib-KHf1nn56_LWX8hPXtCR^l>Jr;NbZ8h z$n25}rSJ;A;b!4h^mE12==X_xVOI(Rbk zdT(Wl(H<1LYPD98La{AINS3lc*U`mDgeP}-ZItn_E`X}_4&!IzdJNsNOG-p3s*6(( zG0F!G-a_{f4a)4E0Xirw1{~uKA^!vvq~yQ)2OAeZ6k^8lK@)nQl> Date: Mon, 21 Mar 2016 09:54:31 +0000 Subject: [PATCH 06/53] git-workflow.md: Rewrote section on becoming a maintainer --- src/doc/git-workflow.md | 82 ++++++++++++++++++++++++++++--------- src/doc/git-workflow.src.md | 82 ++++++++++++++++++++++++++++--------- 2 files changed, 126 insertions(+), 38 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index 293942d2fb..f298d2eddb 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -4,7 +4,7 @@ How do you engage with the Snabb Switch developer community? The answer depends - Use the software, ask questions, report bugs. - Contribute fixes and improvements. -- Maintain Snabb Switch by reviewing and merging pull requests. +- Maintain a part of Snabb Switch by reviewing and merging pull requests. - Create a new application to develop together with the community. ### Using the software @@ -63,34 +63,78 @@ Here are some tips for making your contribution smoothly: Snabb Switch maintainers are the people who review and merge the Pull Requests on Github. Each maintainer takes care of one or more specific -aspects of Snabb Switch. These aspects are called *subsystems*. +part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized -as a network: +as a tree: ![Branches](.images/Branches.png) -Pull Requests are first merged onto the subsystem branch that most -specifically matches their subject matter. For example, a change to -the PDF formatting of the manual would first be reviewed and accepted -by the maintainer of the `pdf-manual` branch. Later, that whole -subsystem branch is merged onto its "next-hop upstream" branch. For -example, the maintainer of the `documentation` branch would merge the -entire `pdf-manual` branch at regular intervals. +Pull Requests are merged onto the most specifically relevant branch to +begin with. Later, whole child branches are merged "upstream" onto +their parent branches. This way once a change has been merged onto any +branch in the tree it will naturally flow upstream to the `master` +branch for release. -#### Registering a subsystem branch - -XXX rewrite. +For example, a change to the LaTeX template for the PDF edition of the +manual would first be reviewed and merged by the maintainer of the +`pdf-manual` branch. The change would then be included in successive +merges upstream to branches `documentation`, `max-next`, `next`, and +finally `master`. Each of these steps provides the opportunity for a +maintainer to improve some aspect of the change before release. -#### Being "the upstream" for Pull Requests +#### Registering a subsystem branch -XXX rewrite. +So you are interested in becoming a Snabb Switch maintainer. Great! +The other maintainers are looking forward to working with you and will +be more than happy to help you learn the ropes. You can learn +everything you need to know on the job: no special qualifications are +required. + +The first step to becoming a maintainer is to decide which kind of +Pull Requests you want to be responsible for. For example you could be +responsible for changes to the Makefiles, or the Intel 10G device +driver, or the `packetblaster` load generator program, or any other +part of the software that is easy to identify when assigning Pull +Requests. You do not have to be an expert in this area already: it is +enough that you are committed to learning about it and being +responsive to contributors. + +Once you have worked out which part you want to maintain then you can +propose this to the community by sending a Pull Request that registers +your new branch in the file `src/doc/branches.md`. This Pull Request +will be the vehicle for talking with the other maintainers about where +to fit your new branch into the tree. + +The moment this Pull Request is merged onto the `next` branch then you +are officially a Snabb Switch maintainer. (Congratulations in advance!) + +#### Being the assigned maintainer for Pull Requests + +Now that you are a registered maintainer you will watch Github for new +Pull Requests and mark yourself as the *Assignee* for changes that +match your area of responsibility. You are the reviewer for these +changes and you decide when to accept them from the contributor. + +Here is the basic criteria for merging a Pull Request: + +- Does the change improve Snabb Switch? It does not have to be perfect + but it should clearly have a net-positive impact. +- Will the next-hop upstream maintainer agree? If not then getting + your branch merged upstream may require you to make some + improvements or even revert the change. + +Beyond this the most important thing is to communicate with the +contributor to make sure they understand exactly what they have to do +in order for their change to be merged. Contributors can easily be +overwhelmed by comments on Pull Requests, often from many different +people, so the assigned maintainer needs to explain very clearly what +the requirements are for merge. (Contributors will be frustrated if +they do not know what feedback they are required to act on, especially +when receiving conflicting ideas from different people, so you need to +decide for yourself what is required and clearly explain this.) #### Sending collected changes upstream to your next-hop XXX rewrite. -#### Putting it all together - -XXX rewrite. - diff --git a/src/doc/git-workflow.src.md b/src/doc/git-workflow.src.md index 16dae03e6e..f882b040b9 100644 --- a/src/doc/git-workflow.src.md +++ b/src/doc/git-workflow.src.md @@ -4,7 +4,7 @@ How do you engage with the Snabb Switch developer community? The answer depends - Use the software, ask questions, report bugs. - Contribute fixes and improvements. -- Maintain Snabb Switch by reviewing and merging pull requests. +- Maintain a part of Snabb Switch by reviewing and merging pull requests. - Create a new application to develop together with the community. ### Using the software @@ -63,10 +63,10 @@ Here are some tips for making your contribution smoothly: Snabb Switch maintainers are the people who review and merge the Pull Requests on Github. Each maintainer takes care of one or more specific -aspects of Snabb Switch. These aspects are called *subsystems*. +part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized -as a network: +as a tree: DIAGRAM: Branches +--lisper @@ -81,27 +81,71 @@ as a network: +--wingo next<--+--lwaftr +--multiproc -Pull Requests are first merged onto the subsystem branch that most -specifically matches their subject matter. For example, a change to -the PDF formatting of the manual would first be reviewed and accepted -by the maintainer of the `pdf-manual` branch. Later, that whole -subsystem branch is merged onto its "next-hop upstream" branch. For -example, the maintainer of the `documentation` branch would merge the -entire `pdf-manual` branch at regular intervals. +Pull Requests are merged onto the most specifically relevant branch to +begin with. Later, whole child branches are merged "upstream" onto +their parent branches. This way once a change has been merged onto any +branch in the tree it will naturally flow upstream to the `master` +branch for release. -#### Registering a subsystem branch - -XXX rewrite. +For example, a change to the LaTeX template for the PDF edition of the +manual would first be reviewed and merged by the maintainer of the +`pdf-manual` branch. The change would then be included in successive +merges upstream to branches `documentation`, `max-next`, `next`, and +finally `master`. Each of these steps provides the opportunity for a +maintainer to improve some aspect of the change before release. -#### Being "the upstream" for Pull Requests +#### Registering a subsystem branch -XXX rewrite. +So you are interested in becoming a Snabb Switch maintainer. Great! +The other maintainers are looking forward to working with you and will +be more than happy to help you learn the ropes. You can learn +everything you need to know on the job: no special qualifications are +required. + +The first step to becoming a maintainer is to decide which kind of +Pull Requests you want to be responsible for. For example you could be +responsible for changes to the Makefiles, or the Intel 10G device +driver, or the `packetblaster` load generator program, or any other +part of the software that is easy to identify when assigning Pull +Requests. You do not have to be an expert in this area already: it is +enough that you are committed to learning about it and being +responsive to contributors. + +Once you have worked out which part you want to maintain then you can +propose this to the community by sending a Pull Request that registers +your new branch in the file `src/doc/branches.md`. This Pull Request +will be the vehicle for talking with the other maintainers about where +to fit your new branch into the tree. + +The moment this Pull Request is merged onto the `next` branch then you +are officially a Snabb Switch maintainer. (Congratulations in advance!) + +#### Being the assigned maintainer for Pull Requests + +Now that you are a registered maintainer you will watch Github for new +Pull Requests and mark yourself as the *Assignee* for changes that +match your area of responsibility. You are the reviewer for these +changes and you decide when to accept them from the contributor. + +Here is the basic criteria for merging a Pull Request: + +- Does the change improve Snabb Switch? It does not have to be perfect + but it should clearly have a net-positive impact. +- Will the next-hop upstream maintainer agree? If not then getting + your branch merged upstream may require you to make some + improvements or even revert the change. + +Beyond this the most important thing is to communicate with the +contributor to make sure they understand exactly what they have to do +in order for their change to be merged. Contributors can easily be +overwhelmed by comments on Pull Requests, often from many different +people, so the assigned maintainer needs to explain very clearly what +the requirements are for merge. (Contributors will be frustrated if +they do not know what feedback they are required to act on, especially +when receiving conflicting ideas from different people, so you need to +decide for yourself what is required and clearly explain this.) #### Sending collected changes upstream to your next-hop XXX rewrite. -#### Putting it all together - -XXX rewrite. - From 4a37b98f4a1f347e94f28f995dadc2efb63d809d Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 22 Mar 2016 09:10:05 +0000 Subject: [PATCH 07/53] git-workflow.md: Wrote about upstreaming subsystem branches also stopped capitalizing "Pull Request" because it seemed awkward. --- src/doc/git-workflow.md | 73 ++++++++++++++++++++++++++----------- src/doc/git-workflow.src.md | 73 ++++++++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 44 deletions(-) diff --git a/src/doc/git-workflow.md b/src/doc/git-workflow.md index f298d2eddb..16e4cce577 100644 --- a/src/doc/git-workflow.md +++ b/src/doc/git-workflow.md @@ -45,24 +45,24 @@ $ make -j ### Contributing fixes and improvements -The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. +The recommended way to contribute improvements to Snabb Switch is with Github *pull requests*. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. 1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. 2. Push your proposed change to a branch on your repository. -3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) -4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. +3. Open a pull request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your pull request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the pull request. Then your work is done and the change is in the pipeline leading to release on the master branch. Here are some tips for making your contribution smoothly: - Use a dedicated "topic branch" for each feature or fix. -- Use the Pull Request text to explain why you are proposing the change. -- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. -- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. +- Use the pull request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the pull request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the pull request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. ### Becoming a maintainer -Snabb Switch maintainers are the people who review and merge the Pull -Requests on Github. Each maintainer takes care of one or more specific +Snabb Switch maintainers are the people who review and merge the pull +requests on Github. Each maintainer takes care of one or more specific part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized @@ -70,7 +70,7 @@ as a tree: ![Branches](.images/Branches.png) -Pull Requests are merged onto the most specifically relevant branch to +Pull requests are merged onto the most specifically relevant branch to begin with. Later, whole child branches are merged "upstream" onto their parent branches. This way once a change has been merged onto any branch in the tree it will naturally flow upstream to the `master` @@ -92,31 +92,31 @@ everything you need to know on the job: no special qualifications are required. The first step to becoming a maintainer is to decide which kind of -Pull Requests you want to be responsible for. For example you could be +pull requests you want to be responsible for. For example you could be responsible for changes to the Makefiles, or the Intel 10G device driver, or the `packetblaster` load generator program, or any other -part of the software that is easy to identify when assigning Pull -Requests. You do not have to be an expert in this area already: it is +part of the software that is easy to identify when assigning pull +requests. You do not have to be an expert in this area already: it is enough that you are committed to learning about it and being responsive to contributors. Once you have worked out which part you want to maintain then you can -propose this to the community by sending a Pull Request that registers -your new branch in the file `src/doc/branches.md`. This Pull Request +propose this to the community by sending a pull request that registers +your new branch in the file `src/doc/branches.md`. This pull request will be the vehicle for talking with the other maintainers about where to fit your new branch into the tree. -The moment this Pull Request is merged onto the `next` branch then you +The moment this pull request is merged onto the `next` branch then you are officially a Snabb Switch maintainer. (Congratulations in advance!) -#### Being the assigned maintainer for Pull Requests +#### Being the assigned maintainer for pull requests Now that you are a registered maintainer you will watch Github for new -Pull Requests and mark yourself as the *Assignee* for changes that +pull requests and mark yourself as the *Assignee* for changes that match your area of responsibility. You are the reviewer for these changes and you decide when to accept them from the contributor. -Here is the basic criteria for merging a Pull Request: +Here is the basic criteria for merging a pull request: - Does the change improve Snabb Switch? It does not have to be perfect but it should clearly have a net-positive impact. @@ -127,14 +127,43 @@ Here is the basic criteria for merging a Pull Request: Beyond this the most important thing is to communicate with the contributor to make sure they understand exactly what they have to do in order for their change to be merged. Contributors can easily be -overwhelmed by comments on Pull Requests, often from many different +overwhelmed by comments on pull requests, often from many different people, so the assigned maintainer needs to explain very clearly what the requirements are for merge. (Contributors will be frustrated if they do not know what feedback they are required to act on, especially when receiving conflicting ideas from different people, so you need to decide for yourself what is required and clearly explain this.) -#### Sending collected changes upstream to your next-hop - -XXX rewrite. +#### Sending your accepted changes upstream + +Once you have merged one or more changes onto your branch the next +step is to open a pull request asking the next upstream maintainer to +merge your whole branch. This way all of the changes you merged will +make their way upstream step-by-step and ultimately be released by +being merged onto the `master` branch. + +This is where it comes in handy that the branches are organized in the +tree structure shown above. When your subsystem branch contains +changes that are ready to merge further upstream then you would open a +pull request to the parent branch in the tree. For example, if you are +the maintainer of the `pdf-manual` branch and you have changes ready +for integration then you would open a pull request to the +`documentation` parent branch. + +How do you decide when to open this pull request? This is a latency +verses throughput trade-off that you need to agree on with the +upstream maintainer. On the one hand it is important for changes to +move upstream quickly when they are beneficial to users or likely to +conflict with other work, on the other hand you may overwhelm the +upstream maintainer if you open pull requests too often. One +suggestion to use as a starting point would be to open a pull request +once or twice per week whenever your branch contains new changes. + +The upstream branch maintainer will be responsible for considering the +broader impact of the changes on your branch. They may need to resolve +conflicts with other changes that they have merged, or want to ask for +input from people who will be affected by the changes, or they may +want to refactor the changes together with some other related code. +They will tell you if they need your help with this and if they have +ideas for how you can make their merging work easier in the future. diff --git a/src/doc/git-workflow.src.md b/src/doc/git-workflow.src.md index f882b040b9..c888819cc3 100644 --- a/src/doc/git-workflow.src.md +++ b/src/doc/git-workflow.src.md @@ -45,24 +45,24 @@ $ make -j ### Contributing fixes and improvements -The recommended way to contribute improvements to Snabb Switch is with Github Pull Requests. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. +The recommended way to contribute improvements to Snabb Switch is with Github *pull requests*. You can do this by following the Github [Using pull requests](https://help.github.com/articles/using-pull-requests/) instructions. Here is a brief summary. 1. "Fork" your own copy of the [`snabbco/snabbswitch`](https://github.com/snabbco/snabbswitch) repository. 2. Push your proposed change to a branch on your repository. -3. Open a Pull Request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) -4. Expect that within a few days a qualified maintainer will become the *Assignee* of your Pull Request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the Pull Request. Then your work is done and the change is in the pipeline leading to release on the master branch. +3. Open a pull request from your branch. Choose the `master` branch of the `snabbco/snabbswitch` repository as the target branch (this should be the default choice.) +4. Expect that within a few days a qualified maintainer will become the *Assignee* of your pull request and work with you to get the change merged. The maintainer may ask you some questions and even require some changes. Once they are satisfied they will merge the change onto their own branch and apply the label `merged` on the pull request. Then your work is done and the change is in the pipeline leading to release on the master branch. Here are some tips for making your contribution smoothly: - Use a dedicated "topic branch" for each feature or fix. -- Use the Pull Request text to explain why you are proposing the change. -- If the change is a work in progress then prefix the Pull Request title with `[wip]`. This signals that you intened to push more commits before the change is complete. -- If the change is a rough draft that you want early feedback on then prefix the Pull Request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. +- Use the pull request text to explain why you are proposing the change. +- If the change is a work in progress then prefix the pull request title with `[wip]`. This signals that you intened to push more commits before the change is complete. +- If the change is a rough draft that you want early feedback on then prefix the pull request name with `[sketch]`. This signals that you may throw the branch away and start over on a new one. ### Becoming a maintainer -Snabb Switch maintainers are the people who review and merge the Pull -Requests on Github. Each maintainer takes care of one or more specific +Snabb Switch maintainers are the people who review and merge the pull +requests on Github. Each maintainer takes care of one or more specific part of Snabb Switch. These parts are called *subsystems*. Each subsystem has a dedicated branch and these branches are organized @@ -81,7 +81,7 @@ as a tree: +--wingo next<--+--lwaftr +--multiproc -Pull Requests are merged onto the most specifically relevant branch to +Pull requests are merged onto the most specifically relevant branch to begin with. Later, whole child branches are merged "upstream" onto their parent branches. This way once a change has been merged onto any branch in the tree it will naturally flow upstream to the `master` @@ -103,31 +103,31 @@ everything you need to know on the job: no special qualifications are required. The first step to becoming a maintainer is to decide which kind of -Pull Requests you want to be responsible for. For example you could be +pull requests you want to be responsible for. For example you could be responsible for changes to the Makefiles, or the Intel 10G device driver, or the `packetblaster` load generator program, or any other -part of the software that is easy to identify when assigning Pull -Requests. You do not have to be an expert in this area already: it is +part of the software that is easy to identify when assigning pull +requests. You do not have to be an expert in this area already: it is enough that you are committed to learning about it and being responsive to contributors. Once you have worked out which part you want to maintain then you can -propose this to the community by sending a Pull Request that registers -your new branch in the file `src/doc/branches.md`. This Pull Request +propose this to the community by sending a pull request that registers +your new branch in the file `src/doc/branches.md`. This pull request will be the vehicle for talking with the other maintainers about where to fit your new branch into the tree. -The moment this Pull Request is merged onto the `next` branch then you +The moment this pull request is merged onto the `next` branch then you are officially a Snabb Switch maintainer. (Congratulations in advance!) -#### Being the assigned maintainer for Pull Requests +#### Being the assigned maintainer for pull requests Now that you are a registered maintainer you will watch Github for new -Pull Requests and mark yourself as the *Assignee* for changes that +pull requests and mark yourself as the *Assignee* for changes that match your area of responsibility. You are the reviewer for these changes and you decide when to accept them from the contributor. -Here is the basic criteria for merging a Pull Request: +Here is the basic criteria for merging a pull request: - Does the change improve Snabb Switch? It does not have to be perfect but it should clearly have a net-positive impact. @@ -138,14 +138,43 @@ Here is the basic criteria for merging a Pull Request: Beyond this the most important thing is to communicate with the contributor to make sure they understand exactly what they have to do in order for their change to be merged. Contributors can easily be -overwhelmed by comments on Pull Requests, often from many different +overwhelmed by comments on pull requests, often from many different people, so the assigned maintainer needs to explain very clearly what the requirements are for merge. (Contributors will be frustrated if they do not know what feedback they are required to act on, especially when receiving conflicting ideas from different people, so you need to decide for yourself what is required and clearly explain this.) -#### Sending collected changes upstream to your next-hop - -XXX rewrite. +#### Sending your accepted changes upstream + +Once you have merged one or more changes onto your branch the next +step is to open a pull request asking the next upstream maintainer to +merge your whole branch. This way all of the changes you merged will +make their way upstream step-by-step and ultimately be released by +being merged onto the `master` branch. + +This is where it comes in handy that the branches are organized in the +tree structure shown above. When your subsystem branch contains +changes that are ready to merge further upstream then you would open a +pull request to the parent branch in the tree. For example, if you are +the maintainer of the `pdf-manual` branch and you have changes ready +for integration then you would open a pull request to the +`documentation` parent branch. + +How do you decide when to open this pull request? This is a latency +verses throughput trade-off that you need to agree on with the +upstream maintainer. On the one hand it is important for changes to +move upstream quickly when they are beneficial to users or likely to +conflict with other work, on the other hand you may overwhelm the +upstream maintainer if you open pull requests too often. One +suggestion to use as a starting point would be to open a pull request +once or twice per week whenever your branch contains new changes. + +The upstream branch maintainer will be responsible for considering the +broader impact of the changes on your branch. They may need to resolve +conflicts with other changes that they have merged, or want to ask for +input from people who will be affected by the changes, or they may +want to refactor the changes together with some other related code. +They will tell you if they need your help with this and if they have +ideas for how you can make their merging work easier in the future. From 94ff2347175cc2e56153f0af3d16f9af86c2a367 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 12 Apr 2016 19:41:12 +0200 Subject: [PATCH 08/53] Amendments to #766: - Use "apps/" instead of "app/" for uniformity - Set shm path to "apps/$name" when calling `app:stop' too - Unlink "apps/$name" after `app:stop' using `shm.unlink' - Add a test case to core.app selftest --- src/core/app.lua | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 099fc3125a..5ff7d476eb 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -156,7 +156,13 @@ function apply_config_actions (actions, conf) -- Table of functions that execute config actions local ops = {} function ops.stop (name) - if app_table[name].stop then app_table[name]:stop() end + if app_table[name].stop then + local shmorig = shm.path + shm.path = app_table[name].shmpath + app_table[name]:stop() + shm.path = shmorig + shm.unlink(app_table[name].shmpath) + end end function ops.keep (name) new_app_table[name] = app_table[name] @@ -166,11 +172,10 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg - local shmpath, shmorig = "app/"..name, shm.path + local shmpath, shmorig = "apps/"..name, shm.path shm.path = shmpath local app = class:new(arg) shm.path = shmorig - local shmpath = "app/"..name if type(app) ~= 'table' then error(("bad return value from app '%s' start() method: %s"):format( name, tostring(app))) @@ -492,6 +497,34 @@ function selftest () assert(app_table.app3 == orig_app3) -- should be the same main({duration = 4, report = {showapps = true}}) assert(app_table.app3 ~= orig_app3) -- should be restarted + -- Test shm.path management + print("shm.path management") + local S = require("syscall") + local App4 = {zone="test"} + function App4:new () + local c = counter.open('test') + counter.set(c, 42) + counter.commit() + return setmetatable({test_counter = c}, + {__index = App4}) + end + function App4:pull () + assert(counter.read(self.test_counter) == 42, "Invalid counter value") + counter.add(self.test_counter) + end + function App4:stop () + assert(counter.read(self.test_counter) == 43, "Invalid counter value") + counter.delete('test') + end + local c_counter = config.new() + config.app(c_counter, "App4", App4) + configure(c_counter) + main({done = function () return app_table.App4.test_counter end}) + assert(S.stat(shm.root.."/"..shm.resolve("apps/App4/test")), + "Missing : apps/App4/test") + configure(config.new()) + assert(not S.stat(shm.root.."/"..shm.resolve("apps/App4")), + "Failed to unlink apps/App4") print("OK") end From fad0f43921d8f226f199f1cbaed02e5e6efc6d30 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 13 Apr 2016 16:44:14 +0200 Subject: [PATCH 09/53] core.counter: Qualify counter names using `shm.resolve'. --- src/core/counter.lua | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/core/counter.lua b/src/core/counter.lua index a479df4040..881cea5717 100644 --- a/src/core/counter.lua +++ b/src/core/counter.lua @@ -44,7 +44,8 @@ local private = {} local numbers = {} -- name -> number function open (name, readonly) - if numbers[name] then error("counter already opened: " .. name) end + local qname = shm.resolve(name) + if numbers[qname] then error("counter already opened: " .. qname) end local n = #public+1 if readonly then public[n] = shm.open(name, counter_t, readonly) @@ -53,13 +54,14 @@ function open (name, readonly) public[n] = shm.create(name, counter_t) private[n] = ffi.new(counter_t) end - numbers[name] = n + numbers[qname] = n return private[n] end function delete (name) - local number = numbers[name] - if not number then error("counter not found for deletion: " .. name) end + local qname = shm.resolve(name) + local number = numbers[qname] + if not number then error("counter not found for deletion: " .. qname) end -- Free shm object shm.unmap(public[number]) -- If we "own" the counter for writing then we unlink it too. @@ -67,7 +69,7 @@ function delete (name) shm.unlink(name) end -- Free local state - numbers[name] = false + numbers[qname] = false public[number] = false private[number] = false end From 7ed4ed0383bb24fd780fb6ecda3197f5c1106165 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 12 Apr 2016 20:21:33 +0200 Subject: [PATCH 10/53] snabb top: add `--app' option to print app counters. --- src/program/top/README | 8 ++++-- src/program/top/top.lua | 61 +++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 365c357326..66471f390c 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -3,10 +3,12 @@ Usage: -h, --help Print usage information. + -a, --app + Print counters of app by and exit. -Display realtime performance statistics for a running Snabb -instance with . If is not supplied and there is only one Snabb -Switch instance, top will connect to that instance. +Display realtime performance statistics for a running Snabb instance with +. If is not supplied and there is only one Snabb instance, top will +connect to that instance. The following global metrics will be displayed: diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 6617027b1b..f66af6df0d 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,20 +12,57 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h" + help = "h", app = "a" } function clearterm () io.write('\027[2J') end function run (args) local opt = {} + local app_name = nil function opt.h (arg) print(usage) main.exit(1) end - args = lib.dogetopt(args, opt, "h", long_opts) + function opt.a (arg) app_name = arg end + args = lib.dogetopt(args, opt, "ha:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = args[1] - local instance_tree = "//"..(select_snabb_instance(target_pid)) + if app_name then app(select_snabb_instance(target_pid), app_name) + else top(select_snabb_instance(target_pid)) end +end + +function select_snabb_instance (pid) + local instances = shm.children("//") + if pid then + -- Try to use given pid + for _, instance in ipairs(instances) do + if instance == pid then return pid end + end + print("No such Snabb instance: "..pid) + elseif #instances == 2 then + -- Two means one is us, so we pick the other. + local own_pid = tostring(S.getpid()) + if instances[1] == own_pid then return instances[2] + else return instances[1] end + elseif #instances == 1 then print("No Snabb instance found.") + else print("Multple Snabb instances found. Select one.") end + os.exit(1) +end + +function app (instance_pid, app_name) + local app_tree = "//"..instance_pid.."/apps/"..app_name + local cnames = shm.children(app_tree) + table.sort(cnames, function (a, b) return a < b end) + for _, cname in ipairs(cnames) do + local cpath = app_tree.."/"..cname + local value = counter.read(counter.open(cpath, 'readonly')) + print_row({30, 30}, {cname, lib.comma_value(value)}) + counter.delete(cpath) + end +end + +function top (instance_pid) + local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) local configs = 0 local last_stats = nil @@ -48,24 +85,6 @@ function run (args) end end -function select_snabb_instance (pid) - local instances = shm.children("//") - if pid then - -- Try to use given pid - for _, instance in ipairs(instances) do - if instance == pid then return pid end - end - print("No such Snabb instance: "..pid) - elseif #instances == 2 then - -- Two means one is us, so we pick the other. - local own_pid = tostring(S.getpid()) - if instances[1] == own_pid then return instances[2] - else return instances[1] end - elseif #instances == 1 then print("No Snabb instance found.") - else print("Multple Snabb instances found. Select one.") end - os.exit(1) -end - function open_counters (tree) local counters = {} for _, name in ipairs({"configs", "breaths", "frees", "freebytes"}) do From eb9005b5c5ec4aa637b8a327f556029f8ef4cf29 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 13 Apr 2016 17:24:44 +0200 Subject: [PATCH 11/53] snabb top: unlink own shm tree to avoid clutter. --- src/program/top/top.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index f66af6df0d..54308c63f3 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -29,6 +29,7 @@ function run (args) if app_name then app(select_snabb_instance(target_pid), app_name) else top(select_snabb_instance(target_pid)) end + ordered_exit(0) end function select_snabb_instance (pid) @@ -46,7 +47,12 @@ function select_snabb_instance (pid) else return instances[1] end elseif #instances == 1 then print("No Snabb instance found.") else print("Multple Snabb instances found. Select one.") end - os.exit(1) + ordered_exit(1) +end + +function ordered_exit (value) + shm.unlink("//"..S.getpid()) -- Unlink own shm tree to avoid clutter + os.exit(value) end function app (instance_pid, app_name) From 5fbe0d6fba0a1c0fd785102bb6da0a184f95dfd5 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 16:39:55 +0200 Subject: [PATCH 12/53] vhost_user: Add RFC 7223 app counters. --- src/apps/vhost/vhost_user.lua | 36 +++++++++++++++++++++++++++++++++++ src/lib/virtio/net_device.lua | 4 ++++ 2 files changed, 40 insertions(+) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 7890b495f5..96e82780eb 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -13,7 +13,9 @@ local lib = require("core.lib") local link = require("core.link") local main = require("core.main") local memory = require("core.memory") +local counter = require("core.counter") local pci = require("lib.hardware.pci") +local ethernet = require("lib.protocol.ethernet") local net_device= require("lib.virtio.net_device") local timer = require("core.timer") local ffi = require("ffi") @@ -52,6 +54,15 @@ function VhostUser:new (args) else self.qemu_connect = self.client_connect end + -- initialize counters + self.counters = {} + for _, name in ipairs({'type', 'discontinuity-time', + 'in-octets', 'in-unicast', 'in-multicast', 'in-discards', + 'out-octets', 'out-unicast', 'out-multicast'}) do + self.counters[name] = counter.open(name) + end + counter.set(self.counters['type'], 0x1001) -- Virtual interface + counter.set(self.counters['discontinuity-time'], C.get_unix_time()) return self end @@ -68,6 +79,9 @@ function VhostUser:stop() self:free_mem_table() if self.link_down_proc then self.link_down_proc() end + + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end function VhostUser:pull () @@ -86,6 +100,28 @@ function VhostUser:push () end end +function VhostUser:tx_callback (p) + counter.add(self.counters['out-octets'], packet.length(p)) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters['out-multicast']) + else + counter.add(self.counters['out-unicast']) + end +end + +function VhostUser:rx_callback (p) + counter.add(self.counters['in-octets'], packet.length(p)) + if ethernet:is_mcast(packet.data(p)) then + counter.add(self.counters['in-multicast']) + else + counter.add(self.counters['in-unicast']) + end +end + +function VhostUser:rxdrop_callback (p) + counter.add(self.counters['in-discards']) +end + -- Try to connect to QEMU. function VhostUser:client_connect () return C.vhost_user_connect(self.socket_path) diff --git a/src/lib/virtio/net_device.lua b/src/lib/virtio/net_device.lua index 1f9f321191..0cbdcd13c3 100644 --- a/src/lib/virtio/net_device.lua +++ b/src/lib/virtio/net_device.lua @@ -163,9 +163,11 @@ function VirtioNetDevice:rx_packet_end(header_id, total_size, rx_p) rx_p.length - self.rx_hdr_csum_start, self.rx_hdr_csum_offset) end + self.owner:rx_callback(rx_p) link.transmit(l, rx_p) else debug("droprx", "len", rx_p.length) + self.owner:rxdrop_callback(rx_p) packet.free(rx_p) end self.virtq[self.ring_id]:put_buffer(header_id, total_size) @@ -253,6 +255,7 @@ function VirtioNetDevice:tx_buffer_add(tx_p, addr, len) end function VirtioNetDevice:tx_packet_end(header_id, total_size, tx_p) + self.owner:tx_callback(tx_p) packet.free(tx_p) self.virtq[self.ring_id]:put_buffer(header_id, total_size) end @@ -313,6 +316,7 @@ end function VirtioNetDevice:tx_packet_end_mrg_rxbuf(header_id, total_size, tx_p) -- free the packet only when all its data is processed if self.tx.finished then + self.owner:tx_callback(tx_p) packet.free(tx_p) self.tx.p = nil self.tx.data_sent = nil From 8bb3215bf759f6c53af07c5c13afd2ea60003334 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 17:57:09 +0200 Subject: [PATCH 13/53] Intel_app: Add RFC 7223 app counters. --- src/apps/intel/intel_app.lua | 45 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 80cace3d9f..f104fad553 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,8 +6,10 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") +local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") +local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local freelist = require("core.freelist") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty @@ -36,6 +38,7 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) + local self = setmetatable({counters = {}}, Intel82599) if conf.vmdq then if devices[conf.pciaddr] == nil then @@ -45,12 +48,26 @@ function Intel82599:new (arg) local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - return setmetatable({dev=vf:open(conf)}, Intel82599) + self.dev = vf:open(conf) else - local dev = intel10g.new_sf(conf):open() - if not dev then return null end - return setmetatable({dev=dev, zone="intel"}, Intel82599) + self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") + self.zone = "intel" end + + self.counters['type'] = counter.open('type') + self.counters['discontinuity-time'] = counter.open('discontinuity-time') + self.counters['in-octets'] = counter.open('in-octets') + self.counters['in-discards'] = counter.open('in-discards') + self.counters['out-octets'] = counter.open('out-octets') + counter.set(self.counters['type'], 0x1000) -- Hardware interface + counter.set(self.counters['discontinuity-time'], C.get_unix_time()) + if conf.vmdq then + self.counters['phys-address'] = counter.open('phys-address') + counter.set(self.counters['phys-address'], + macaddress:new(conf.macaddr).bits) + end + + return self end function Intel82599:stop() @@ -71,6 +88,9 @@ function Intel82599:stop() if close_pf then close_pf:close() end + + -- delete counters + for name, _ in pairs(self.counters) do counter.delete(name) end end @@ -79,6 +99,11 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) + + if conf.vmdq then + counter.set(self.counters['phys-address'], + macaddress:new(conf.macaddr).bits) + end end -- Allocate receive buffers from the given freelist. @@ -96,6 +121,13 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() + if self.dev.rxstats then + counter.set(self.counters['in-octets'], + bit.lshift(self.dev.pf.qs.QBRC_H[self.dev.rxstats]()+0LL, 32) + + self.dev.pf.qs.QBRC_L[self.dev.rxstats]()) + counter.set(self.counters['in-discards'], + self.dev.pf.qs.QPRDC[self.dev.rxstats]()) + end end function Intel82599:add_receive_buffers () @@ -116,6 +148,11 @@ function Intel82599:push () end end self.dev:sync_transmit() + if self.dev.txstats then + counter.set(self.counters['out-octets'], + bit.lshift(self.dev.pf.qs.QBTC_H[self.dev.txstats]()+0LL, 32) + + self.dev.pf.qs.QBTC_L[self.dev.txstats]()) + end end -- Report on relevant status and statistics. From 7a5547859a92ba935ed93ef1d144f1a9d8a1ab39 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 18:13:51 +0200 Subject: [PATCH 14/53] snabb top: Add --link parameter to list link counters. --- src/program/top/README | 3 ++- src/program/top/top.lua | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 66471f390c..ae91f6e4c4 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -4,7 +4,8 @@ Usage: -h, --help Print usage information. -a, --app - Print counters of app by and exit. + -l, --link + Print counters of app or link by and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 54308c63f3..3b1f57a600 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,7 +12,7 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h", app = "a" + help = "h", app = "a", link = "l" } function clearterm () io.write('\027[2J') end @@ -20,15 +20,18 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local app_name = nil + local link_name = nil function opt.h (arg) print(usage) main.exit(1) end function opt.a (arg) app_name = arg end - args = lib.dogetopt(args, opt, "ha:", long_opts) + function opt.l (arg) link_name = arg end + args = lib.dogetopt(args, opt, "ha:l:", long_opts) if #args > 1 then print(usage) main.exit(1) end - local target_pid = args[1] + local target_pid = select_snabb_instance(args[1]) - if app_name then app(select_snabb_instance(target_pid), app_name) - else top(select_snabb_instance(target_pid)) end + if app_name then list_counters("//"..target_pid.."/apps/"..app_name) + elseif link_name then list_counters("//"..target_pid.."/counters/"..link_name) + else top(target_pid) end ordered_exit(0) end @@ -55,12 +58,11 @@ function ordered_exit (value) os.exit(value) end -function app (instance_pid, app_name) - local app_tree = "//"..instance_pid.."/apps/"..app_name - local cnames = shm.children(app_tree) +function list_counters (path) + local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do - local cpath = app_tree.."/"..cname + local cpath = path.."/"..cname local value = counter.read(counter.open(cpath, 'readonly')) print_row({30, 30}, {cname, lib.comma_value(value)}) counter.delete(cpath) From dde5da2ac77cda91ed806b68399d7680c92004e3 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Apr 2016 19:17:20 +0200 Subject: [PATCH 15/53] core.app: Put app counters under "counters/", update snabb top. --- src/core/app.lua | 10 +++++----- src/program/top/README | 5 ++--- src/program/top/top.lua | 18 ++++++++---------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 352f5999f3..fa251390bb 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -175,7 +175,7 @@ function apply_config_actions (actions, conf) function ops.start (name) local class = conf.apps[name].class local arg = conf.apps[name].arg - local shmpath, shmorig = "apps/"..name, shm.path + local shmpath, shmorig = "counters/"..name, shm.path shm.path = shmpath local app = class:new(arg) shm.path = shmorig @@ -530,11 +530,11 @@ function selftest () config.app(c_counter, "App4", App4) configure(c_counter) main({done = function () return app_table.App4.test_counter end}) - assert(S.stat(shm.root.."/"..shm.resolve("apps/App4/test")), - "Missing : apps/App4/test") + assert(S.stat(shm.root.."/"..shm.resolve("counters/App4/test")), + "Missing : counters/App4/test") configure(config.new()) - assert(not S.stat(shm.root.."/"..shm.resolve("apps/App4")), - "Failed to unlink apps/App4") + assert(not S.stat(shm.root.."/"..shm.resolve("counters/App4")), + "Failed to unlink counters/App4") print("OK") end diff --git a/src/program/top/README b/src/program/top/README index ae91f6e4c4..ce815d6f89 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -3,9 +3,8 @@ Usage: -h, --help Print usage information. - -a, --app - -l, --link - Print counters of app or link by and exit. + -c, --counters + Print counters of object by and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 3b1f57a600..d69a999695 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -12,26 +12,23 @@ local histogram = require("core.histogram") local usage = require("program.top.README_inc") local long_opts = { - help = "h", app = "a", link = "l" + help = "h", counters = "c" } function clearterm () io.write('\027[2J') end function run (args) local opt = {} - local app_name = nil - local link_name = nil + local object = nil function opt.h (arg) print(usage) main.exit(1) end - function opt.a (arg) app_name = arg end - function opt.l (arg) link_name = arg end - args = lib.dogetopt(args, opt, "ha:l:", long_opts) + function opt.c (arg) object = arg end + args = lib.dogetopt(args, opt, "hc:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) - if app_name then list_counters("//"..target_pid.."/apps/"..app_name) - elseif link_name then list_counters("//"..target_pid.."/counters/"..link_name) - else top(target_pid) end + if object then list_counters(target_pid, object) + else top(target_pid) end ordered_exit(0) end @@ -58,7 +55,8 @@ function ordered_exit (value) os.exit(value) end -function list_counters (path) +function list_counters (pid, object) + local path = "//"..pid.."/counters/"..object local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do From 924ff4e6e82880d296d6be9ddae4dda3aef4dff0 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 19:30:15 +0200 Subject: [PATCH 16/53] lib.json: Import JSON4Lua 1.0.0, include encode functionality. --- src/lib/json.lua | 658 ++++++++++++++++++++++++++++++----------------- 1 file changed, 417 insertions(+), 241 deletions(-) diff --git a/src/lib/json.lua b/src/lib/json.lua index 86cbf85e43..9ab4abd38e 100644 --- a/src/lib/json.lua +++ b/src/lib/json.lua @@ -1,241 +1,417 @@ --- JSON4Lua: JSON encoding / decoding support for the Lua language. --- json Module. --- Author: Craig Mason-Jones --- Homepage: http://json.luaforge.net/ --- Version: 0.9.40 --- This module is released under the MIT License (MIT). --- --- NOTE: This is only the decode functionality ripped out from JSON4Lua. --- See: https://github.com/craigmj/json4lua - -module(..., package.seeall) - -local math = require('math') -local string = require("string") -local table = require("table") - -local base = _G - --- Private functions -local decode_scanArray -local decode_scanComment -local decode_scanConstant -local decode_scanNumber -local decode_scanObject -local decode_scanString -local decode_scanWhitespace - ---- Decodes a JSON string and returns the decoded value as a Lua data structure / value. --- @param s The string to scan. --- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. --- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, --- and the position of the first character after --- the scanned JSON object. -function decode(s, startPos) - startPos = startPos and startPos or 1 - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') - local curChar = string.sub(s,startPos,startPos) - -- Object - if curChar=='{' then - return decode_scanObject(s,startPos) - end - -- Array - if curChar=='[' then - return decode_scanArray(s,startPos) - end - -- Number - if string.find("+-0123456789.e", curChar, 1, true) then - return decode_scanNumber(s,startPos) - end - -- String - if curChar==[["]] or curChar==[[']] then - return decode_scanString(s,startPos) - end - if string.sub(s,startPos,startPos+1)=='/*' then - return decode(s, decode_scanComment(s,startPos)) - end - -- Otherwise, it must be a constant - return decode_scanConstant(s,startPos) -end - ---- The null function allows one to specify a null value in an associative array (which is otherwise --- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } -function null() - return null -- so json.null() will also return null ;-) -end ------------------------------------------------------------------------------ --- Internal, PRIVATE functions. --- Following a Python-like convention, I have prefixed all these 'PRIVATE' --- functions with an underscore. ------------------------------------------------------------------------------ - ---- Scans an array from JSON into a Lua object --- startPos begins at the start of the array. --- Returns the array and the next starting position --- @param s The string being scanned. --- @param startPos The starting position for the scan. --- @return table, int The scanned array as a table, and the position of the next character to scan. -function decode_scanArray(s,startPos) - local array = {} -- The return value - local stringLen = string.len(s) - base.assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) - startPos = startPos + 1 - -- Infinite loop for array elements - repeat - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') - local curChar = string.sub(s,startPos,startPos) - if (curChar==']') then - return array, startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - base.assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') - object, startPos = decode(s,startPos) - table.insert(array,object) - until false -end - ---- Scans a comment and discards the comment. --- Returns the position of the next character following the comment. --- @param string s The JSON string to scan. --- @param int startPos The starting position of the comment -function decode_scanComment(s, startPos) - base.assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) - local endPos = string.find(s,'*/',startPos+2) - base.assert(endPos~=nil, "Unterminated comment in string at " .. startPos) - return endPos+2 -end - ---- Scans for given constants: true, false or null --- Returns the appropriate Lua type, and the position of the next character to read. --- @param s The string being scanned. --- @param startPos The position in the string at which to start scanning. --- @return object, int The object (true, false or nil) and the position at which the next character should be --- scanned. -function decode_scanConstant(s, startPos) - local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } - local constNames = {"true","false","null"} - - for i,k in base.pairs(constNames) do - --print ("[" .. string.sub(s,startPos, startPos + string.len(k) -1) .."]", k) - if string.sub(s,startPos, startPos + string.len(k) -1 )==k then - return consts[k], startPos + string.len(k) - end - end - base.assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) -end - ---- Scans a number from the JSON encoded string. --- (in fact, also is able to scan numeric +- eqns, which is not --- in the JSON spec.) --- Returns the number, and the position of the next character --- after the number. --- @param s The string being scanned. --- @param startPos The position at which to start scanning. --- @return number, int The extracted number and the position of the next character to scan. -function decode_scanNumber(s,startPos) - local endPos = startPos+1 - local stringLen = string.len(s) - local acceptableChars = "+-0123456789.e" - while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) - and endPos<=stringLen - ) do - endPos = endPos + 1 - end - local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) - local stringEval = base.loadstring(stringValue) - base.assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON object into a Lua object. --- startPos begins at the start of the object. --- Returns the object and the next starting position. --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return table, int The scanned object as a table and the position of the next character to scan. -function decode_scanObject(s,startPos) - local object = {} - local stringLen = string.len(s) - local key, value - base.assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) - startPos = startPos + 1 - repeat - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') - local curChar = string.sub(s,startPos,startPos) - if (curChar=='}') then - return object,startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') - -- Scan the key - key, startPos = decode(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - startPos = decode_scanWhitespace(s,startPos) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - base.assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) - startPos = decode_scanWhitespace(s,startPos+1) - base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - value, startPos = decode(s,startPos) - object[key]=value - until false -- infinite loop while key-value pairs are found -end - ---- Scans a JSON string from the opening inverted comma or single quote to the --- end of the string. --- Returns the string extracted as a Lua string, --- and the position of the next non-string character --- (after the closing inverted comma or single quote). --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return string, int The extracted string as a Lua string, and the next character to parse. -function decode_scanString(s,startPos) - base.assert(startPos, 'decode_scanString(..) called without start position') - local startChar = string.sub(s,startPos,startPos) - base.assert(startChar==[[']] or startChar==[["]],'decode_scanString called for a non-string') - local escaped = false - local endPos = startPos + 1 - local bEnded = false - local stringLen = string.len(s) - repeat - local curChar = string.sub(s,endPos,endPos) - -- Character escaping is only used to escape the string delimiters - if not escaped then - if curChar==[[\]] then - escaped = true - else - bEnded = curChar==startChar - end - else - -- If we're escaped, we accept the current character come what may - escaped = false - end - endPos = endPos + 1 - base.assert(endPos <= stringLen+1, "String decoding failed: unterminated string at position " .. endPos) - until bEnded - local stringValue = 'return ' .. string.sub(s, startPos, endPos-1) - local stringEval = base.loadstring(stringValue) - base.assert(stringEval, 'Failed to load string [ ' .. stringValue .. '] in JSON4Lua.decode_scanString at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON string skipping all whitespace from the current start position. --- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. --- @param s The string being scanned --- @param startPos The starting position where we should begin removing whitespace. --- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string --- was reached. -function decode_scanWhitespace(s,startPos) - local whitespace=" \n\r\t" - local stringLen = string.len(s) - while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do - startPos = startPos + 1 - end - return startPos -end +----------------------------------------------------------------------------- +-- JSON4Lua: JSON encoding / decoding support for the Lua language. +-- json Module. +-- Author: Craig Mason-Jones +-- Homepage: http://github.com/craigmj/json4lua/ +-- Version: 1.0.0 +-- This module is released under the MIT License (MIT). +-- Please see LICENCE.txt for details. +-- +-- USAGE: +-- This module exposes two functions: +-- json.encode(o) +-- Returns the table / string / boolean / number / nil / json.null value as a JSON-encoded string. +-- json.decode(json_string) +-- Returns a Lua object populated with the data encoded in the JSON string json_string. +-- +-- REQUIREMENTS: +-- compat-5.1 if using Lua 5.0 +-- +-- CHANGELOG +-- 0.9.20 Introduction of local Lua functions for private functions (removed _ function prefix). +-- Fixed Lua 5.1 compatibility issues. +-- Introduced json.null to have null values in associative arrays. +-- json.encode() performance improvement (more than 50%) through table.concat rather than .. +-- Introduced decode ability to ignore /**/ comments in the JSON string. +-- 0.9.10 Fix to array encoding / decoding to correctly manage nil/null values in arrays. +----------------------------------------------------------------------------- + +----------------------------------------------------------------------------- +-- Imports and dependencies +----------------------------------------------------------------------------- +local math = require('math') +local string = require("string") +local table = require("table") + +----------------------------------------------------------------------------- +-- Module declaration +----------------------------------------------------------------------------- +local json = {} -- Public namespace +local json_private = {} -- Private namespace + +-- Public functions + +-- Private functions +local decode_scanArray +local decode_scanComment +local decode_scanConstant +local decode_scanNumber +local decode_scanObject +local decode_scanString +local decode_scanWhitespace +local encodeString +local isArray +local isEncodable + +----------------------------------------------------------------------------- +-- PUBLIC FUNCTIONS +----------------------------------------------------------------------------- +--- Encodes an arbitrary Lua object / variable. +-- @param v The Lua object / variable to be JSON encoded. +-- @return String containing the JSON encoding in internal Lua string format (i.e. not unicode) +function json.encode (v) + -- Handle nil values + if v==nil then + return "null" + end + + local vtype = type(v) + + -- Handle strings + if vtype=='string' then + return '"' .. json_private.encodeString(v) .. '"' -- Need to handle encoding in string + end + + -- Handle booleans + if vtype=='number' or vtype=='boolean' then + return tostring(v) + end + + -- Handle tables + if vtype=='table' then + local rval = {} + -- Consider arrays separately + local bArray, maxCount = isArray(v) + if bArray then + for i = 1,maxCount do + table.insert(rval, json.encode(v[i])) + end + else -- An object, not an array + for i,j in pairs(v) do + if isEncodable(i) and isEncodable(j) then + table.insert(rval, '"' .. json_private.encodeString(i) .. '":' .. json.encode(j)) + end + end + end + if bArray then + return '[' .. table.concat(rval,',') ..']' + else + return '{' .. table.concat(rval,',') .. '}' + end + end + + -- Handle null values + if vtype=='function' and v==null then + return 'null' + end + + assert(false,'encode attempt to encode unsupported type ' .. vtype .. ':' .. tostring(v)) +end + + +--- Decodes a JSON string and returns the decoded value as a Lua data structure / value. +-- @param s The string to scan. +-- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. +-- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, +-- and the position of the first character after +-- the scanned JSON object. +function json.decode(s, startPos) + startPos = startPos and startPos or 1 + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') + local curChar = string.sub(s,startPos,startPos) + -- Object + if curChar=='{' then + return decode_scanObject(s,startPos) + end + -- Array + if curChar=='[' then + return decode_scanArray(s,startPos) + end + -- Number + if string.find("+-0123456789.e", curChar, 1, true) then + return decode_scanNumber(s,startPos) + end + -- String + if curChar==[["]] or curChar==[[']] then + return decode_scanString(s,startPos) + end + if string.sub(s,startPos,startPos+1)=='/*' then + return decode(s, decode_scanComment(s,startPos)) + end + -- Otherwise, it must be a constant + return decode_scanConstant(s,startPos) +end + +--- The null function allows one to specify a null value in an associative array (which is otherwise +-- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } +function null() + return null -- so json.null() will also return null ;-) +end +----------------------------------------------------------------------------- +-- Internal, PRIVATE functions. +-- Following a Python-like convention, I have prefixed all these 'PRIVATE' +-- functions with an underscore. +----------------------------------------------------------------------------- + +--- Scans an array from JSON into a Lua object +-- startPos begins at the start of the array. +-- Returns the array and the next starting position +-- @param s The string being scanned. +-- @param startPos The starting position for the scan. +-- @return table, int The scanned array as a table, and the position of the next character to scan. +function decode_scanArray(s,startPos) + local array = {} -- The return value + local stringLen = string.len(s) + assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) + startPos = startPos + 1 + -- Infinite loop for array elements + repeat + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') + local curChar = string.sub(s,startPos,startPos) + if (curChar==']') then + return array, startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') + object, startPos = json.decode(s,startPos) + table.insert(array,object) + until false +end + +--- Scans a comment and discards the comment. +-- Returns the position of the next character following the comment. +-- @param string s The JSON string to scan. +-- @param int startPos The starting position of the comment +function decode_scanComment(s, startPos) + assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) + local endPos = string.find(s,'*/',startPos+2) + assert(endPos~=nil, "Unterminated comment in string at " .. startPos) + return endPos+2 +end + +--- Scans for given constants: true, false or null +-- Returns the appropriate Lua type, and the position of the next character to read. +-- @param s The string being scanned. +-- @param startPos The position in the string at which to start scanning. +-- @return object, int The object (true, false or nil) and the position at which the next character should be +-- scanned. +function decode_scanConstant(s, startPos) + local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } + local constNames = {"true","false","null"} + + for i,k in pairs(constNames) do + if string.sub(s,startPos, startPos + string.len(k) -1 )==k then + return consts[k], startPos + string.len(k) + end + end + assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) +end + +--- Scans a number from the JSON encoded string. +-- (in fact, also is able to scan numeric +- eqns, which is not +-- in the JSON spec.) +-- Returns the number, and the position of the next character +-- after the number. +-- @param s The string being scanned. +-- @param startPos The position at which to start scanning. +-- @return number, int The extracted number and the position of the next character to scan. +function decode_scanNumber(s,startPos) + local endPos = startPos+1 + local stringLen = string.len(s) + local acceptableChars = "+-0123456789.e" + while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) + and endPos<=stringLen + ) do + endPos = endPos + 1 + end + local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) + local stringEval = loadstring(stringValue) + assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON object into a Lua object. +-- startPos begins at the start of the object. +-- Returns the object and the next starting position. +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return table, int The scanned object as a table and the position of the next character to scan. +function decode_scanObject(s,startPos) + local object = {} + local stringLen = string.len(s) + local key, value + assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) + startPos = startPos + 1 + repeat + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') + local curChar = string.sub(s,startPos,startPos) + if (curChar=='}') then + return object,startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') + -- Scan the key + key, startPos = json.decode(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + startPos = decode_scanWhitespace(s,startPos) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) + startPos = decode_scanWhitespace(s,startPos+1) + assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + value, startPos = json.decode(s,startPos) + object[key]=value + until false -- infinite loop while key-value pairs are found +end + +-- START SoniEx2 +-- Initialize some things used by decode_scanString +-- You know, for efficiency +local escapeSequences = { + ["\\t"] = "\t", + ["\\f"] = "\f", + ["\\r"] = "\r", + ["\\n"] = "\n", + ["\\b"] = "\b" +} +setmetatable(escapeSequences, {__index = function(t,k) + -- skip "\" aka strip escape + return string.sub(k,2) +end}) +-- END SoniEx2 + +--- Scans a JSON string from the opening inverted comma or single quote to the +-- end of the string. +-- Returns the string extracted as a Lua string, +-- and the position of the next non-string character +-- (after the closing inverted comma or single quote). +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return string, int The extracted string as a Lua string, and the next character to parse. +function decode_scanString(s,startPos) + assert(startPos, 'decode_scanString(..) called without start position') + local startChar = string.sub(s,startPos,startPos) + -- START SoniEx2 + -- PS: I don't think single quotes are valid JSON + assert(startChar == [["]] or startChar == [[']],'decode_scanString called for a non-string') + --assert(startPos, "String decoding failed: missing closing " .. startChar .. " for string at position " .. oldStart) + local t = {} + local i,j = startPos,startPos + while string.find(s, startChar, j+1) ~= j+1 do + local oldj = j + i,j = string.find(s, "\\.", j+1) + local x,y = string.find(s, startChar, oldj+1) + if not i or x < i then + i,j = x,y-1 + end + table.insert(t, string.sub(s, oldj+1, i-1)) + if string.sub(s, i, j) == "\\u" then + local a = string.sub(s,j+1,j+4) + j = j + 4 + local n = tonumber(a, 16) + assert(n, "String decoding failed: bad Unicode escape " .. a .. " at position " .. i .. " : " .. j) + -- math.floor(x/2^y) == lazy right shift + -- a % 2^b == bitwise_and(a, (2^b)-1) + -- 64 = 2^6 + -- 4096 = 2^12 (or 2^6 * 2^6) + local x + if n < 0x80 then + x = string.char(n % 0x80) + elseif n < 0x800 then + -- [110x xxxx] [10xx xxxx] + x = string.char(0xC0 + (math.floor(n/64) % 0x20), 0x80 + (n % 0x40)) + else + -- [1110 xxxx] [10xx xxxx] [10xx xxxx] + x = string.char(0xE0 + (math.floor(n/4096) % 0x10), 0x80 + (math.floor(n/64) % 0x40), 0x80 + (n % 0x40)) + end + table.insert(t, x) + else + table.insert(t, escapeSequences[string.sub(s, i, j)]) + end + end + table.insert(t,string.sub(j, j+1)) + assert(string.find(s, startChar, j+1), "String decoding failed: missing closing " .. startChar .. " at position " .. j .. "(for string at position " .. startPos .. ")") + return table.concat(t,""), j+2 + -- END SoniEx2 +end + +--- Scans a JSON string skipping all whitespace from the current start position. +-- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. +-- @param s The string being scanned +-- @param startPos The starting position where we should begin removing whitespace. +-- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string +-- was reached. +function decode_scanWhitespace(s,startPos) + local whitespace=" \n\r\t" + local stringLen = string.len(s) + while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do + startPos = startPos + 1 + end + return startPos +end + +--- Encodes a string to be JSON-compatible. +-- This just involves back-quoting inverted commas, back-quotes and newlines, I think ;-) +-- @param s The string to return as a JSON encoded (i.e. backquoted string) +-- @return The string appropriately escaped. + +local escapeList = { + ['"'] = '\\"', + ['\\'] = '\\\\', + ['/'] = '\\/', + ['\b'] = '\\b', + ['\f'] = '\\f', + ['\n'] = '\\n', + ['\r'] = '\\r', + ['\t'] = '\\t' +} + +function json_private.encodeString(s) + local s = tostring(s) + return s:gsub(".", function(c) return escapeList[c] end) -- SoniEx2: 5.0 compat +end + +-- Determines whether the given Lua type is an array or a table / dictionary. +-- We consider any table an array if it has indexes 1..n for its n items, and no +-- other data in the table. +-- I think this method is currently a little 'flaky', but can't think of a good way around it yet... +-- @param t The table to evaluate as an array +-- @return boolean, number True if the table can be represented as an array, false otherwise. If true, +-- the second returned value is the maximum +-- number of indexed elements in the array. +function isArray(t) + -- Next we count all the elements, ensuring that any non-indexed elements are not-encodable + -- (with the possible exception of 'n') + local maxIndex = 0 + for k,v in pairs(t) do + if (type(k)=='number' and math.floor(k)==k and 1<=k) then -- k,v is an indexed pair + if (not isEncodable(v)) then return false end -- All array elements must be encodable + maxIndex = math.max(maxIndex,k) + else + if (k=='n') then + if v ~= table.getn(t) then return false end -- False if n does not hold the number of elements + else -- Else of (k=='n') + if isEncodable(v) then return false end + end -- End of (k~='n') + end -- End of k,v not an indexed pair + end -- End of loop across all pairs + return true, maxIndex +end + +--- Determines whether the given Lua object / table / variable can be JSON encoded. The only +-- types that are JSON encodable are: string, boolean, number, nil, table and json.null. +-- In this implementation, all other types are ignored. +-- @param o The object to examine. +-- @return boolean True if the object should be JSON encoded, false if it should be ignored. +function isEncodable(o) + local t = type(o) + return (t=='string' or t=='boolean' or t=='number' or t=='nil' or t=='table') or (t=='function' and o==null) +end + +return json \ No newline at end of file From 8e34093991bb1c4579c84fd0569b54cafc762889 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 21:05:00 +0200 Subject: [PATCH 17/53] lib.macaddress: Support numeric initialization; add method to get numeric representation. --- src/apps/intel/intel_app.lua | 4 ++-- src/lib/macaddress.lua | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index f104fad553..a4224cf0d0 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -64,7 +64,7 @@ function Intel82599:new (arg) if conf.vmdq then self.counters['phys-address'] = counter.open('phys-address') counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr).bits) + macaddress:new(conf.macaddr):int()) end return self @@ -102,7 +102,7 @@ function Intel82599:reconfig(arg) if conf.vmdq then counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr).bits) + macaddress:new(conf.macaddr):int()) end end diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index e003fa1cc7..e55bdcfb3c 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -13,17 +13,21 @@ function mac_mt:new (m) return m end local macobj = mac_t() - local i = 0; - for b in m:gmatch('%x%x') do - if i == 6 then - -- avoid out of bound array index + if type(m) == 'string' then + local i = 0; + for b in m:gmatch('%x%x') do + if i == 6 then + -- avoid out of bound array index + return nil, "malformed MAC address: " .. m + end + macobj.bytes[i] = tonumber(b, 16) + i = i + 1 + end + if i < 6 then return nil, "malformed MAC address: " .. m end - macobj.bytes[i] = tonumber(b, 16) - i = i + 1 - end - if i < 6 then - return nil, "malformed MAC address: " .. m + else + macobj.bits = m end return macobj end @@ -38,6 +42,10 @@ function mac_mt.__eq (a, b) return a.bits == b.bits end +function mac_mt:int () + return self.bits +end + function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From 5f9efd280f683e394c677e6d606b2d4624831d5f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Apr 2016 21:05:34 +0200 Subject: [PATCH 18/53] =?UTF-8?q?core.link:=20Create=20=E2=80=9Cdiscontinu?= =?UTF-8?q?ity-time=E2=80=9D=20counters.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/link.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/link.lua b/src/core/link.lua index ed825e2606..34e2ffc942 100644 --- a/src/core/link.lua +++ b/src/core/link.lua @@ -28,6 +28,8 @@ function new (name) for _, c in ipairs(counternames) do r.stats[c] = counter.open("counters/"..name.."/"..c) end + counter.set(counter.open("counters/"..name.."/discontinuity-time"), + C.get_unix_time()) return r end @@ -35,6 +37,7 @@ function free (r, name) for _, c in ipairs(counternames) do counter.delete("counters/"..name.."/"..c) end + counter.delete("counters/"..name.."/discontinuity-time") shm.unmap(r) shm.unlink("links/"..name) end From 7b391482949cf10caac34871bb5890af80ab2f6b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 19 Apr 2016 17:44:30 +0200 Subject: [PATCH 19/53] snabb top: add `--yang' option to print YANG model as JSON. --- src/program/top/README | 2 + src/program/top/top.lua | 109 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index ce815d6f89..9fa3500804 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -5,6 +5,8 @@ Usage: Print usage information. -c, --counters Print counters of object by and exit. + -y, --yang + Print YANG model encoded in JSON and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index d69a999695..34ae80be6e 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -9,10 +9,12 @@ local shm = require("core.shm") local counter = require("core.counter") local S = require("syscall") local histogram = require("core.histogram") +local json = require("lib.json") +local macaddress = require("lib.macaddress") local usage = require("program.top.README_inc") local long_opts = { - help = "h", counters = "c" + help = "h", counters = "c", yang = "y" } function clearterm () io.write('\027[2J') end @@ -20,15 +22,18 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local object = nil + local yang = false function opt.h (arg) print(usage) main.exit(1) end function opt.c (arg) object = arg end - args = lib.dogetopt(args, opt, "hc:", long_opts) + function opt.y () yang = true end + args = lib.dogetopt(args, opt, "hc:y", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) - if object then list_counters(target_pid, object) - else top(target_pid) end + if object then list_counters(target_pid, object) + elseif yang then dump_yang(target_pid) + else top(target_pid) end ordered_exit(0) end @@ -55,18 +60,106 @@ function ordered_exit (value) os.exit(value) end +function read_counter (name, path) + if path then name = path.."/"..name end + local value = counter.read(counter.open(name, 'readonly')) + counter.delete(name) + return value +end + function list_counters (pid, object) local path = "//"..pid.."/counters/"..object local cnames = shm.children(path) table.sort(cnames, function (a, b) return a < b end) for _, cname in ipairs(cnames) do - local cpath = path.."/"..cname - local value = counter.read(counter.open(cpath, 'readonly')) - print_row({30, 30}, {cname, lib.comma_value(value)}) - counter.delete(cpath) + print_row({30, 30}, {cname, lib.comma_value(read_counter(cname, path))}) end end +function dump_yang (instance_pid) + local instance_tree = "//"..instance_pid + local interface_state = {} + local types = { [0x1000] = 'hardware', + [0x1001] = 'virtual', + [0x1002] = 'link' } + + for _, link in ipairs(shm.children(instance_tree.."/links")) do + local counters = instance_tree.."/counters/"..link + local statistics = {} + statistics['discontinuity-time'] = + totime(read_counter('discontinuity-time', counters)) + statistics['in-octets'] = + tohex64(read_counter('rxbytes', counters)) + statistics['out-octets'] = + tohex64(read_counter('txbytes', counters)) + statistics['out-discards'] = + truncate32(tonumber(read_counter('txdrop', counters))) + table.insert(interface_state, { name = link, + type = types[0x1002], + statistics = statistics }) + end + + for _, name in ipairs(shm.children(instance_tree.."/counters")) do + local counters = instance_tree.."/counters/"..name + local exists = {} + for _, c in ipairs(shm.children(counters)) do exists[c] = true end + local type = nil + if exists['type'] then + type = types[tonumber(read_counter('type', counters))] + end + if type then + local statistics = {} + statistics['discontinuity-time'] = + totime(read_counter('discontinuity-time', counters)) + for _, c in ipairs({'in-octets', 'in-unicast', + 'in-broadcast', 'in-multicast', + 'out-octets', 'out-unicast', + 'out-broadcast', 'out-multicast'}) do + if exists[c] then + statistics[c] = tohex64(read_counter(c, counters)) + end + end + for _, c in ipairs({'in-discards', 'out-discards'}) do + if exists[c] then + statistics[c] = truncate32(read_counter(c, counters)) + end + end + local interface = { name = name, type = type, statistics = statistics} + if exists['phys-address'] then + interface['phys-address'] = + tomac(read_counter('phys-address', counters)) + end + table.insert(interface_state, interface) + end + end + + print(json.encode({['interface-state'] = interface_state})) +end + +function tomac (n) + return tostring(macaddress:new(n)) +end + +function totime (n) + return os.date("!%FT%TZ", tonumber(n)) +end + +function truncate32 (n) + local box = ffi.new("union { uint64_t i64; uint32_t i32[2]; }") + box.i64 = n + if ffi.abi("le") then return tonumber(box.i32[0]) + elseif ffi.abi("be") then return tonumber(box.i32[1]) end +end + +function tohex64 (n) + local box = ffi.new("uint64_t[1]") + box[0] = n + local s = ffi.string(box, 8) + if ffi.abi("le") then s = s:reverse() end + return string.format("0x%02X%02X%02X%02X%02X%02X%02X%02X", s:byte(1, 8)) +end + + function top (instance_pid) local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) From 6c46b5d5d4967f9c20e3f057d5f27a69ea22eef8 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 2 May 2016 04:30:39 +0000 Subject: [PATCH 20/53] doc: Remove stale .images/Branches.png file Ditaa images are not kept in tree anymore... --- src/doc/.images/Branches.png | Bin 9218 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/doc/.images/Branches.png diff --git a/src/doc/.images/Branches.png b/src/doc/.images/Branches.png deleted file mode 100644 index 7793578f6932b5113ec39ecebeef57d30daacc0c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9218 zcmaia2UJsA*DWf74NwsgA)ulnA|M??iHM36X-X$VrAjXeNLTQJw4flp1rShrF9}7Z z2}lUNgMdIF^w8_u!F#`Z-+TA}{xKLCC+F<5v-jGo%sChSYAW({CoZ0#qN1Wxcz92P zit6wNxNbjo7<}dtIa{cxI2#r2$v%16KR4v2{KRap!Jg?CeZZ6QnC_xronnLj3>=?% z{-IQhN4Eph=F+bBJ7vOiV?}IQY~ZPu!OR~mG?9_r7ES_>Wbb9%^0(sUMTS50*0!?L z4eh7Nc*zsFxm(q|9!Zvztchc}ap~vwDsA*soRFDm!qli#qL8C$!lC^mR8$02It*ov zvb0oGm{uAHD;3q9>N|u(R8-UuaNC>y&u6bNVDQ2HOT_^-j6c)7!+Lo%OuC3 zl$(D~b@0kxgNYA1(jKwie)`3SL8jN{3}sp15%?pJd{lIGbv->j?d|PbVPONMwjFGg z$5;VkFB1jxLIJ^u*-=fj@=4PRVioCqr+}zwZ zZhRjYaNF5f!a2=3xEmbT%+JqHNJyY%VE-sV+3<=Qq1%Hty< zOoug=gZ=#EpFH`H6JCvQU;4x5U3|PWCO9|O*3K?(*s$Cob#s1iZ?8a08zOG^YVu&! za0(&AMNkdshrOJhD zYH3MoPgP14Hz#14Bl%+&WIVQ3PBQa#c6Pe&uJ#Di0^_}6KLPE@frlAfsoU7tVB*m- zHZwz1ylk6C%gD%_eT8i7{QTHN;o-x#gM55^-E2apKcZe(kByBX8>>9FgK!LN;&~+{ z5n7Z*E_x-Wq|hQGjL)q5pnJ>gQz9cHT^K@N{?PaJ^JB%uNIK`(I8Nc?<^1R++~{LP ztp(?0p4T3`cn)~Nm;Eqz!XIvqo2$zU)%!9q7~*Ven5xWYh18rJd~bV42f03&6+6MG zD)p=)#c{0m_>j1eP)B^j(St3}LFwq|^h<5JbU2;n2S4|vVzK?E*zc3YJ%*Xt*&=+> zCAbG>ouAZQ)m0upehd!Y!Xjff0&@ho%ujY&h`)FY{P5w!D;$u+f}g$1%*bAe&RLGZkxffrFJF$AqLrWELJb~qi(4U9Ci{4v9J%s zt@NIZq^&Hi%o)Pxd2FxAv%B2$LSB-VuFU12-Fj}wMWnI|3^H^*UF|Rtuo{t?n#v%( zqQ>vSk-APD?>RWu?K~Zwq%PPky?{c;kCIzqVmYcD(ym2IDT>q(Z~9j<60;&ABH4zO zdS;@Rh^LWHoK+Y3Ds8$lG%J&OB%FLJ-PUJ9jT+FaA%?PqLr-G`O#;}DsU%3TFfxvl zNU@=z8v6SBnwsJ7-u;}KVrFL6*3+}F;`jU37{*0%?{VlCvmdLCi(`Xps;Ob*8-s#Q zahDVoDaQya1iXL$e$F%IDVvYX-p2UG#!C*4MEmi2CMKrnQhQNY$*R$`>OgcbM6!G7 z1_vvvo(`yu>5Z?6E%5&kehzP1T z?^*j;yH0NP9+ofiX87(yh(&%yg~56SFd-hr8|&7QR63v^S6p`A8Rf*>kkpp!IXGQo zYSx-CS7kWm@XdvUzm#pHec85p*Qu0UVDR&u$e|xCyNt+J+tNg88k#17Z{lAXzuu*1 zdmwQ2>c{+i^-{0B%_-E0s}csTFJBUAINTRs=fLxxzE&}VKYkn*9sSudgk9Rg-XP+u z+#083XYj=7ogwFuX_{`bk-k(!GbSuN+(hd6K0VZHkDS<&tL}{{40~BPmEfk*(-f5i zi!d7YaiI=XZ-ElaMv4cc>4$V}?Yqa$4|TO*jNmZ7?YzG3Y<*mDPkEf(Fjfe8gNW=p zQ)6-zJ8L>Wn@EX;J)^ed`{Ci?jMtS3$#HizpFFvxTW~MX0c~Cp?l3&Pr>$Gmd5TICD1|!K8E_8Vo7$?Ydn30&LYU^k3ar! zaHeAmm7-^t(m@~)aQJSwzHPMpCI6G%vJEXc6MIPF+}e3FUYj8}+%N@fZ(fQgrAM0l zLuhD-l*JY2+{a>-hG^JZ=}>i>d^=ij815wGyrxPxbPT#LF^BWq#h#n6OKEsd$)V5g zGsKV058)gls$VXRt*opR6$$U&hf72YIq5ukV$4=Mv5|}`7UyGNm)Zap6zyO-!YXQ& zS1v$t!D30@#z5DfuW9329`u3z#Wj(Xw&!fUGL>X@7Ujsjrpmgj^{lo;8WR%}tnudZ z!~@2MlXipUu?O3jL=y3?fraNw=X3ac(5T%qa#m5#3tv(+X58wVOJsZ!Nc)-S3w%5T z6ER1#eQF3MWV*jEdo_>g%$cM*5Gz9(_^y?YYJm=ECYCs~AUAiTeW^pY<*b@YbC6k? zY0JgP1`~-R9nT*<3LaMS<>1D~aSUwb_d{xaq3iEsDkGNGwpDFMIOWmAW#?fR`N#`& z8Ah?{jSE%VIW~(eVk5**X2X|Xbu3TmotJHRNaghRs+DlO^94BUDQNfVIKHsmaB9O} zQ+szGx+fmy}j+7|}yPZLS}!Z*5%wg{!;4WIV|vPk>|G03Tw#tAm%phH49gTsXXg zR$&%Y85?mYKY~B6upe87#G5on=vJpNa&XvIZ8UzTMl*BbI*OrC2o`SpqO=a~)RVZ3 zMt6otwl)z_PCb{LMl8v!+7)A(y zVoB=BEiEb8TaK(oIt8oTzkeSNkGDV6(Uu|)$!#mQ8~q0A8fvQ0thHMS0SuwUqRXWc zxAE9pMLu25ne7Zjf6&C)x-?82_n;x)bC3brPtwZP>k{fA7Dn*#;BFvaCBK zo9;G{eW<+Afi>NwBc811&RxCF5;plGTF8CF4hDmb=Uwr`*Vf*HFd3@}qh({oY(Yq4 z>vnqiI7pz<3NUcxRxAe$r8GBU4NH4Ne9$c`_gPM2I84Tehp)~SwqEpO8sFJi9Ie4FRk)B>c3+XFTAi;Dd*fO|=I}<1qxv*V z5J`~Kv**M%f&(vXSrXb3RagX`zv=w3GTjk;&wLLO58U~o*s){B2na*nYR}ye*q%9i zY-+!cj}L%mw^qxI8_&Xr+uIAA=K9~hJtpB7VIiDjHHReRql}>|PHurlf%-^oGiQ6Y zi@Y=GDYEiz@oQc4+YpEuBOW(<3eJu+5PjWz0@>FBE%NfFw}a6@D)#rbw~~%)zGim> z=!3R1A!(tI3O+y6l?fjpQ9vtzz?hLMpp;TTBKSf9t_OFi01%>J zT3K2OHm0E8|2hCbE&xLS00I}kKf#39hPAl?4wqkN`1$!+LckeW|M;u~VEAd|mR~Du z-~CSPt5>gp_vPy3dt>B&Kkx@N+=3wCG#izlmz8x32CFVBi`u#j)^H5!p{c0Y^lE=? zZNPS5ZyOR1{2U(s{{7{;h=>SGI8+ ziV?~QQWCdstJ&N#C{xoG5f!Bea4X^}7uQIIOJQ*_YPvmDgH70+0Kz#lBLg`#F)>kK z@90Q2OZ13{js0yYg1$HU#?708!>jKndU6px6LocUSi2ul0v-Leb#>FB03v|xJvtjQ zhz|P*(1Bx_%#RvdzD4nRy;3tHE z7?BwV4hCec8d}eBzR_M2}x4l3nldmmFZT4X*48kw># zA>h?d7t?Y;(8c3#aC2|U#cpkF@r{(vl2aoMv9`1e^j+2?8JRCS(i-;wO8cskRSDZS z0cEGHcy6F(%Q^Nib^M+(y5~s00pM!Q4L*MCuAeOZ2CA5lgSi7T+S==mgz4I9{?n{5s8Ti1A`v|jF*VWVD86hL^b>6UyV^w7Z8Y%4QpK{ zd-hC2LqkpNU1{n1#?rX+T>pf4P@_7(sHhhG7|55x-3iieTIXZd`(kNr$BW%}mKv6p zmO#oLuMg${F=9KOb@RlD6Dvnejg24Xr>2^;e^k&23JS`~%`N>jAfu;!Ufr}VfN5)I z#FfNL?r)XZR>mfG>LZazL&JpFq2eKWQ;$f)t}Ne~nVEUZ5%xX#+ofeFCV|6;sN?CW zDk0FnI$$uH(S5jBv!(R-_^Ln zO{E-NUW|CG$J;!GGo3tti=YW;PjqxNc8%DU(tWugq&vJ2Q~0cg#$(yGYBSBn`YG#p zszSr|SRlWFleQV;#};2@x-K2SFGKxR(mNqfnXP#tvabi}zKUFl}CMRDR4{y4SnKKc%vNnXZSf`_!)63M#b}%3T6-VH1BvGFA^5x5X zI7=Ml#{_)dr7&h-{WUE+@%7qVqZhTlTMu0K$r*s0aI#!iuG|e|VPSE-^EoT)!&tPC zS|vpyY#|wPzNgx7wu5Er4bABvtX)ft$Ygw}mgO7I50+z>K!pR&)j&VF%(q^l+cIZq;5wLn^#k(RwqB18PPQTW zPxIG5t2+jDNV`_ok**r+ga7IEl($397QeXr;Ld&9tt*j%TU&!tJjmG)pYZE?a zvcHr!1NbaOJ|u$m(4j+sjiZ6UF>u6bfmk-1gNFQwXZLK4?Ha5C`GKTAX9+i&6Aie} zbO@v|dqQJ#b4Ri#!$f9o)$;nfnD1*}Utdt4EV9l>I?oF)SO6rgck9Lt0$mW#g$KbG zQzLJE-$ec?*ECJ;G}n~cju;kOXnLK1@@3vVZ=umB;DwMF0had_&4C?>iu`)}UaY2wNK{%v=AokYvAtUdqSee-^` zq{S`|_qA)+T*IHkjzPOM+xq)sqM|+}wmodS5F}BPkDt>u7y?&fB{>I+fFx#Wm#81o#La1LXGR|>NYnv zzAIR0X>}L2%2=K{;;^?hlbKszTT4w#>*nIp_qtlmqQ+VVb>pFUC3$qC0SXspVJfCN7^=tDIq_Ox9U2hy1!;Ev=799L$6;| z*u73Hdwd_%*xivNIlq-7hcm_!I+;X9+$bQqh#z2L7;<$+Gje1^R;-KZ$g^CBi6*9g zW&oR>Ub>fPi@-5_i#kn_1gRmQ;vW=!3M2qv_%zZSvCe`ZOSvo?D=P;9y;MY4GC3=S@C*nwK*Raxe}z>^ixbp6IE^v6bJz-u(T$xrIf6KsFT2cjU4; zA`ruP;lho^nhn8py5f2n*bp`%zilir*D%bjEW zrMno;^SpXb0J?qljheTg^Q&cR?tCSvUgNH9zO>SP6^Cj9cMMOq64?Wh< zK>B?9c9*py!v}NZ_G#}m)Onb21+{30&QUzha%XU&AXbB zKe3WOR4&!CVm)+qqxp2J+{^i%>7jsKXzs4K)Fb)ef^OwB@qoziUWC;d>gsaW37O%e zyaI+MRu%whZz(6i@lyNqxP=)SK$nC0Tt@$3Mjk&{!)fFzCvSADBfczdZ|DjUi3GnV z)GAq(QfSpD2>(_zb8re6vTVb_TCjQ3Ae?!23Axji+^Hy+jN&G?x8DS65g0#HxUlsl;E|X^Bu;7wPz$ zv-Nqoy1L4f=nj@W$SU}R!Q2r92#HroNr}Uim^8e@t}iCiF_q|*qj~Q1>D#U=L?GHD zKMjLWc{R+AY-(y+CX+!aU1-I0baocGu3D(7stzrCo`kxs&v!9<&i3Y`X=*{+gN!%@ zUU55(l-c-s1oBBGQHHk;aAh4(j151GjqI(7@g_h~P4?6x0S>wUlwVmSZLO{0K!FDE ztGXJjcME8_DBA);Z(3g7;&gf<{fQIHYipjf`3-`Cf>%{_LPqfHD(ev+K74dzg?aAe zz;Q?yS=rYzMYD(iz*|6S34LyCZ0zB&3-o<~pVbsicfKt%Gt zW%<_ulgtGS7UajtU%!YlAgE?Dh;yYQPFlA0jPI0hgqhxrIKMv@PI-!`6 zcM@73OKsXJy&hM$VZCJ6*Vl(7^z?}R#Y}tnbn@F>Gg6j|ho|{v zEjV>(XB7pco|_0)6b)DG=Dl_k#YcSD@z9#I3=qxYU#5R!Xuq6^*i~#Oe0_vgajB?tWJvFR(gAGlDto{&|EB6S1(^rN`@#$rK10grZ$a-4V@ z8}cG6E6`7`)GaP9E-x=*u}~vn-dnejqMnj&;(Y^zB)Qb6xHxz6pjP|6c~e_kJ6P^0 zUyb1OiM~%@k$RbDd{KYEPZC;T-J2&HKtY;CWx!d7K<{US4XF|*o>V6`{8CQrozzFA z2!6!8)sEAe9U$U*lpa)(C_n{R1 zDb(aI&=WJdcVc~{O}E4@YzEIYnepCTe?$AG3m&p9G8}s2AYMS;ppp2 z!`*Zj(67Fydpn!1y2!hC?qb)3afunC{-np7&K**PHC*1_F-F9bM)vKM(X<>No!X;CD9N-}hGjNnvkQ50y0iu0+gv5kbl_5dkP>P{I1~DA5+6Un?1PS-)7Q{A%gj7AJ}zX@ z`6(+a3kX=lTLxL|9K$IwaCh!4U&{? zm>d>?)>BVXkX!L1$246tU@#}v3qo;;lFs<&qT4`gr=g`Ksd7L-+N-Ln!eY0wONW!m zAjGKFjR^0bCUq17Dxfuf*TyDCqQRVcx*Ib-KHf1nn56_LWX8hPXtCR^l>Jr;NbZ8h z$n25}rSJ;A;b!4h^mE12==X_xVOI(Rbk zdT(Wl(H<1LYPD98La{AINS3lc*U`mDgeP}-ZItn_E`X}_4&!IzdJNsNOG-p3s*6(( zG0F!G-a_{f4a)4E0Xirw1{~uKA^!vvq~yQ)2OAeZ6k^8lK@)nQl> Date: Wed, 20 Apr 2016 13:43:36 +0200 Subject: [PATCH 21/53] snabb top --yang: Represent uint64_t as decimal string. --- src/program/top/top.lua | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 34ae80be6e..317c0a2a77 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -89,9 +89,9 @@ function dump_yang (instance_pid) statistics['discontinuity-time'] = totime(read_counter('discontinuity-time', counters)) statistics['in-octets'] = - tohex64(read_counter('rxbytes', counters)) + tostring64(read_counter('rxbytes', counters)) statistics['out-octets'] = - tohex64(read_counter('txbytes', counters)) + tostring64(read_counter('txbytes', counters)) statistics['out-discards'] = truncate32(tonumber(read_counter('txdrop', counters))) table.insert(interface_state, { name = link, @@ -116,7 +116,7 @@ function dump_yang (instance_pid) 'out-octets', 'out-unicast', 'out-broadcast', 'out-multicast'}) do if exists[c] then - statistics[c] = tohex64(read_counter(c, counters)) + statistics[c] = tostring64(read_counter(c, counters)) end end for _, c in ipairs({'in-discards', 'out-discards'}) do @@ -151,12 +151,10 @@ function truncate32 (n) elseif ffi.abi("be") then return tonumber(box.i32[1]) end end -function tohex64 (n) - local box = ffi.new("uint64_t[1]") - box[0] = n - local s = ffi.string(box, 8) - if ffi.abi("le") then s = s:reverse() end - return string.format("0x%02X%02X%02X%02X%02X%02X%02X%02X", s:byte(1, 8)) +function tostring64 (n) + assert(ffi.istype('uint64_t', n)) + local s = tostring(n) + return s:sub(1, #s - 3) end From ee00d167b2aa437861f5ee04804e80755053ef36 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 18:26:55 +0200 Subject: [PATCH 22/53] [core.lib] Generalize `timer' to optionally accept 'repeating' option and support injecting a function to determine the current time. --- src/README.md | 14 +++++++++++--- src/core/lib.lua | 21 +++++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/README.md b/src/README.md index b406eeabd9..4cfbb73dd7 100644 --- a/src/README.md +++ b/src/README.md @@ -543,10 +543,18 @@ Returns a table that acts as a bounds checked wrapper around a C array of ctype and the caller must ensure that the allocated memory region at *base*/*offset* is at least `sizeof(type)*size` bytes long. -— Function **lib.timer** *s* +— Function **lib.timer** *duration*, *mode*, *timefun* + +Returns a closure that will return `false` until *duration* has elapsed. If +*mode* is `'repeating'` the timer will reset itself after returning `true`, +thus implementing an interval timer. *Timefun* is used to get a monotonic time. +*Timefun* defaults to `C.get_time_ns`. + +The “deadline” for a given *duration* is computed by adding *duration* to the +result of calling *timefun*, and is saved in the resulting closure. A +*duration* has elapsed when its deadline is less than or equal the value +obtained using *timefun* when calling the closure. -Returns a function that accepts no parameters and acts as a predicate to -test if *ns* nanoseconds have elapsed. — Function **lib.waitfor** *condition* diff --git a/src/core/lib.lua b/src/core/lib.lua index b6223cc8e6..6e37a90964 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -246,10 +246,23 @@ function bounds_checked (type, base, offset, size) return wrap(ffi.cast(tptr, ffi.cast("uint8_t *", base) + offset)) end --- Return a function that will return false until NS nanoseconds have elapsed. -function timer (ns) - local deadline = C.get_time_ns() + ns - return function () return C.get_time_ns() >= deadline end +-- Return a function that will return false until duration has elapsed. +-- If mode is 'repeating' the timer will reset itself after returning true, +-- thus implementing an interval timer. Timefun defaults to `C.get_time_ns'. +function timer (duration, mode, timefun) + timefun = timefun or C.get_time_ns + local deadline = timefun() + duration + local function oneshot () + return timefun() >= deadline + end + local function repeating () + if timefun() >= deadline then + deadline = deadline + duration + return true + else return false end + end + if mode == 'repeating' then return repeating + else return oneshot end end -- Loop until the function `condition` returns true. From 45490b8e162c96e16f739672c951d9b92f896fd5 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 19:30:55 +0200 Subject: [PATCH 23/53] Revert "Intel_app: Add RFC 7223 app counters." This reverts commit 8bb3215bf759f6c53af07c5c13afd2ea60003334. --- src/apps/intel/intel_app.lua | 45 ++++-------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index a4224cf0d0..80cace3d9f 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,10 +6,8 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") -local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") -local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local freelist = require("core.freelist") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty @@ -38,7 +36,6 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) - local self = setmetatable({counters = {}}, Intel82599) if conf.vmdq then if devices[conf.pciaddr] == nil then @@ -48,26 +45,12 @@ function Intel82599:new (arg) local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - self.dev = vf:open(conf) + return setmetatable({dev=vf:open(conf)}, Intel82599) else - self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") - self.zone = "intel" + local dev = intel10g.new_sf(conf):open() + if not dev then return null end + return setmetatable({dev=dev, zone="intel"}, Intel82599) end - - self.counters['type'] = counter.open('type') - self.counters['discontinuity-time'] = counter.open('discontinuity-time') - self.counters['in-octets'] = counter.open('in-octets') - self.counters['in-discards'] = counter.open('in-discards') - self.counters['out-octets'] = counter.open('out-octets') - counter.set(self.counters['type'], 0x1000) -- Hardware interface - counter.set(self.counters['discontinuity-time'], C.get_unix_time()) - if conf.vmdq then - self.counters['phys-address'] = counter.open('phys-address') - counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr):int()) - end - - return self end function Intel82599:stop() @@ -88,9 +71,6 @@ function Intel82599:stop() if close_pf then close_pf:close() end - - -- delete counters - for name, _ in pairs(self.counters) do counter.delete(name) end end @@ -99,11 +79,6 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) - - if conf.vmdq then - counter.set(self.counters['phys-address'], - macaddress:new(conf.macaddr):int()) - end end -- Allocate receive buffers from the given freelist. @@ -121,13 +96,6 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() - if self.dev.rxstats then - counter.set(self.counters['in-octets'], - bit.lshift(self.dev.pf.qs.QBRC_H[self.dev.rxstats]()+0LL, 32) - + self.dev.pf.qs.QBRC_L[self.dev.rxstats]()) - counter.set(self.counters['in-discards'], - self.dev.pf.qs.QPRDC[self.dev.rxstats]()) - end end function Intel82599:add_receive_buffers () @@ -148,11 +116,6 @@ function Intel82599:push () end end self.dev:sync_transmit() - if self.dev.txstats then - counter.set(self.counters['out-octets'], - bit.lshift(self.dev.pf.qs.QBTC_H[self.dev.txstats]()+0LL, 32) - + self.dev.pf.qs.QBTC_L[self.dev.txstats]()) - end end -- Report on relevant status and statistics. From f0ed10b70781505e38809acd189744067993f516 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 28 Apr 2016 20:34:23 +0200 Subject: [PATCH 24/53] intel_app: expose per-pciaddress statistics in `counters/'. --- src/apps/intel/intel_app.lua | 69 +++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 80cace3d9f..0fff2dfc5e 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -6,6 +6,8 @@ local zone = require("jit.zone") local basic_apps = require("apps.basic.basic_apps") local ffi = require("ffi") local lib = require("core.lib") +local shm = require("core.shm") +local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") local intel10g = require("apps.intel.intel10g") @@ -36,21 +38,49 @@ end -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) local conf = config.parse_app_arg(arg) + local self = {} if conf.vmdq then if devices[conf.pciaddr] == nil then - devices[conf.pciaddr] = {pf=intel10g.new_pf(conf):open(), vflist={}} + local pf = intel10g.new_pf(conf):open() + devices[conf.pciaddr] = {pf=pf, vflist={}, stats={register=pf.s}} end local dev = devices[conf.pciaddr] local poolnum = firsthole(dev.vflist)-1 local vf = dev.pf:new_vf(poolnum) dev.vflist[poolnum+1] = vf - return setmetatable({dev=vf:open(conf)}, Intel82599) + self.dev = vf:open(conf) + self.stats = devices[conf.pciaddr].stats else - local dev = intel10g.new_sf(conf):open() - if not dev then return null end - return setmetatable({dev=dev, zone="intel"}, Intel82599) + self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") + self.stats = { register = self.dev.s } + self.zone = "intel" end + if not self.stats.counters then + local counters = {} + local path = "/counters/"..conf.pciaddr.."/" + counters['type'] = counter.open(path..'type') + counters['discontinuity-time'] = counter.open(path..'discontinuity-time') + counters['in-octets'] = counter.open(path..'in-octets') + counters['in-multicast'] = counter.open(path..'in-multicast') + counters['in-broadcast'] = counter.open(path..'in-broadcast') + counters['in-discards'] = counter.open(path..'in-discards') + counters['out-octets'] = counter.open(path..'out-octets') + counters['out-multicast'] = counter.open(path..'out-multicast') + counters['out-broadcast'] = counter.open(path..'out-broadcast') + counters['out-discards'] = counter.open(path..'out-discards') + counter.set(counters['type'], 0x1000) -- Hardware interface + counter.set(counters['discontinuity-time'], C.get_unix_time()) + if not conf.vmdq and conf.macaddr then + counters['phys-address'] = counter.open(path..'phys-address') + counter.set(counters['phys-address'], + macaddress:new(conf.macaddr):int()) + end + self.stats.counters = counters + self.stats.path = path + self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) + end + return setmetatable(self, Intel82599) end function Intel82599:stop() @@ -71,6 +101,12 @@ function Intel82599:stop() if close_pf then close_pf:close() end + if not self.dev.pf or close_pf then + for name, _ in pairs(self.stats.counters) do + counter.delete(self.stats.path..name) + end + shm.unlink(self.stats.path) + end end @@ -79,6 +115,11 @@ function Intel82599:reconfig(arg) assert((not not self.dev.pf) == (not not conf.vmdq), "Can't reconfig from VMDQ to single-port or viceversa") self.dev:reconfig(conf) + + if not self.dev.pf and conf.macaddr then + counter.set(self.stats.counters['phys-address'], + macaddress:new(conf.macaddr):int()) + end end -- Allocate receive buffers from the given freelist. @@ -96,6 +137,9 @@ function Intel82599:pull () transmit(l, self.dev:receive()) end self:add_receive_buffers() + if self.stats.sync_timer() then + self:sync_stats() + end end function Intel82599:add_receive_buffers () @@ -105,6 +149,21 @@ function Intel82599:add_receive_buffers () end end +-- Synchronize self.stats.register a and self.stats.counters. +function Intel82599:sync_stats () + --- XXX - it is questionable if I choose the right register to counter + --- mapping + local counters, register = self.stats.counters, self.stats.register + counter.set(counters['in-octets'], register.GORC64()) + counter.set(counters['in-multicast'], register.MPRC()) + counter.set(counters['in-broadcast'], register.BPRC()) + counter.set(counters['in-discards'], register.TPR() - register.GPRC()) + counter.set(counters['out-octets'], register.GOTC64()) + counter.set(counters['out-multicast'], register.MPTC()) + counter.set(counters['out-broadcast'], register.BPTC()) + counter.set(counters['out-discards'], register.TPT() - register.GPTC()) +end + -- Push packets from our 'rx' link onto the network. function Intel82599:push () local l = self.input.rx From c186591e312fb90c4c2d847be01255fb7dba0c8b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 25 Apr 2016 16:25:06 +0200 Subject: [PATCH 25/53] lib.protocol.ethernet: Add n_mcast, branch-free Multicast predicate. --- src/apps/vhost/vhost_user.lua | 16 ++++++---------- src/lib/protocol/README.md | 9 +++++++++ src/lib/protocol/ethernet.lua | 7 ++++++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 96e82780eb..16936709bf 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -102,20 +102,16 @@ end function VhostUser:tx_callback (p) counter.add(self.counters['out-octets'], packet.length(p)) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters['out-multicast']) - else - counter.add(self.counters['out-unicast']) - end + local mcast = ethernet:n_mcast(packet.data(p)) + counter.add(self.counters['out-multicast'], mcast) + counter.add(self.counters['out-unicast'], 1 - mcast) end function VhostUser:rx_callback (p) counter.add(self.counters['in-octets'], packet.length(p)) - if ethernet:is_mcast(packet.data(p)) then - counter.add(self.counters['in-multicast']) - else - counter.add(self.counters['in-unicast']) - end + local mcast = ethernet:n_mcast(packet.data(p)) + counter.add(self.counters['in-multicast'], mcast) + counter.add(self.counters['in-unicast'], 1 - mcast) end function VhostUser:rxdrop_callback (p) diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 6f239687d7..4a13483a4b 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -98,6 +98,15 @@ Returns the binary representation of MAC address denoted by *string*. Returns the string representation of *mac* address. +— Function **ethernet:is_mcast** *mac* + +Returns a true value if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet). + +— Function **ethernet:n_mcast** *mac* + +Returns 1 if *mac* address denotes a [Multicast address](https://en.wikipedia.org/wiki/Multicast_address#Ethernet) +and 0 otherwise. + — Function **ethernet:ipv6_mcast** *ip* Returns the MAC address for IPv6 multicast *ip* as defined by RFC2464, diff --git a/src/lib/protocol/ethernet.lua b/src/lib/protocol/ethernet.lua index 0a55c20f1b..b5abed8d31 100644 --- a/src/lib/protocol/ethernet.lua +++ b/src/lib/protocol/ethernet.lua @@ -76,9 +76,14 @@ function ethernet:ipv6_mcast(ip) return result end +-- Return 1 if MAC address has its group bit set and 0 otherwise +function ethernet:n_mcast (addr) + return band(addr[0], 0x01) +end + -- Check whether a MAC address has its group bit set function ethernet:is_mcast (addr) - return band(addr[0], 0x01) ~= 0 + return ethernet:n_mcast(addr) ~= 0 end -- Instance methods From b09e843465686fd1afad4c6c48a248f33de6b07e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 27 May 2016 15:24:11 +0200 Subject: [PATCH 26/53] Fix for f0ed10b: require macaddress module. --- src/apps/intel/intel_app.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index f5773d66f3..2645e2d22c 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -10,6 +10,7 @@ local shm = require("core.shm") local counter = require("core.counter") local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") +local macaddress = require("lib.macaddress") local intel10g = require("apps.intel.intel10g") local receive, transmit, full, empty = link.receive, link.transmit, link.full, link.empty Intel82599 = {} From 62e2fbfe631a56631f7c115fd73b82550256fd83 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 1 Jun 2016 15:25:44 +0200 Subject: [PATCH 27/53] Revert "lib.json: Import JSON4Lua 1.0.0, include encode functionality." This reverts commit 924ff4e6e82880d296d6be9ddae4dda3aef4dff0. --- src/lib/json.lua | 658 +++++++++++++++++------------------------------ 1 file changed, 241 insertions(+), 417 deletions(-) diff --git a/src/lib/json.lua b/src/lib/json.lua index 9ab4abd38e..86cbf85e43 100644 --- a/src/lib/json.lua +++ b/src/lib/json.lua @@ -1,417 +1,241 @@ ------------------------------------------------------------------------------ --- JSON4Lua: JSON encoding / decoding support for the Lua language. --- json Module. --- Author: Craig Mason-Jones --- Homepage: http://github.com/craigmj/json4lua/ --- Version: 1.0.0 --- This module is released under the MIT License (MIT). --- Please see LICENCE.txt for details. --- --- USAGE: --- This module exposes two functions: --- json.encode(o) --- Returns the table / string / boolean / number / nil / json.null value as a JSON-encoded string. --- json.decode(json_string) --- Returns a Lua object populated with the data encoded in the JSON string json_string. --- --- REQUIREMENTS: --- compat-5.1 if using Lua 5.0 --- --- CHANGELOG --- 0.9.20 Introduction of local Lua functions for private functions (removed _ function prefix). --- Fixed Lua 5.1 compatibility issues. --- Introduced json.null to have null values in associative arrays. --- json.encode() performance improvement (more than 50%) through table.concat rather than .. --- Introduced decode ability to ignore /**/ comments in the JSON string. --- 0.9.10 Fix to array encoding / decoding to correctly manage nil/null values in arrays. ------------------------------------------------------------------------------ - ------------------------------------------------------------------------------ --- Imports and dependencies ------------------------------------------------------------------------------ -local math = require('math') -local string = require("string") -local table = require("table") - ------------------------------------------------------------------------------ --- Module declaration ------------------------------------------------------------------------------ -local json = {} -- Public namespace -local json_private = {} -- Private namespace - --- Public functions - --- Private functions -local decode_scanArray -local decode_scanComment -local decode_scanConstant -local decode_scanNumber -local decode_scanObject -local decode_scanString -local decode_scanWhitespace -local encodeString -local isArray -local isEncodable - ------------------------------------------------------------------------------ --- PUBLIC FUNCTIONS ------------------------------------------------------------------------------ ---- Encodes an arbitrary Lua object / variable. --- @param v The Lua object / variable to be JSON encoded. --- @return String containing the JSON encoding in internal Lua string format (i.e. not unicode) -function json.encode (v) - -- Handle nil values - if v==nil then - return "null" - end - - local vtype = type(v) - - -- Handle strings - if vtype=='string' then - return '"' .. json_private.encodeString(v) .. '"' -- Need to handle encoding in string - end - - -- Handle booleans - if vtype=='number' or vtype=='boolean' then - return tostring(v) - end - - -- Handle tables - if vtype=='table' then - local rval = {} - -- Consider arrays separately - local bArray, maxCount = isArray(v) - if bArray then - for i = 1,maxCount do - table.insert(rval, json.encode(v[i])) - end - else -- An object, not an array - for i,j in pairs(v) do - if isEncodable(i) and isEncodable(j) then - table.insert(rval, '"' .. json_private.encodeString(i) .. '":' .. json.encode(j)) - end - end - end - if bArray then - return '[' .. table.concat(rval,',') ..']' - else - return '{' .. table.concat(rval,',') .. '}' - end - end - - -- Handle null values - if vtype=='function' and v==null then - return 'null' - end - - assert(false,'encode attempt to encode unsupported type ' .. vtype .. ':' .. tostring(v)) -end - - ---- Decodes a JSON string and returns the decoded value as a Lua data structure / value. --- @param s The string to scan. --- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. --- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, --- and the position of the first character after --- the scanned JSON object. -function json.decode(s, startPos) - startPos = startPos and startPos or 1 - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') - local curChar = string.sub(s,startPos,startPos) - -- Object - if curChar=='{' then - return decode_scanObject(s,startPos) - end - -- Array - if curChar=='[' then - return decode_scanArray(s,startPos) - end - -- Number - if string.find("+-0123456789.e", curChar, 1, true) then - return decode_scanNumber(s,startPos) - end - -- String - if curChar==[["]] or curChar==[[']] then - return decode_scanString(s,startPos) - end - if string.sub(s,startPos,startPos+1)=='/*' then - return decode(s, decode_scanComment(s,startPos)) - end - -- Otherwise, it must be a constant - return decode_scanConstant(s,startPos) -end - ---- The null function allows one to specify a null value in an associative array (which is otherwise --- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } -function null() - return null -- so json.null() will also return null ;-) -end ------------------------------------------------------------------------------ --- Internal, PRIVATE functions. --- Following a Python-like convention, I have prefixed all these 'PRIVATE' --- functions with an underscore. ------------------------------------------------------------------------------ - ---- Scans an array from JSON into a Lua object --- startPos begins at the start of the array. --- Returns the array and the next starting position --- @param s The string being scanned. --- @param startPos The starting position for the scan. --- @return table, int The scanned array as a table, and the position of the next character to scan. -function decode_scanArray(s,startPos) - local array = {} -- The return value - local stringLen = string.len(s) - assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) - startPos = startPos + 1 - -- Infinite loop for array elements - repeat - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') - local curChar = string.sub(s,startPos,startPos) - if (curChar==']') then - return array, startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') - object, startPos = json.decode(s,startPos) - table.insert(array,object) - until false -end - ---- Scans a comment and discards the comment. --- Returns the position of the next character following the comment. --- @param string s The JSON string to scan. --- @param int startPos The starting position of the comment -function decode_scanComment(s, startPos) - assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) - local endPos = string.find(s,'*/',startPos+2) - assert(endPos~=nil, "Unterminated comment in string at " .. startPos) - return endPos+2 -end - ---- Scans for given constants: true, false or null --- Returns the appropriate Lua type, and the position of the next character to read. --- @param s The string being scanned. --- @param startPos The position in the string at which to start scanning. --- @return object, int The object (true, false or nil) and the position at which the next character should be --- scanned. -function decode_scanConstant(s, startPos) - local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } - local constNames = {"true","false","null"} - - for i,k in pairs(constNames) do - if string.sub(s,startPos, startPos + string.len(k) -1 )==k then - return consts[k], startPos + string.len(k) - end - end - assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) -end - ---- Scans a number from the JSON encoded string. --- (in fact, also is able to scan numeric +- eqns, which is not --- in the JSON spec.) --- Returns the number, and the position of the next character --- after the number. --- @param s The string being scanned. --- @param startPos The position at which to start scanning. --- @return number, int The extracted number and the position of the next character to scan. -function decode_scanNumber(s,startPos) - local endPos = startPos+1 - local stringLen = string.len(s) - local acceptableChars = "+-0123456789.e" - while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) - and endPos<=stringLen - ) do - endPos = endPos + 1 - end - local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) - local stringEval = loadstring(stringValue) - assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) - return stringEval(), endPos -end - ---- Scans a JSON object into a Lua object. --- startPos begins at the start of the object. --- Returns the object and the next starting position. --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return table, int The scanned object as a table and the position of the next character to scan. -function decode_scanObject(s,startPos) - local object = {} - local stringLen = string.len(s) - local key, value - assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) - startPos = startPos + 1 - repeat - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') - local curChar = string.sub(s,startPos,startPos) - if (curChar=='}') then - return object,startPos+1 - end - if (curChar==',') then - startPos = decode_scanWhitespace(s,startPos+1) - end - assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') - -- Scan the key - key, startPos = json.decode(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - startPos = decode_scanWhitespace(s,startPos) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) - startPos = decode_scanWhitespace(s,startPos+1) - assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) - value, startPos = json.decode(s,startPos) - object[key]=value - until false -- infinite loop while key-value pairs are found -end - --- START SoniEx2 --- Initialize some things used by decode_scanString --- You know, for efficiency -local escapeSequences = { - ["\\t"] = "\t", - ["\\f"] = "\f", - ["\\r"] = "\r", - ["\\n"] = "\n", - ["\\b"] = "\b" -} -setmetatable(escapeSequences, {__index = function(t,k) - -- skip "\" aka strip escape - return string.sub(k,2) -end}) --- END SoniEx2 - ---- Scans a JSON string from the opening inverted comma or single quote to the --- end of the string. --- Returns the string extracted as a Lua string, --- and the position of the next non-string character --- (after the closing inverted comma or single quote). --- @param s The string being scanned. --- @param startPos The starting position of the scan. --- @return string, int The extracted string as a Lua string, and the next character to parse. -function decode_scanString(s,startPos) - assert(startPos, 'decode_scanString(..) called without start position') - local startChar = string.sub(s,startPos,startPos) - -- START SoniEx2 - -- PS: I don't think single quotes are valid JSON - assert(startChar == [["]] or startChar == [[']],'decode_scanString called for a non-string') - --assert(startPos, "String decoding failed: missing closing " .. startChar .. " for string at position " .. oldStart) - local t = {} - local i,j = startPos,startPos - while string.find(s, startChar, j+1) ~= j+1 do - local oldj = j - i,j = string.find(s, "\\.", j+1) - local x,y = string.find(s, startChar, oldj+1) - if not i or x < i then - i,j = x,y-1 - end - table.insert(t, string.sub(s, oldj+1, i-1)) - if string.sub(s, i, j) == "\\u" then - local a = string.sub(s,j+1,j+4) - j = j + 4 - local n = tonumber(a, 16) - assert(n, "String decoding failed: bad Unicode escape " .. a .. " at position " .. i .. " : " .. j) - -- math.floor(x/2^y) == lazy right shift - -- a % 2^b == bitwise_and(a, (2^b)-1) - -- 64 = 2^6 - -- 4096 = 2^12 (or 2^6 * 2^6) - local x - if n < 0x80 then - x = string.char(n % 0x80) - elseif n < 0x800 then - -- [110x xxxx] [10xx xxxx] - x = string.char(0xC0 + (math.floor(n/64) % 0x20), 0x80 + (n % 0x40)) - else - -- [1110 xxxx] [10xx xxxx] [10xx xxxx] - x = string.char(0xE0 + (math.floor(n/4096) % 0x10), 0x80 + (math.floor(n/64) % 0x40), 0x80 + (n % 0x40)) - end - table.insert(t, x) - else - table.insert(t, escapeSequences[string.sub(s, i, j)]) - end - end - table.insert(t,string.sub(j, j+1)) - assert(string.find(s, startChar, j+1), "String decoding failed: missing closing " .. startChar .. " at position " .. j .. "(for string at position " .. startPos .. ")") - return table.concat(t,""), j+2 - -- END SoniEx2 -end - ---- Scans a JSON string skipping all whitespace from the current start position. --- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. --- @param s The string being scanned --- @param startPos The starting position where we should begin removing whitespace. --- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string --- was reached. -function decode_scanWhitespace(s,startPos) - local whitespace=" \n\r\t" - local stringLen = string.len(s) - while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do - startPos = startPos + 1 - end - return startPos -end - ---- Encodes a string to be JSON-compatible. --- This just involves back-quoting inverted commas, back-quotes and newlines, I think ;-) --- @param s The string to return as a JSON encoded (i.e. backquoted string) --- @return The string appropriately escaped. - -local escapeList = { - ['"'] = '\\"', - ['\\'] = '\\\\', - ['/'] = '\\/', - ['\b'] = '\\b', - ['\f'] = '\\f', - ['\n'] = '\\n', - ['\r'] = '\\r', - ['\t'] = '\\t' -} - -function json_private.encodeString(s) - local s = tostring(s) - return s:gsub(".", function(c) return escapeList[c] end) -- SoniEx2: 5.0 compat -end - --- Determines whether the given Lua type is an array or a table / dictionary. --- We consider any table an array if it has indexes 1..n for its n items, and no --- other data in the table. --- I think this method is currently a little 'flaky', but can't think of a good way around it yet... --- @param t The table to evaluate as an array --- @return boolean, number True if the table can be represented as an array, false otherwise. If true, --- the second returned value is the maximum --- number of indexed elements in the array. -function isArray(t) - -- Next we count all the elements, ensuring that any non-indexed elements are not-encodable - -- (with the possible exception of 'n') - local maxIndex = 0 - for k,v in pairs(t) do - if (type(k)=='number' and math.floor(k)==k and 1<=k) then -- k,v is an indexed pair - if (not isEncodable(v)) then return false end -- All array elements must be encodable - maxIndex = math.max(maxIndex,k) - else - if (k=='n') then - if v ~= table.getn(t) then return false end -- False if n does not hold the number of elements - else -- Else of (k=='n') - if isEncodable(v) then return false end - end -- End of (k~='n') - end -- End of k,v not an indexed pair - end -- End of loop across all pairs - return true, maxIndex -end - ---- Determines whether the given Lua object / table / variable can be JSON encoded. The only --- types that are JSON encodable are: string, boolean, number, nil, table and json.null. --- In this implementation, all other types are ignored. --- @param o The object to examine. --- @return boolean True if the object should be JSON encoded, false if it should be ignored. -function isEncodable(o) - local t = type(o) - return (t=='string' or t=='boolean' or t=='number' or t=='nil' or t=='table') or (t=='function' and o==null) -end - -return json \ No newline at end of file +-- JSON4Lua: JSON encoding / decoding support for the Lua language. +-- json Module. +-- Author: Craig Mason-Jones +-- Homepage: http://json.luaforge.net/ +-- Version: 0.9.40 +-- This module is released under the MIT License (MIT). +-- +-- NOTE: This is only the decode functionality ripped out from JSON4Lua. +-- See: https://github.com/craigmj/json4lua + +module(..., package.seeall) + +local math = require('math') +local string = require("string") +local table = require("table") + +local base = _G + +-- Private functions +local decode_scanArray +local decode_scanComment +local decode_scanConstant +local decode_scanNumber +local decode_scanObject +local decode_scanString +local decode_scanWhitespace + +--- Decodes a JSON string and returns the decoded value as a Lua data structure / value. +-- @param s The string to scan. +-- @param [startPos] Optional starting position where the JSON string is located. Defaults to 1. +-- @param Lua object, number The object that was scanned, as a Lua table / string / number / boolean or nil, +-- and the position of the first character after +-- the scanned JSON object. +function decode(s, startPos) + startPos = startPos and startPos or 1 + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=string.len(s), 'Unterminated JSON encoded object found at position in [' .. s .. ']') + local curChar = string.sub(s,startPos,startPos) + -- Object + if curChar=='{' then + return decode_scanObject(s,startPos) + end + -- Array + if curChar=='[' then + return decode_scanArray(s,startPos) + end + -- Number + if string.find("+-0123456789.e", curChar, 1, true) then + return decode_scanNumber(s,startPos) + end + -- String + if curChar==[["]] or curChar==[[']] then + return decode_scanString(s,startPos) + end + if string.sub(s,startPos,startPos+1)=='/*' then + return decode(s, decode_scanComment(s,startPos)) + end + -- Otherwise, it must be a constant + return decode_scanConstant(s,startPos) +end + +--- The null function allows one to specify a null value in an associative array (which is otherwise +-- discarded if you set the value with 'nil' in Lua. Simply set t = { first=json.null } +function null() + return null -- so json.null() will also return null ;-) +end +----------------------------------------------------------------------------- +-- Internal, PRIVATE functions. +-- Following a Python-like convention, I have prefixed all these 'PRIVATE' +-- functions with an underscore. +----------------------------------------------------------------------------- + +--- Scans an array from JSON into a Lua object +-- startPos begins at the start of the array. +-- Returns the array and the next starting position +-- @param s The string being scanned. +-- @param startPos The starting position for the scan. +-- @return table, int The scanned array as a table, and the position of the next character to scan. +function decode_scanArray(s,startPos) + local array = {} -- The return value + local stringLen = string.len(s) + base.assert(string.sub(s,startPos,startPos)=='[','decode_scanArray called but array does not start at position ' .. startPos .. ' in string:\n'..s ) + startPos = startPos + 1 + -- Infinite loop for array elements + repeat + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen,'JSON String ended unexpectedly scanning array.') + local curChar = string.sub(s,startPos,startPos) + if (curChar==']') then + return array, startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + base.assert(startPos<=stringLen, 'JSON String ended unexpectedly scanning array.') + object, startPos = decode(s,startPos) + table.insert(array,object) + until false +end + +--- Scans a comment and discards the comment. +-- Returns the position of the next character following the comment. +-- @param string s The JSON string to scan. +-- @param int startPos The starting position of the comment +function decode_scanComment(s, startPos) + base.assert( string.sub(s,startPos,startPos+1)=='/*', "decode_scanComment called but comment does not start at position " .. startPos) + local endPos = string.find(s,'*/',startPos+2) + base.assert(endPos~=nil, "Unterminated comment in string at " .. startPos) + return endPos+2 +end + +--- Scans for given constants: true, false or null +-- Returns the appropriate Lua type, and the position of the next character to read. +-- @param s The string being scanned. +-- @param startPos The position in the string at which to start scanning. +-- @return object, int The object (true, false or nil) and the position at which the next character should be +-- scanned. +function decode_scanConstant(s, startPos) + local consts = { ["true"] = true, ["false"] = false, ["null"] = nil } + local constNames = {"true","false","null"} + + for i,k in base.pairs(constNames) do + --print ("[" .. string.sub(s,startPos, startPos + string.len(k) -1) .."]", k) + if string.sub(s,startPos, startPos + string.len(k) -1 )==k then + return consts[k], startPos + string.len(k) + end + end + base.assert(nil, 'Failed to scan constant from string ' .. s .. ' at starting position ' .. startPos) +end + +--- Scans a number from the JSON encoded string. +-- (in fact, also is able to scan numeric +- eqns, which is not +-- in the JSON spec.) +-- Returns the number, and the position of the next character +-- after the number. +-- @param s The string being scanned. +-- @param startPos The position at which to start scanning. +-- @return number, int The extracted number and the position of the next character to scan. +function decode_scanNumber(s,startPos) + local endPos = startPos+1 + local stringLen = string.len(s) + local acceptableChars = "+-0123456789.e" + while (string.find(acceptableChars, string.sub(s,endPos,endPos), 1, true) + and endPos<=stringLen + ) do + endPos = endPos + 1 + end + local stringValue = 'return ' .. string.sub(s,startPos, endPos-1) + local stringEval = base.loadstring(stringValue) + base.assert(stringEval, 'Failed to scan number [ ' .. stringValue .. '] in JSON string at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON object into a Lua object. +-- startPos begins at the start of the object. +-- Returns the object and the next starting position. +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return table, int The scanned object as a table and the position of the next character to scan. +function decode_scanObject(s,startPos) + local object = {} + local stringLen = string.len(s) + local key, value + base.assert(string.sub(s,startPos,startPos)=='{','decode_scanObject called but object does not start at position ' .. startPos .. ' in string:\n' .. s) + startPos = startPos + 1 + repeat + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly while scanning object.') + local curChar = string.sub(s,startPos,startPos) + if (curChar=='}') then + return object,startPos+1 + end + if (curChar==',') then + startPos = decode_scanWhitespace(s,startPos+1) + end + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly scanning object.') + -- Scan the key + key, startPos = decode(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + startPos = decode_scanWhitespace(s,startPos) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + base.assert(string.sub(s,startPos,startPos)==':','JSON object key-value assignment mal-formed at ' .. startPos) + startPos = decode_scanWhitespace(s,startPos+1) + base.assert(startPos<=stringLen, 'JSON string ended unexpectedly searching for value of key ' .. key) + value, startPos = decode(s,startPos) + object[key]=value + until false -- infinite loop while key-value pairs are found +end + +--- Scans a JSON string from the opening inverted comma or single quote to the +-- end of the string. +-- Returns the string extracted as a Lua string, +-- and the position of the next non-string character +-- (after the closing inverted comma or single quote). +-- @param s The string being scanned. +-- @param startPos The starting position of the scan. +-- @return string, int The extracted string as a Lua string, and the next character to parse. +function decode_scanString(s,startPos) + base.assert(startPos, 'decode_scanString(..) called without start position') + local startChar = string.sub(s,startPos,startPos) + base.assert(startChar==[[']] or startChar==[["]],'decode_scanString called for a non-string') + local escaped = false + local endPos = startPos + 1 + local bEnded = false + local stringLen = string.len(s) + repeat + local curChar = string.sub(s,endPos,endPos) + -- Character escaping is only used to escape the string delimiters + if not escaped then + if curChar==[[\]] then + escaped = true + else + bEnded = curChar==startChar + end + else + -- If we're escaped, we accept the current character come what may + escaped = false + end + endPos = endPos + 1 + base.assert(endPos <= stringLen+1, "String decoding failed: unterminated string at position " .. endPos) + until bEnded + local stringValue = 'return ' .. string.sub(s, startPos, endPos-1) + local stringEval = base.loadstring(stringValue) + base.assert(stringEval, 'Failed to load string [ ' .. stringValue .. '] in JSON4Lua.decode_scanString at position ' .. startPos .. ' : ' .. endPos) + return stringEval(), endPos +end + +--- Scans a JSON string skipping all whitespace from the current start position. +-- Returns the position of the first non-whitespace character, or nil if the whole end of string is reached. +-- @param s The string being scanned +-- @param startPos The starting position where we should begin removing whitespace. +-- @return int The first position where non-whitespace was encountered, or string.len(s)+1 if the end of string +-- was reached. +function decode_scanWhitespace(s,startPos) + local whitespace=" \n\r\t" + local stringLen = string.len(s) + while ( string.find(whitespace, string.sub(s,startPos,startPos), 1, true) and startPos <= stringLen) do + startPos = startPos + 1 + end + return startPos +end From f4834a515b5eef30de76dca2e572b77b28fd625f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 1 Jun 2016 15:26:23 +0200 Subject: [PATCH 28/53] snabb top: revert --yang dump. --- src/program/top/README | 2 - src/program/top/top.lua | 91 +---------------------------------------- 2 files changed, 2 insertions(+), 91 deletions(-) diff --git a/src/program/top/README b/src/program/top/README index 9fa3500804..ce815d6f89 100644 --- a/src/program/top/README +++ b/src/program/top/README @@ -5,8 +5,6 @@ Usage: Print usage information. -c, --counters Print counters of object by and exit. - -y, --yang - Print YANG model encoded in JSON and exit. Display realtime performance statistics for a running Snabb instance with . If is not supplied and there is only one Snabb instance, top will diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 317c0a2a77..8334d76be3 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -9,12 +9,10 @@ local shm = require("core.shm") local counter = require("core.counter") local S = require("syscall") local histogram = require("core.histogram") -local json = require("lib.json") -local macaddress = require("lib.macaddress") local usage = require("program.top.README_inc") local long_opts = { - help = "h", counters = "c", yang = "y" + help = "h", counters = "c" } function clearterm () io.write('\027[2J') end @@ -22,17 +20,14 @@ function clearterm () io.write('\027[2J') end function run (args) local opt = {} local object = nil - local yang = false function opt.h (arg) print(usage) main.exit(1) end function opt.c (arg) object = arg end - function opt.y () yang = true end - args = lib.dogetopt(args, opt, "hc:y", long_opts) + args = lib.dogetopt(args, opt, "hc:", long_opts) if #args > 1 then print(usage) main.exit(1) end local target_pid = select_snabb_instance(args[1]) if object then list_counters(target_pid, object) - elseif yang then dump_yang(target_pid) else top(target_pid) end ordered_exit(0) end @@ -76,88 +71,6 @@ function list_counters (pid, object) end end -function dump_yang (instance_pid) - local instance_tree = "//"..instance_pid - local interface_state = {} - local types = { [0x1000] = 'hardware', - [0x1001] = 'virtual', - [0x1002] = 'link' } - - for _, link in ipairs(shm.children(instance_tree.."/links")) do - local counters = instance_tree.."/counters/"..link - local statistics = {} - statistics['discontinuity-time'] = - totime(read_counter('discontinuity-time', counters)) - statistics['in-octets'] = - tostring64(read_counter('rxbytes', counters)) - statistics['out-octets'] = - tostring64(read_counter('txbytes', counters)) - statistics['out-discards'] = - truncate32(tonumber(read_counter('txdrop', counters))) - table.insert(interface_state, { name = link, - type = types[0x1002], - statistics = statistics }) - end - - for _, name in ipairs(shm.children(instance_tree.."/counters")) do - local counters = instance_tree.."/counters/"..name - local exists = {} - for _, c in ipairs(shm.children(counters)) do exists[c] = true end - local type = nil - if exists['type'] then - type = types[tonumber(read_counter('type', counters))] - end - if type then - local statistics = {} - statistics['discontinuity-time'] = - totime(read_counter('discontinuity-time', counters)) - for _, c in ipairs({'in-octets', 'in-unicast', - 'in-broadcast', 'in-multicast', - 'out-octets', 'out-unicast', - 'out-broadcast', 'out-multicast'}) do - if exists[c] then - statistics[c] = tostring64(read_counter(c, counters)) - end - end - for _, c in ipairs({'in-discards', 'out-discards'}) do - if exists[c] then - statistics[c] = truncate32(read_counter(c, counters)) - end - end - local interface = { name = name, type = type, statistics = statistics} - if exists['phys-address'] then - interface['phys-address'] = - tomac(read_counter('phys-address', counters)) - end - table.insert(interface_state, interface) - end - end - - print(json.encode({['interface-state'] = interface_state})) -end - -function tomac (n) - return tostring(macaddress:new(n)) -end - -function totime (n) - return os.date("!%FT%TZ", tonumber(n)) -end - -function truncate32 (n) - local box = ffi.new("union { uint64_t i64; uint32_t i32[2]; }") - box.i64 = n - if ffi.abi("le") then return tonumber(box.i32[0]) - elseif ffi.abi("be") then return tonumber(box.i32[1]) end -end - -function tostring64 (n) - assert(ffi.istype('uint64_t', n)) - local s = tostring(n) - return s:sub(1, #s - 3) -end - - function top (instance_pid) local instance_tree = "//"..instance_pid local counters = open_counters(instance_tree) From 793c394322294d8eb444244a5954315156392a37 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:35:41 +0200 Subject: [PATCH 29/53] snabbify statistics model. --- model.txt | 41 ++++++++++++++++++++++++++ src/apps/intel/intel_app.lua | 54 ++++++++++++++++------------------- src/apps/vhost/vhost_user.lua | 30 ++++++++++--------- src/core/link.h | 2 +- src/core/link.lua | 15 +++++----- 5 files changed, 89 insertions(+), 53 deletions(-) create mode 100644 model.txt diff --git a/model.txt b/model.txt new file mode 100644 index 0000000000..79df1e9847 --- /dev/null +++ b/model.txt @@ -0,0 +1,41 @@ +RFC7223 Snabb +======= ====== +name +description? +type type (~= identity) +enabled? +link-up-down-trap-enable? +admin-status +oper-status status (= enum) +last-change? +if-index +phys-address? macaddr (uint64) +higher-layer-if* +lower-layer-if* +speed? = +discontinuity-time dtime (seconds since epoch) +in-octets? rxbytes +in-unicast-pkts? +in-broadcast-pkts? rxbcast +in-multicast-pkts? +in-discards? +in-errors? +in-unknown-protos? +out-octets? txbytes +out-unicast-pkts? +out-broadcast-pkts? txbcast +out-multicast-pkts? +out-discards? +out-errors? + + +snabb/type: + +0x1000 Hardware interface +0x1001 Virtual interface + +snabb/stats: + +(tx/rx) - bcast ⊆ mcast ⊆ packets + +(tx/rx) - drop (all dropped packets) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 2645e2d22c..7b30e66ac0 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -57,28 +57,22 @@ function Intel82599:new (arg) self.zone = "intel" end if not self.stats.counters then - local counters = {} - local path = "/counters/"..conf.pciaddr.."/" - counters['type'] = counter.open(path..'type') - counters['discontinuity-time'] = counter.open(path..'discontinuity-time') - counters['in-octets'] = counter.open(path..'in-octets') - counters['in-multicast'] = counter.open(path..'in-multicast') - counters['in-broadcast'] = counter.open(path..'in-broadcast') - counters['in-discards'] = counter.open(path..'in-discards') - counters['out-octets'] = counter.open(path..'out-octets') - counters['out-multicast'] = counter.open(path..'out-multicast') - counters['out-broadcast'] = counter.open(path..'out-broadcast') - counters['out-discards'] = counter.open(path..'out-discards') - counter.set(counters['type'], 0x1000) -- Hardware interface - counter.set(counters['discontinuity-time'], C.get_unix_time()) + self.stats.path = "/counters/"..conf.pciaddr.."/" + self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) + self.stats.counters = {} + for _, name in ipairs( + {'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', + 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop'}) do + self.stats.counters[name] = counter.open(self.stats.path..name) + end + counter.set(self.stats.counters.type, 0x1000) -- Hardware interface + counter.set(self.stats.counters.dtime, C.get_unix_time()) if not conf.vmdq and conf.macaddr then - counters['phys-address'] = counter.open(path..'phys-address') - counter.set(counters['phys-address'], + self.stats.counters.macaddr = counter.open(self.stats.path..'macaddr') + counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end - self.stats.counters = counters - self.stats.path = path - self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) end return setmetatable(self, Intel82599) end @@ -117,7 +111,7 @@ function Intel82599:reconfig(arg) self.dev:reconfig(conf) if not self.dev.pf and conf.macaddr then - counter.set(self.stats.counters['phys-address'], + counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end end @@ -151,17 +145,17 @@ end -- Synchronize self.stats.register a and self.stats.counters. function Intel82599:sync_stats () - --- XXX - it is questionable if I choose the right register to counter - --- mapping local counters, register = self.stats.counters, self.stats.register - counter.set(counters['in-octets'], register.GORC64()) - counter.set(counters['in-multicast'], register.MPRC()) - counter.set(counters['in-broadcast'], register.BPRC()) - counter.set(counters['in-discards'], register.TPR() - register.GPRC()) - counter.set(counters['out-octets'], register.GOTC64()) - counter.set(counters['out-multicast'], register.MPTC()) - counter.set(counters['out-broadcast'], register.BPTC()) - counter.set(counters['out-discards'], register.TPT() - register.GPTC()) + counter.set(counters.rxbytes, register.GORC64()) + counter.set(counters.rxpackets, register.GPRC()) + counter.set(counters.rxmcast, register.MPRC()) + counter.set(counters.rxbcast, register.BPRC()) + counter.set(counters.rxdrop, register.TPR() - register.GPRC()) + counter.set(counters.txbytes, register.GOTC64()) + counter.set(counters.txpackets, register.GPTC()) + counter.set(counters.txmcast, register.MPTC()) + counter.set(counters.txbcast, register.BPTC()) + counter.set(counters.txdrop, register.TPT() - register.GPTC()) end -- Push packets from our 'rx' link onto the network. diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index 16936709bf..fe2711769e 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -29,6 +29,12 @@ assert(ffi.sizeof("struct vhost_user_msg") == 276, "ABI error") VhostUser = {} +local provided_counters = { + 'type', 'dtime', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxdrop', + 'txbytes', 'txpackets', 'txmcast' +} + function VhostUser:new (args) local o = { state = 'init', dev = nil, @@ -56,13 +62,11 @@ function VhostUser:new (args) end -- initialize counters self.counters = {} - for _, name in ipairs({'type', 'discontinuity-time', - 'in-octets', 'in-unicast', 'in-multicast', 'in-discards', - 'out-octets', 'out-unicast', 'out-multicast'}) do + for _, name in ipairs(provided_counters) do self.counters[name] = counter.open(name) end - counter.set(self.counters['type'], 0x1001) -- Virtual interface - counter.set(self.counters['discontinuity-time'], C.get_unix_time()) + counter.set(self.counters.type, 0x1001) -- Virtual interface + counter.set(self.counters.dtime, C.get_unix_time()) return self end @@ -101,21 +105,19 @@ function VhostUser:push () end function VhostUser:tx_callback (p) - counter.add(self.counters['out-octets'], packet.length(p)) - local mcast = ethernet:n_mcast(packet.data(p)) - counter.add(self.counters['out-multicast'], mcast) - counter.add(self.counters['out-unicast'], 1 - mcast) + counter.add(self.counters.txbytes, packet.length(p)) + counter.add(self.counters.txpackets) + counter.add(self.counters.txmcast, ethernet:n_mcast(packet.data(p))) end function VhostUser:rx_callback (p) - counter.add(self.counters['in-octets'], packet.length(p)) - local mcast = ethernet:n_mcast(packet.data(p)) - counter.add(self.counters['in-multicast'], mcast) - counter.add(self.counters['in-unicast'], 1 - mcast) + counter.add(self.counters.rxbytes, packet.length(p)) + counter.add(self.counters.rxpackets) + counter.add(self.counters.rxmcast, ethernet:n_mcast(packet.data(p))) end function VhostUser:rxdrop_callback (p) - counter.add(self.counters['in-discards']) + counter.add(self.counters.rxdrop) end -- Try to connect to QEMU. diff --git a/src/core/link.h b/src/core/link.h index c4ca2b45a5..f4caadaf91 100644 --- a/src/core/link.h +++ b/src/core/link.h @@ -9,7 +9,7 @@ struct link { // http://en.wikipedia.org/wiki/Circular_buffer struct packet *packets[LINK_RING_SIZE]; struct { - struct counter *txbytes, *rxbytes, *txpackets, *rxpackets, *txdrop; + struct counter *dtime, *txbytes, *rxbytes, *txpackets, *rxpackets, *txdrop; } stats; // Two cursors: // read: the next element to be read diff --git a/src/core/link.lua b/src/core/link.lua index 34e2ffc942..aacff1e741 100644 --- a/src/core/link.lua +++ b/src/core/link.lua @@ -21,23 +21,23 @@ local band = require("bit").band local size = C.LINK_RING_SIZE -- NB: Huge slow-down if this is not local max = C.LINK_MAX_PACKETS -local counternames = {"rxpackets", "txpackets", "rxbytes", "txbytes", "txdrop"} +local provided_counters = { + "dtime", "rxpackets", "rxbytes", "txpackets", "txbytes", "txdrop" +} function new (name) local r = shm.create("links/"..name, "struct link") - for _, c in ipairs(counternames) do + for _, c in ipairs(provided_counters) do r.stats[c] = counter.open("counters/"..name.."/"..c) end - counter.set(counter.open("counters/"..name.."/discontinuity-time"), - C.get_unix_time()) + counter.set(r.stats.dtime, C.get_unix_time()) return r end function free (r, name) - for _, c in ipairs(counternames) do + for _, c in ipairs(provided_counters) do counter.delete("counters/"..name.."/"..c) end - counter.delete("counters/"..name.."/discontinuity-time") shm.unmap(r) shm.unlink("links/"..name) end @@ -95,8 +95,7 @@ end function stats (r) local stats = {} - for _, c - in ipairs(counternames) do + for _, c in ipairs(provided_counters) do stats[c] = tonumber(counter.read(r.stats[c])) end return stats From 2b9ef2c82df610ec1470a5bb11615f5d8d0510bd Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:37:13 +0200 Subject: [PATCH 30/53] intel_app: add ifTable MIB related counters. --- model.txt | 47 +++++++++++++----------- src/apps/intel/intel_app.lua | 69 ++++++++++++++++++++++++++---------- 2 files changed, 77 insertions(+), 39 deletions(-) diff --git a/model.txt b/model.txt index 79df1e9847..8f4e128823 100644 --- a/model.txt +++ b/model.txt @@ -1,41 +1,48 @@ -RFC7223 Snabb -======= ====== +RFC7223 Snabb ifTable MIB +======= ===== =========== name description? type type (~= identity) enabled? link-up-down-trap-enable? admin-status -oper-status status (= enum) +oper-status status (= enum) ifOperStatus last-change? if-index -phys-address? macaddr (uint64) +phys-address? macaddr (uint64) ifPhysAddress higher-layer-if* lower-layer-if* -speed? = -discontinuity-time dtime (seconds since epoch) -in-octets? rxbytes +speed? speed ifSpeed +discontinuity-time dtime (seconds since epoch) ifCounterDiscontinuityTime +in-octets? rxbytes ifInOctets + rxpackets in-unicast-pkts? -in-broadcast-pkts? rxbcast -in-multicast-pkts? -in-discards? -in-errors? +in-broadcast-pkts? rxbcast ifInBroadcastPkts + rxmcast +in-multicast-pkts? ifInMulticastPkts +in-discards? rxdrop ifInDiscards +in-errors? rxerrors ifInErrors in-unknown-protos? -out-octets? txbytes +out-octets? txbytes ifOutOctets + txpackets out-unicast-pkts? -out-broadcast-pkts? txbcast -out-multicast-pkts? -out-discards? -out-errors? - +out-broadcast-pkts? txbcast ifOutBroadcastPkts + txmcast +out-multicast-pkts? ifOutMulticastPkts +out-discards? txdrop ifOutDiscards +out-errors? txerrors ifOutErrors + rxpackets + txpackets + rxmcast + txmcast + mtu ifMtu + promisc ifPromiscuousMode snabb/type: 0x1000 Hardware interface 0x1001 Virtual interface -snabb/stats: +snabb/bcast, mcast, packets: (tx/rx) - bcast ⊆ mcast ⊆ packets - -(tx/rx) - drop (all dropped packets) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 7b30e66ac0..623fb1a3a3 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -34,6 +34,11 @@ local function firsthole(t) end end +local provided_counters = { + 'type', 'dtime', 'mtu', 'speed', 'status', 'promisc', 'macaddr', + 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', 'rxerrors', + 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop', 'txerrors' +} -- Create an Intel82599 App for the device with 'pciaddress'. function Intel82599:new (arg) @@ -43,7 +48,9 @@ function Intel82599:new (arg) if conf.vmdq then if devices[conf.pciaddr] == nil then local pf = intel10g.new_pf(conf):open() - devices[conf.pciaddr] = {pf=pf, vflist={}, stats={register=pf.s}} + devices[conf.pciaddr] = { pf = pf, + vflist = {}, + stats = { s = pf.s, r = pf.r, qs = pf.qs } } end local dev = devices[conf.pciaddr] local poolnum = firsthole(dev.vflist)-1 @@ -53,23 +60,22 @@ function Intel82599:new (arg) self.stats = devices[conf.pciaddr].stats else self.dev = assert(intel10g.new_sf(conf):open(), "Can not open device.") - self.stats = { register = self.dev.s } + self.stats = { s = self.dev.s, r = self.dev.r, qs = self.dev.qs } self.zone = "intel" end if not self.stats.counters then self.stats.path = "/counters/"..conf.pciaddr.."/" self.stats.sync_timer = lib.timer(0.001, 'repeating', engine.now) self.stats.counters = {} - for _, name in ipairs( - {'type', 'dtime', - 'rxbytes', 'rxpackets', 'rxmcast', 'rxbcast', 'rxdrop', - 'txbytes', 'txpackets', 'txmcast', 'txbcast', 'txdrop'}) do + for _, name in ipairs(provided_counters) do self.stats.counters[name] = counter.open(self.stats.path..name) end counter.set(self.stats.counters.type, 0x1000) -- Hardware interface counter.set(self.stats.counters.dtime, C.get_unix_time()) + counter.set(self.stats.counters.mtu, self.dev.mtu) + counter.set(self.stats.counters.speed, 10000000) -- 10 Gbits + counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then - self.stats.counters.macaddr = counter.open(self.stats.path..'macaddr') counter.set(self.stats.counters.macaddr, macaddress:new(conf.macaddr):int()) end @@ -143,19 +149,37 @@ function Intel82599:add_receive_buffers () end end --- Synchronize self.stats.register a and self.stats.counters. +-- Synchronize self.stats.s/r a and self.stats.counters. +local link_up_mask = lib.bits{Link_up=30} +local promisc_mask = lib.bits{UPE=9} function Intel82599:sync_stats () - local counters, register = self.stats.counters, self.stats.register - counter.set(counters.rxbytes, register.GORC64()) - counter.set(counters.rxpackets, register.GPRC()) - counter.set(counters.rxmcast, register.MPRC()) - counter.set(counters.rxbcast, register.BPRC()) - counter.set(counters.rxdrop, register.TPR() - register.GPRC()) - counter.set(counters.txbytes, register.GOTC64()) - counter.set(counters.txpackets, register.GPTC()) - counter.set(counters.txmcast, register.MPTC()) - counter.set(counters.txbcast, register.BPTC()) - counter.set(counters.txdrop, register.TPT() - register.GPTC()) + local counters = self.stats.counters + local s, r, qs = self.stats.s, self.stats.r, self.stats.qs + counter.set(counters.rxbytes, s.GORC64()) + counter.set(counters.rxpackets, s.GPRC()) + local mprc, bprc = s.MPRC(), s.BPRC() + counter.set(counters.rxmcast, mprc + bprc) + counter.set(counters.rxbcast, bprc) + -- The RX receive drop counts are only available through the RX stats + -- register. We only read stats register #0 here. + counter.set(counters.rxdrop, qs.QPRDC[0]()) + counter.set(counters.rxerrors, s.TPR() - s.GPRC()) + counter.set(counters.txbytes, s.GOTC64()) + counter.set(counters.txpackets, s.GPTC()) + local mptc, bptc = s.MPTC(), s.BPTC() + counter.set(counters.txmcast, mptc + bptc) + counter.set(counters.txbcast, bptc) + counter.set(counters.txerrors, s.TPT() - s.GPTC()) + if bit.band(r.LINKS(), link_up_mask) == link_up_mask then + counter.set(counters.status, 1) -- Up + else + counter.set(counters.status, 2) -- Down + end + if bit.band(r.FCTRL(), promisc_mask) ~= 0ULL then + counter.set(counters.promisc, 1) -- True + else + counter.set(counters.promisc, 2) -- False + end end -- Push packets from our 'rx' link onto the network. @@ -163,6 +187,13 @@ function Intel82599:push () local l = self.input.rx if l == nil then return end while not empty(l) and self.dev:can_transmit() do + -- We must not send packets that are bigger than the MTU. This + -- check is currently disabled to satisfy some selftests until + -- agreement on this strategy is reached. + -- if p.length > self.dev.mtu then + -- counter.add(self.stats.counters.txdrop) + -- packet.free(p) + -- else do local p = receive(l) self.dev:transmit(p) --packet.deref(p) From e9f5dda5452ed2bcb1f1394cabf8b1f604357e15 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 17:37:34 +0200 Subject: [PATCH 31/53] lib.macaddress: add bytes method. --- src/lib/macaddress.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index e55bdcfb3c..0508df17d6 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -46,6 +46,10 @@ function mac_mt:int () return self.bits end +function mac_mt:bytes () + return self.bytes +end + function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From 6b12e9c3e3e45ce2f60bfac0cf399ba555a03f0f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 2 Jun 2016 18:45:40 +0200 Subject: [PATCH 32/53] spin out SNMP code from apps.intel.intel10 to lib.ipc.shmem.iftable_mib. --- src/apps/intel/intel10g.lua | 188 +----------------------------- src/lib/ipc/shmem/iftable_mib.lua | 154 ++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 182 deletions(-) create mode 100644 src/lib/ipc/shmem/iftable_mib.lua diff --git a/src/apps/intel/intel10g.lua b/src/apps/intel/intel10g.lua index 51e2d640fa..b5ec3e3ef3 100644 --- a/src/apps/intel/intel10g.lua +++ b/src/apps/intel/intel10g.lua @@ -6,7 +6,8 @@ --- affordable (~$400) network cards made by Intel and others. --- --- You will need to familiarize yourself with the excellent [data ---- sheet]() to understand this module. +--- sheet](http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82599-10-gbe-controller-datasheet.pdf) +--- to understand this module. module(...,package.seeall) @@ -17,7 +18,6 @@ local pci = require("lib.hardware.pci") local register = require("lib.hardware.register") local index_set = require("lib.index_set") local macaddress = require("lib.macaddress") -local mib = require("lib.ipc.shmem.mib") local timer = require("core.timer") local bits, bitset = lib.bits, lib.bitset @@ -49,10 +49,6 @@ local default = { -- The default MTU allows for an IP packet of a total size of 9000 -- bytes without VLAN tagging. mtu = 9014, - - snmp = { - status_timer = 5, -- Interval for IF status check and MIB update - } } local function pass (...) return ... end @@ -79,7 +75,6 @@ function new_sf (conf) rdh = 0, -- Cache of receive head (RDH) register rdt = 0, -- Cache of receive tail (RDT) register rxnext = 0, -- Index of next buffer to receive - snmp = conf.snmp, } return setmetatable(dev, M_sf) end @@ -118,9 +113,6 @@ end --- See data sheet section 4.6.3 "Initialization Sequence." function M_sf:init () - if self.snmp then - self:init_snmp() - end self:init_dma_memory() self.redos = 0 @@ -186,156 +178,6 @@ do end end -function M_sf:init_snmp () - -- Rudimentary population of a row in the ifTable MIB. Allocation - -- of the ifIndex is delegated to the SNMP agent via the name of - -- the interface in ifDescr (currently the PCI address). - local ifTable = mib:new({ directory = self.snmp.directory or nil, - filename = self.pciaddress }) - self.snmp.ifTable = ifTable - -- ifTable - ifTable:register('ifDescr', 'OctetStr', self.pciaddress) - ifTable:register('ifType', 'Integer32', 6) -- ethernetCsmacd - ifTable:register('ifMtu', 'Integer32', self.mtu) - ifTable:register('ifSpeed', 'Gauge32', 10000000) - - -- After a reset of the NIC, the "native" MAC address is copied to - -- the receive address register #0 from the EEPROM - local ral, rah = self.r.RAL[0](), self.r.RAH[0]() - assert(bit.band(rah, bits({ AV = 31 })) == bits({ AV = 31 }), - "MAC address on "..self.pciaddress.." is not valid ") - local mac = ffi.new("struct { uint32_t lo; uint16_t hi; }") - mac.lo = ral - mac.hi = bit.band(rah, 0xFFFF) - ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }, - ffi.string(mac, 6)) - ifTable:register('ifAdminStatus', 'Integer32', 1) -- up - ifTable:register('ifOperStatus', 'Integer32', 2) -- down - ifTable:register('ifLastChange', 'TimeTicks', 0) - ifTable:register('_X_ifLastChange_TicksBase', 'Counter64', C.get_unix_time()) - ifTable:register('ifInOctets', 'Counter32', 0) - ifTable:register('ifInUcastPkts', 'Counter32', 0) - ifTable:register('ifInDiscards', 'Counter32', 0) - ifTable:register('ifInErrors', 'Counter32', 0) -- TBD - ifTable:register('ifInUnknownProtos', 'Counter32', 0) -- TBD - ifTable:register('ifOutOctets', 'Counter32', 0) - ifTable:register('ifOutUcastPkts', 'Counter32', 0) - ifTable:register('ifOutDiscards', 'Counter32', 0) - ifTable:register('ifOutErrors', 'Counter32', 0) -- TBD - -- ifXTable - ifTable:register('ifName', { type = 'OctetStr', length = 255 }, - self.pciaddress) - ifTable:register('ifInMulticastPkts', 'Counter32', 0) - ifTable:register('ifInBroadcastPkts', 'Counter32', 0) - ifTable:register('ifOutMulticastPkts', 'Counter32', 0) - ifTable:register('ifOutBroadcastPkts', 'Counter32', 0) - ifTable:register('ifHCInOctets', 'Counter64', 0) - ifTable:register('ifHCInUcastPkts', 'Counter64', 0) - ifTable:register('ifHCInMulticastPkts', 'Counter64', 0) - ifTable:register('ifHCInBroadcastPkts', 'Counter64', 0) - ifTable:register('ifHCOutOctets', 'Counter64', 0) - ifTable:register('ifHCOutUcastPkts', 'Counter64', 0) - ifTable:register('ifHCOutMulticastPkts', 'Counter64', 0) - ifTable:register('ifHCOutBroadcastPkts', 'Counter64', 0) - ifTable:register('ifLinkUpDownTrapEnable', 'Integer32', 2) -- disabled - ifTable:register('ifHighSpeed', 'Gauge32', 10000) - ifTable:register('ifPromiscuousMode', 'Integer32', 2) -- false - ifTable:register('ifConnectorPresent', 'Integer32', 1) -- true - ifTable:register('ifAlias', { type = 'OctetStr', length = 64 }, - self.pciaddress) -- TBD add description - ifTable:register('ifCounterDiscontinuityTime', 'TimeTicks', 0) -- TBD - ifTable:register('_X_ifCounterDiscontinuityTime', 'Counter64', 0) -- TBD - - --- Create a timer to periodically check the interface status. - --- Static variables are used in the timer function to avoid - --- garbage. - local status = { [1] = 'up', [2] = 'down' } - local mask = bits{Link_up=30} - local promisc = bits({UPE = 9}) - -- Pre-allocate storage for the results of register-reads - local r = { - in_mcast_pkts = { r = self.s.MPRC }, - in_bcast_pkts = { r = self.s.BPRC }, - in_pkts = { r = self.s.GPRC }, - in_octets64 = { r = self.s.GORC64 }, - - out_mcast_pkts = { r = self.s.MPTC }, - out_bcast_pkts = { r = self.s.BPTC }, - out_pkts = { r = self.s.GPTC }, - out_octets64 = { r = self.s.GOTC64 }, - } - local r_keys = {} - for k, _ in pairs(r) do - table.insert(r_keys, k) - r[k].v = ffi.new("uint64_t [1]") - end - local function read_registers() - for _, k in ipairs(r_keys) do - r[k].v[0] = r[k].r() - end - end - self.logger = lib.logger_new({ module = 'intel10g' }) - local t = timer.new("Interface "..self.pciaddress.." status checker", - function(t) - local old = ifTable:get('ifOperStatus') - local new = 1 - if band(self.r.LINKS(), mask) ~= mask then - new = 2 - end - if old ~= new then - self.logger:log("Interface "..self.pciaddress.. - " status change: "..status[old].. - " => "..status[new]) - ifTable:set('ifOperStatus', new) - ifTable:set('ifLastChange', 0) - ifTable:set('_X_ifLastChange_TicksBase', - C.get_unix_time()) - end - - ifTable:set('ifPromiscuousMode', - (bit.band(self.r.FCTRL(), promisc) ~= 0ULL - and 1) or 2) - -- Update counters - read_registers() - ifTable:set('ifHCInMulticastPkts', r.in_mcast_pkts.v[0]) - ifTable:set('ifInMulticastPkts', r.in_mcast_pkts.v[0]) - ifTable:set('ifHCInBroadcastPkts', r.in_bcast_pkts.v[0]) - ifTable:set('ifInBroadcastPkts', r.in_bcast_pkts.v[0]) - local in_ucast_pkts = r.in_pkts.v[0] - r.in_bcast_pkts.v[0] - - r.in_mcast_pkts.v[0] - ifTable:set('ifHCInUcastPkts', in_ucast_pkts) - ifTable:set('ifInUcastPkts', in_ucast_pkts) - ifTable:set('ifHCInOctets', r.in_octets64.v[0]) - ifTable:set('ifInOctets', r.in_octets64.v[0]) - - ifTable:set('ifHCOutMulticastPkts', r.out_mcast_pkts.v[0]) - ifTable:set('ifOutMulticastPkts', r.out_mcast_pkts.v[0]) - ifTable:set('ifHCOutBroadcastPkts', r.out_bcast_pkts.v[0]) - ifTable:set('ifOutBroadcastPkts', r.out_bcast_pkts.v[0]) - local out_ucast_pkts = r.out_pkts.v[0] - r.out_bcast_pkts.v[0] - - r.out_mcast_pkts.v[0] - ifTable:set('ifHCOutUcastPkts', out_ucast_pkts) - ifTable:set('ifOutUcastPkts', out_ucast_pkts) - ifTable:set('ifHCOutOctets', r.out_octets64.v[0]) - ifTable:set('ifOutOctets', r.out_octets64.v[0]) - - -- The RX receive drop counts are only - -- available through the RX stats register. - -- We only read stats register #0 here. See comment - -- in init_statistics() - ifTable:set('ifInDiscards', self.qs.QPRDC[0]()) - - ifTable:set('ifInErrors', self.s.CRCERRS() + - self.s.ILLERRC() + self.s.ERRBC() + - self.s.RUC() + self.s.RFC() + - self.s.ROC() + self.s.RJC()) - end, - 1e9 * (self.snmp.status_timer or - default.snmp.status_timer), 'repeating') - timer.activate(t) - return self -end - function M_sf:global_reset () local reset = bits{LinkReset=3, DeviceReset=26} self.r.CTRL(reset) @@ -448,22 +290,10 @@ end local txdesc_flags = bits{ifcs=25, dext=29, dtyp0=20, dtyp1=21, eop=24} function M_sf:transmit (p) - -- We must not send packets that are bigger than the MTU. This - -- check is currently disabled to satisfy some selftests until - -- agreement on this strategy is reached. - -- if p.length > self.mtu then - -- if self.snmp then - -- local errors = self.snmp.ifTable:ptr('ifOutDiscards') - -- errors[0] = errors[0] + 1 - -- end - -- packet.free(p) - -- else - do - self.txdesc[self.tdt].address = memory.virtual_to_physical(p.data) - self.txdesc[self.tdt].options = bor(p.length, txdesc_flags, lshift(p.length+0ULL, 46)) - self.txpackets[self.tdt] = p - self.tdt = band(self.tdt + 1, num_descriptors - 1) - end + self.txdesc[self.tdt].address = memory.virtual_to_physical(p.data) + self.txdesc[self.tdt].options = bor(p.length, txdesc_flags, lshift(p.length+0ULL, 46)) + self.txpackets[self.tdt] = p + self.tdt = band(self.tdt + 1, num_descriptors - 1) end function M_sf:sync_transmit () @@ -649,7 +479,6 @@ function new_pf (conf) mac_set = index_set:new(127, "MAC address table"), vlan_set = index_set:new(64, "VLAN Filter table"), mirror_set = index_set:new(4, "Mirror pool table"), - snmp = conf.snmp, } return setmetatable(dev, M_pf) end @@ -675,9 +504,6 @@ function M_pf:close() end function M_pf:init () - if self.snmp then - self:init_snmp() - end self.redos = 0 local mask = bits{Link_up=30} for i = 1, 100 do @@ -703,7 +529,6 @@ function M_pf:init () return self end -M_pf.init_snmp = M_sf.init_snmp M_pf.global_reset = M_sf.global_reset M_pf.disable_interrupts = M_sf.disable_interrupts M_pf.set_receive_descriptors = pass @@ -784,7 +609,6 @@ function M_pf:new_vf (poolnum) base = self.base, -- mmap()ed register file s = self.s, -- Statistics registers mtu = self.mtu, - snmp = self.snmp, -- and others are our own r = {}, -- Configuration registers poolnum = poolnum, diff --git a/src/lib/ipc/shmem/iftable_mib.lua b/src/lib/ipc/shmem/iftable_mib.lua new file mode 100644 index 0000000000..0f09aef0aa --- /dev/null +++ b/src/lib/ipc/shmem/iftable_mib.lua @@ -0,0 +1,154 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) +local mib = require("lib.ipc.shmem.mib") +local counter = require("core.counter") +local macaddress = require("lib.macaddress") +local ffi = require("ffi") + +local iftypes = { + [0x1000] = 6, -- ethernetCsmacd + [0x1001] = 53, -- propVirtual +} + +function init_snmp (name, counters, directory, interval) + -- Rudimentary population of a row in the ifTable MIB. Allocation + -- of the ifIndex is delegated to the SNMP agent via the name of + -- the interface in ifDescr. + local ifTable = mib:new({ directory = directory or nil, + filename = name }) + -- ifTable + ifTable:register('ifDescr', 'OctetStr', name) + ifTable:register('ifType', 'Integer32') + if counters.type then + ifTable:set('ifType', iftypes[counter.read(counters.type)] or 1) -- other + end + ifTable:register('ifMtu', 'Integer32') + if counters.mtu then + ifTable:set('ifMtu', counter.read(counters.mtu)) + end + ifTable:register('ifSpeed', 'Gauge32') + ifTable:register('ifHighSpeed', 'Gauge32') + if counters.speed then + ifTable:set('ifSpeed', counter.read(counters.speed)) + ifTable:set('ifHighSpeed', counter.read(counters.speed) / 1000) + end + ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }) + if counters.macaddr then + local mac = macaddress:new(counter.read(counters.macaddr)) + ifTable:set('ifPhysAddress', ffi.string(mac:bytes(), 6)) + end + ifTable:register('ifAdminStatus', 'Integer32', 1) -- up + ifTable:register('ifOperStatus', 'Integer32', 2) -- down + ifTable:register('ifLastChange', 'TimeTicks', 0) + ifTable:register('_X_ifLastChange_TicksBase', 'Counter64', + C.get_unix_time()) + ifTable:register('ifInOctets', 'Counter32', 0) + ifTable:register('ifInUcastPkts', 'Counter32', 0) + ifTable:register('ifInDiscards', 'Counter32', 0) + ifTable:register('ifInErrors', 'Counter32', 0) -- TBD + ifTable:register('ifInUnknownProtos', 'Counter32', 0) -- TBD + ifTable:register('ifOutOctets', 'Counter32', 0) + ifTable:register('ifOutUcastPkts', 'Counter32', 0) + ifTable:register('ifOutDiscards', 'Counter32', 0) + ifTable:register('ifOutErrors', 'Counter32', 0) -- TBD + -- ifXTable + ifTable:register('ifName', { type = 'OctetStr', length = 255 }, name) + ifTable:register('ifInMulticastPkts', 'Counter32', 0) + ifTable:register('ifInBroadcastPkts', 'Counter32', 0) + ifTable:register('ifOutMulticastPkts', 'Counter32', 0) + ifTable:register('ifOutBroadcastPkts', 'Counter32', 0) + ifTable:register('ifHCInOctets', 'Counter64', 0) + ifTable:register('ifHCInUcastPkts', 'Counter64', 0) + ifTable:register('ifHCInMulticastPkts', 'Counter64', 0) + ifTable:register('ifHCInBroadcastPkts', 'Counter64', 0) + ifTable:register('ifHCOutOctets', 'Counter64', 0) + ifTable:register('ifHCOutUcastPkts', 'Counter64', 0) + ifTable:register('ifHCOutMulticastPkts', 'Counter64', 0) + ifTable:register('ifHCOutBroadcastPkts', 'Counter64', 0) + ifTable:register('ifLinkUpDownTrapEnable', 'Integer32', 2) -- disabled + ifTable:register('ifPromiscuousMode', 'Integer32', 2) -- false + ifTable:register('ifConnectorPresent', 'Integer32', 1) -- true + ifTable:register('ifAlias', { type = 'OctetStr', length = 64 }, + name) -- TBD add description + ifTable:register('ifCounterDiscontinuityTime', 'TimeTicks', 0) + ifTable:register('_X_ifCounterDiscontinuityTime', 'Counter64') + if counters.dtime then + ifTable:set('_X_ifCounterDiscontinuityTime', counter.read(counters.dtime)) + end + + local logger = lib.logger_new({ module = 'iftable_mib' }) + local function t () + local old, new + if counters.status then + old = ifTable:get('ifOperStatus') + new = counter.read(counters.status) + else + new = 1 + end + if old ~= new then + logger:log("Interface "..name.. + " status change: "..status[old].." => "..status[new]) + ifTable:set('ifOperStatus', new) + ifTable:set('ifLastChange', 0) + ifTable:set('_X_ifLastChange_TicksBase', C.get_unix_time()) + end + + if counters.promisc then + ifTable:set('ifPromiscuousMode', counter.read(counters.promisc)) + end + -- Update counters + if counters.rxpackets and counters.rxmcast and counters.rxbcast then + local rxbcast = counter.read(counters.rxbcast) + local rxmcast = counter.read(counters.rxmcast) + local rxpackets = counter.read(counters.rxpackets) + local inMcast = rxmcast - rxbcast + local inUcast = rxpackets - rxmcast + ifTable:set('ifHCInMulticastPkts', inMcast) + ifTable:set('ifInMulticastPkts', inMcast) + ifTable:set('ifHCInBroadcastPkts', rxbcast) + ifTable:set('ifInBroadcastPkts', rxbcast) + ifTable:set('ifHCInUcastPkts', inUcast) + ifTable:set('ifInUcastPkts', inUcast) + end + if counters.rxbytes then + local rxbytes = counter.read(counters.rxbytes) + ifTable:set('ifHCInOctets', rxbytes) + ifTable:set('ifInOctets', rxbytes) + end + if counters.rxdrop then + ifTable:set('ifInDiscards', counter.read(counters.rxdrop)) + end + if counters.rxerrors then + ifTable:set('ifInErrors', counter.read(counters.rxerrors)) + end + if counters.txpackets and counters.txmcast and counters.txbcast then + local txbcast = counter.read(counters.txbcast) + local txmcast = counter.read(counters.txmcast) + local txpackets = counter.read(counters.txpackets) + local outMcast = txmcast - txbcast + local outUcast = txpackets - txmcast + ifTable:set('ifHCOutMulticastPkts', outMcast) + ifTable:set('ifOutMulticastPkts', outMcast) + ifTable:set('ifHCOutBroadcastPkts', txbcast) + ifTable:set('ifOutBroadcastPkts', txbcast) + ifTable:set('ifHCOutUcastPkts', outUcast) + ifTable:set('ifOutUcastPkts', outUcast) + end + if counters.txbytes then + local txbytes = counter.read(counters.txbytes) + ifTable:set('ifHCOutOctets', txbytes) + ifTable:set('ifOutOctets', txbytes) + end + if counters.txdrop then + ifTable:set('ifOutDiscards', counter.read(counters.txdrop)) + end + if counters.txerrors then + ifTable:set('ifOutErrors', counter.read(counters.txerrors)) + end + end + local t = timer.new("Interface "..name.." status checker", + t, 1e9 * (interval or 5), 'repeating') + timer.activate(t) + return t +end From 7c697b6775ee8bae9d488710c806eb62a3871a1a Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 3 Jun 2016 18:07:55 +0200 Subject: [PATCH 33/53] main: fork into worker and supervisor; perform clean up. --- src/core/main.lua | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index eac20ec884..d8998eb701 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -10,10 +10,11 @@ local STP = require("lib.lua.StackTracePlus") local ffi = require("ffi") local zone = require("jit.zone") local lib = require("core.lib") +local shm = require("core.shm") local C = ffi.C -- Load ljsyscall early to help detect conflicts -- (e.g. FFI type name conflict between Snabb and ljsyscall) -require("syscall") +local S = require("syscall") require("lib.lua.strict") require("lib.lua.class") @@ -44,10 +45,30 @@ function main () print("unsupported program: "..program:gsub("_", "-")) usage(1) else - require(modulename(program)).run(args) + -- Fork into worker process and supervisor + local worker_pid = S.fork() + if worker_pid == 0 then + -- Worker + S.prctl("set_pdeathsig", "hup") + require(modulename(program)).run(args) + else + -- Supervisor + local exit_signals = "hup, int, quit, term, chld" + local signalfd = S.signalfd(exit_signals) + S.sigprocmask("block", exit_signals) + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + if not signals[1].chld then S.kill(worker_pid, "kill") end + shutdown(S.getpid()) + end end end +-- Cleanup after worker process. +function shutdown (pid) + shm.unlink("//"..pid) +end + -- Take the program name from the first argument, unless the first -- argument is "snabb", in which case pop it off, handle any options -- passed to snabb itself, and use the next argument. From be09fe5ca54ef27cc02d0b3504e247aa121a3d36 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 16:04:55 +0200 Subject: [PATCH 34/53] main: send HUP instead of KILL. --- src/core/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index d8998eb701..7e7d4e2e0a 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -58,7 +58,7 @@ function main () S.sigprocmask("block", exit_signals) local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) - if not signals[1].chld then S.kill(worker_pid, "kill") end + if not signals[1].chld then S.kill(worker_pid, "hup") end shutdown(S.getpid()) end end From 3db174bb05fd7b4947cf61186c898a9d669494fd Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 16:35:56 +0200 Subject: [PATCH 35/53] main: return correct exit code. --- src/core/main.lua | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index 7e7d4e2e0a..bde7574b39 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -58,8 +58,17 @@ function main () S.sigprocmask("block", exit_signals) local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) - if not signals[1].chld then S.kill(worker_pid, "hup") end + local exit_status + if signals[1].chld then + local _, _, worker = S.waitpid(worker_pid) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end + else + S.kill(worker_pid, "hup") + exit_status = 128 + signals[1].signo + end shutdown(S.getpid()) + os.exit(exit_status) end end end From ce2f2206872f881ee0a1e22c13a07261476a3803 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:00:30 +0200 Subject: [PATCH 36/53] main: supervise xpcall(main, hanlder). --- src/core/main.lua | 62 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index bde7574b39..cb312d2cb4 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -45,39 +45,10 @@ function main () print("unsupported program: "..program:gsub("_", "-")) usage(1) else - -- Fork into worker process and supervisor - local worker_pid = S.fork() - if worker_pid == 0 then - -- Worker - S.prctl("set_pdeathsig", "hup") - require(modulename(program)).run(args) - else - -- Supervisor - local exit_signals = "hup, int, quit, term, chld" - local signalfd = S.signalfd(exit_signals) - S.sigprocmask("block", exit_signals) - local signals, err = S.util.signalfd_read(signalfd) - assert(signals, tostring(err)) - local exit_status - if signals[1].chld then - local _, _, worker = S.waitpid(worker_pid) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end - else - S.kill(worker_pid, "hup") - exit_status = 128 + signals[1].signo - end - shutdown(S.getpid()) - os.exit(exit_status) - end + require(modulename(program)).run(args) end end --- Cleanup after worker process. -function shutdown (pid) - shm.unlink("//"..pid) -end - -- Take the program name from the first argument, unless the first -- argument is "snabb", in which case pop it off, handle any options -- passed to snabb itself, and use the next argument. @@ -160,6 +131,11 @@ function handler (reason) os.exit(1) end +-- Cleanup after Snabb process. +function shutdown (pid) + shm.unlink("//"..pid) +end + function selftest () print("selftest") assert(programname("/bin/snabb-1.0") == "snabb", @@ -184,4 +160,28 @@ function selftest () "Incorrect program name selected") end -xpcall(main, handler) +-- Fork into worker process and supervisor +local worker_pid = S.fork() +if worker_pid == 0 then + -- Worker + S.prctl("set_pdeathsig", "hup") + xpcall(main, handler) +else + -- Supervisor + local exit_signals = "hup, int, quit, term, chld" + local signalfd = S.signalfd(exit_signals) + S.sigprocmask("block", exit_signals) + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + local exit_status + if signals[1].chld then + local _, _, worker = S.waitpid(worker_pid) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end + else + S.kill(worker_pid, "hup") + exit_status = 128 + signals[1].signo + end + shutdown(S.getpid()) + os.exit(exit_status) +end From 4950cbb7d4beb3c69bb81a4f6412b9951bc583fb Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:03:27 +0200 Subject: [PATCH 37/53] main: check status of S.waitpid. --- src/core/main.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index cb312d2cb4..7369639993 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -175,7 +175,8 @@ else assert(signals, tostring(err)) local exit_status if signals[1].chld then - local _, _, worker = S.waitpid(worker_pid) + local status, err, worker = S.waitpid(worker_pid) + assert(status, tostring(err)) if worker.WIFEXITED then exit_status = worker.EXITSTATUS else exit_status = 128 + worker.WTERMSIG end else From 28c568c4526f1848f25078494dc694f67610aab1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:06:31 +0200 Subject: [PATCH 38/53] main: cleanup after worker_pid. --- src/core/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index 7369639993..0a27c076e5 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -183,6 +183,6 @@ else S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo end - shutdown(S.getpid()) + shutdown(worker_pid) os.exit(exit_status) end From 4af7091f01939ef462480b39567eaa366fbb236f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 6 Jun 2016 17:19:41 +0200 Subject: [PATCH 39/53] Remove obsolete `snabb gc'. --- src/program/gc/README | 7 ------- src/program/gc/README.inc | 1 - src/program/gc/gc.lua | 29 ----------------------------- 3 files changed, 37 deletions(-) delete mode 100644 src/program/gc/README delete mode 120000 src/program/gc/README.inc delete mode 100644 src/program/gc/gc.lua diff --git a/src/program/gc/README b/src/program/gc/README deleted file mode 100644 index 703a1ef95b..0000000000 --- a/src/program/gc/README +++ /dev/null @@ -1,7 +0,0 @@ -Usage: - gc [OPTIONS] - - -h, --help - Print usage information. - -Remove stale runtime files created by Snabb instances. diff --git a/src/program/gc/README.inc b/src/program/gc/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/gc/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/gc/gc.lua b/src/program/gc/gc.lua deleted file mode 100644 index d3de0a12e1..0000000000 --- a/src/program/gc/gc.lua +++ /dev/null @@ -1,29 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - -module(..., package.seeall) - -local lib = require("core.lib") -local shm = require("core.shm") -local syscall = require("syscall") -local usage = require("program.gc.README_inc") - -local long_opts = { - help = "h" -} - -function run (args) - local opt = {} - function opt.h (arg) print(usage) main.exit(1) end - args = lib.dogetopt(args, opt, "h", long_opts) - - if #args > 0 then print(usage) main.exit(1) end - - -- Unlink stale snabb resources. - for _, pid in ipairs(shm.children("//")) do - if not syscall.kill(tonumber(pid), 0) then - shm.unlink("//"..pid) - end - end - -- Unlink own resource - shm.unlink("//"..syscall.getpid()) -end From 1c9f58585f7480df16ce3ffe532a1d99286ff1ea Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 15:47:49 +0200 Subject: [PATCH 40/53] Move model.txt to src/doc/statistics.md. --- model.txt | 48 ----------------------------------- src/doc/statistics.md | 58 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 48 deletions(-) delete mode 100644 model.txt create mode 100644 src/doc/statistics.md diff --git a/model.txt b/model.txt deleted file mode 100644 index 8f4e128823..0000000000 --- a/model.txt +++ /dev/null @@ -1,48 +0,0 @@ -RFC7223 Snabb ifTable MIB -======= ===== =========== -name -description? -type type (~= identity) -enabled? -link-up-down-trap-enable? -admin-status -oper-status status (= enum) ifOperStatus -last-change? -if-index -phys-address? macaddr (uint64) ifPhysAddress -higher-layer-if* -lower-layer-if* -speed? speed ifSpeed -discontinuity-time dtime (seconds since epoch) ifCounterDiscontinuityTime -in-octets? rxbytes ifInOctets - rxpackets -in-unicast-pkts? -in-broadcast-pkts? rxbcast ifInBroadcastPkts - rxmcast -in-multicast-pkts? ifInMulticastPkts -in-discards? rxdrop ifInDiscards -in-errors? rxerrors ifInErrors -in-unknown-protos? -out-octets? txbytes ifOutOctets - txpackets -out-unicast-pkts? -out-broadcast-pkts? txbcast ifOutBroadcastPkts - txmcast -out-multicast-pkts? ifOutMulticastPkts -out-discards? txdrop ifOutDiscards -out-errors? txerrors ifOutErrors - rxpackets - txpackets - rxmcast - txmcast - mtu ifMtu - promisc ifPromiscuousMode - -snabb/type: - -0x1000 Hardware interface -0x1001 Virtual interface - -snabb/bcast, mcast, packets: - -(tx/rx) - bcast ⊆ mcast ⊆ packets diff --git a/src/doc/statistics.md b/src/doc/statistics.md new file mode 100644 index 0000000000..8ba1566a60 --- /dev/null +++ b/src/doc/statistics.md @@ -0,0 +1,58 @@ +### Statistics counters in Snabb + +Below is list of statistics counters defined by Snabb, and their relation to +RFC 7223 and ifTable MIB. All counters are unsigned 64bit integers. Each Snabb +app can optionally implement any number of these counters. + +| Snabb | RFC 7223 | ifTable MIB +| ----- | -------- | ----------- +| | name | +| | description? | ifDescr +| type (~= identity) | type | ifType +| | enabled? | +| | link-up-down-trap-enable? | +| | admin-status | +| status (= enum) | oper-status | ifOperStatus +| | last-change? | ifLastChange +| | if-index | +| macaddr (uint64) | phys-address? | ifPhysAddress +| | higher-layer-if* | +| | lower-layer-if* | +| speed | speed? | ifSpeed +| dtime (seconds since epoch) | discontinuity-time | ifCounterDiscontinuityTime +| rxbytes | in-octets? | ifInOctets +| rxpackets | | +| | in-unicast-pkts? | +| rxbcast | in-broadcast-pkts? | ifInBroadcastPkts +| rxmcast | | +| | in-multicast-pkts? | ifInMulticastPkts +| rxdrop | in-discards? | ifInDiscards +| rxerrors | in-errors? | ifInErrors +| | in-unknown-protos? | +| txbytes | out-octets? | ifOutOctets +| txpackets | | +| | out-unicast-pkts? | +| txbcast | out-broadcast-pkts? | ifOutBroadcastPkts +| txmcast | | +| | out-multicast-pkts? | ifOutMulticastPkts +| txdrop | out-discards? | ifOutDiscards +| txerrors | out-errors? | ifOutErrors +| rxpackets | | +| txpackets | | +| rxmcast | | +| txmcast | | +| mtu | | ifMtu +| promisc | | ifPromiscuousMode + + +#### type + + - `0x1000` Hardware interface + - `0x1001` Virtual interface + +#### rxbcast, rxmcast, rxpackets, txbcast, txmcast, txpackets + +Snabb defines total packet counts, multicast packet counts (packets with group +bit set) and broadcast packet counts for both input and output. + + E.g.: (tx/rx)bcast ⊆ (tx/rx)mcast ⊆ (tx/rx)packets From 339f8f1951256bb63273e47089aa8230b432346d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 16:27:24 +0200 Subject: [PATCH 41/53] apps.intel_app: Revert rxerrors calculation, remove broken txerrors. --- src/apps/intel/intel_app.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 623fb1a3a3..0e463d1e32 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -163,13 +163,13 @@ function Intel82599:sync_stats () -- The RX receive drop counts are only available through the RX stats -- register. We only read stats register #0 here. counter.set(counters.rxdrop, qs.QPRDC[0]()) - counter.set(counters.rxerrors, s.TPR() - s.GPRC()) + counter.set(counters.rxerrors, s.CRCERRS() + s.ILLERRC() + s.ERRBC() + + s.RUC() + s.RFC() + s.ROC() + s.RJC()) counter.set(counters.txbytes, s.GOTC64()) counter.set(counters.txpackets, s.GPTC()) local mptc, bptc = s.MPTC(), s.BPTC() counter.set(counters.txmcast, mptc + bptc) counter.set(counters.txbcast, bptc) - counter.set(counters.txerrors, s.TPT() - s.GPTC()) if bit.band(r.LINKS(), link_up_mask) == link_up_mask then counter.set(counters.status, 1) -- Up else From 7ec7f807d15484f531ca31b56d671e0939b06fb4 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 7 Jun 2016 16:30:57 +0200 Subject: [PATCH 42/53] Revert "lib.macaddress: add bytes method." and remove int method. This reverts commit e9f5dda5452ed2bcb1f1394cabf8b1f604357e15. --- src/apps/intel/intel_app.lua | 4 ++-- src/lib/ipc/shmem/iftable_mib.lua | 2 +- src/lib/macaddress.lua | 8 -------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 0e463d1e32..fc5829c723 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -77,7 +77,7 @@ function Intel82599:new (arg) counter.set(self.stats.counters.status, 2) -- down if not conf.vmdq and conf.macaddr then counter.set(self.stats.counters.macaddr, - macaddress:new(conf.macaddr):int()) + macaddress:new(conf.macaddr).bits) end end return setmetatable(self, Intel82599) @@ -118,7 +118,7 @@ function Intel82599:reconfig(arg) if not self.dev.pf and conf.macaddr then counter.set(self.stats.counters.macaddr, - macaddress:new(conf.macaddr):int()) + macaddress:new(conf.macaddr).bits) end end diff --git a/src/lib/ipc/shmem/iftable_mib.lua b/src/lib/ipc/shmem/iftable_mib.lua index 0f09aef0aa..8d96d29e67 100644 --- a/src/lib/ipc/shmem/iftable_mib.lua +++ b/src/lib/ipc/shmem/iftable_mib.lua @@ -36,7 +36,7 @@ function init_snmp (name, counters, directory, interval) ifTable:register('ifPhysAddress', { type = 'OctetStr', length = 6 }) if counters.macaddr then local mac = macaddress:new(counter.read(counters.macaddr)) - ifTable:set('ifPhysAddress', ffi.string(mac:bytes(), 6)) + ifTable:set('ifPhysAddress', ffi.string(mac.bytes, 6)) end ifTable:register('ifAdminStatus', 'Integer32', 1) -- up ifTable:register('ifOperStatus', 'Integer32', 2) -- down diff --git a/src/lib/macaddress.lua b/src/lib/macaddress.lua index 0508df17d6..a973b1d7b3 100644 --- a/src/lib/macaddress.lua +++ b/src/lib/macaddress.lua @@ -42,14 +42,6 @@ function mac_mt.__eq (a, b) return a.bits == b.bits end -function mac_mt:int () - return self.bits -end - -function mac_mt:bytes () - return self.bytes -end - function mac_mt:subbits (i,j) local b = bit.rshift(self.bits, i) local mask = bit.bnot(bit.lshift(0xffffffffffffLL, j-i)) From 327804f56d2512cce7967b26fe8bbab726178aaf Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 9 Jun 2016 17:09:14 +0200 Subject: [PATCH 43/53] doc/statistics.md: remove duplicates. --- src/doc/statistics.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/doc/statistics.md b/src/doc/statistics.md index 8ba1566a60..852d77829f 100644 --- a/src/doc/statistics.md +++ b/src/doc/statistics.md @@ -37,10 +37,6 @@ app can optionally implement any number of these counters. | | out-multicast-pkts? | ifOutMulticastPkts | txdrop | out-discards? | ifOutDiscards | txerrors | out-errors? | ifOutErrors -| rxpackets | | -| txpackets | | -| rxmcast | | -| txmcast | | | mtu | | ifMtu | promisc | | ifPromiscuousMode From 90fc893bef4caa2260793b672ad8ef29f1d7591c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 13:00:02 +0200 Subject: [PATCH 44/53] core.app: fix bug in #766 introduced while merging 6cac870a. --- src/core/app.lua | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index dbcc7f7fa9..94d08a0e23 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -72,20 +72,21 @@ end function with_restart (app, method) local oldshm = shm.path shm.path = app.shmpath + local status, result if use_restart then -- Run fn in protected mode using pcall. - local status, result_or_error = pcall(method, app) + status, result = pcall(method, app) -- If pcall caught an error mark app as "dead" (record time and cause -- of death). if not status then - app.dead = { error = result_or_error, time = now() } + app.dead = { error = result, time = now() } end - return status, result_or_error else - return true, method(app) + status, result = true, method(app) end shm.path = oldshm + return status, result_or_error end -- Restart dead apps. From ceb080f6dd40ea5d7173197d2e06e3fc6e16dcf1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 13:10:20 +0200 Subject: [PATCH 45/53] core.app: fix bug in #766, use app.shmpath during reconfig. --- src/core/app.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index 94d08a0e23..38a1973bf3 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -206,7 +206,10 @@ function apply_config_actions (actions, conf) if app_table[name].reconfig then local arg = conf.apps[name].arg local app = app_table[name] + local shmorig = shm.path + shm.path = app.shmpath app:reconfig(arg) + shm.path = shmorig new_app_table[name] = app table.insert(new_app_array, app) app_name_to_index[name] = #new_app_array From 1d4e766fa2dccc62c0b35ebf904aebb8eae8540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 10 Jun 2016 13:33:52 +0100 Subject: [PATCH 46/53] src/Makefile: silence grep Otherwise it prints misleading errors: grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory grep: jit/vmdef.lua: No such file or directory grep: program/lisper/dev-env/syscall.lua: No such file or directory grep: program/lisper/dev-env-docker/syscall.lua: No such file or directory --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 610186ce0b..467b88e9eb 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,7 +39,7 @@ EXE := bin/snabb $(patsubst %,bin/%,$(PROGRAM)) # core.memory core.lib ... # for each module that has a top-level selftest () function. TESTMODS = $(shell find . -regex '[^\#]*\.lua' -printf '%P ' | \ - xargs grep -l '^function selftest *[[:punct:]]' | \ + xargs grep -s -l '^function selftest *[[:punct:]]' | \ sed -e 's_\.lua__' -e 's_/_._g') # TESTSCRIPTS expands to: From d82729abf465c0e9a800e7de276d00f144c04b3a Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 10 Jun 2016 14:41:58 +0200 Subject: [PATCH 47/53] core.app: fix typo in previous commit. --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 38a1973bf3..5d7fa0e8fd 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -86,7 +86,7 @@ function with_restart (app, method) status, result = true, method(app) end shm.path = oldshm - return status, result_or_error + return status, result end -- Restart dead apps. From ed687f19f8459413145fabc56a462d7b38513ba8 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:11:18 +0200 Subject: [PATCH 48/53] main: properly handle unsupported worker states (SIGSTOP/SIGCONT)- --- src/core/main.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 0a27c076e5..32a1b482e2 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -177,8 +177,13 @@ else if signals[1].chld then local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG + else + S.kill(worker_pid, "hup") + exit_status = 255 + print("Error: Unsupported worker state (stopped / continued).") + end else S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo From db9e343d2c86c8db0f1d1e8fca9be21d80425599 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:11:59 +0200 Subject: [PATCH 49/53] main: comment syscall usage. --- src/core/main.lua | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 32a1b482e2..6a100a9a97 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -163,31 +163,41 @@ end -- Fork into worker process and supervisor local worker_pid = S.fork() if worker_pid == 0 then - -- Worker + -- Worker: Use prctl to ensure we are killed (SIGHUP) when our parent quits + -- and run main. S.prctl("set_pdeathsig", "hup") xpcall(main, handler) else - -- Supervisor + -- Supervisor: Queue exit_signals using signalfd, prevent them from killing + -- us instantly using sigprocmask. local exit_signals = "hup, int, quit, term, chld" local signalfd = S.signalfd(exit_signals) S.sigprocmask("block", exit_signals) + -- Read signals from signalfd. Only process the first signal because any + -- signal causes shutdown. local signals, err = S.util.signalfd_read(signalfd) assert(signals, tostring(err)) local exit_status if signals[1].chld then + -- SIGCHILD means worker state changed: retrieve its status using waitpid + -- and set exit status accordingly. local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) if worker.WIFEXITED then exit_status = worker.EXITSTATUS elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG else + -- Stopping/continuing the worker is unsupported and causes shutdown. S.kill(worker_pid, "hup") exit_status = 255 print("Error: Unsupported worker state (stopped / continued).") end else + -- Supervisor received exit signal: kill worker by sending SIGHUP and + -- and set exit status accordingly. S.kill(worker_pid, "hup") exit_status = 128 + signals[1].signo end + -- Clean up and exit. shutdown(worker_pid) os.exit(exit_status) end From c3b8672860bd2075a65c874270a8c9e9aaba06b6 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 14 Jun 2016 14:30:49 +0200 Subject: [PATCH 50/53] Revert 'ed687f19f8459413145fabc56a462d7b38513ba8'. Conflicts: - src/core/main.lua --- src/core/main.lua | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 6a100a9a97..29ed5a50f4 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -183,14 +183,8 @@ else -- and set exit status accordingly. local status, err, worker = S.waitpid(worker_pid) assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG - else - -- Stopping/continuing the worker is unsupported and causes shutdown. - S.kill(worker_pid, "hup") - exit_status = 255 - print("Error: Unsupported worker state (stopped / continued).") - end + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + else exit_status = 128 + worker.WTERMSIG end else -- Supervisor received exit signal: kill worker by sending SIGHUP and -- and set exit status accordingly. From 5eb0f697e3d0de565acf589b615d3d8a288c6d2c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 14:36:05 +0200 Subject: [PATCH 51/53] core.main: handle SIGSTOP and SIGCONT to worker. This reverts commit c3b8672860bd2075a65c874270a8c9e9aaba06b6. --- src/core/main.lua | 48 +++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 29ed5a50f4..0054890f77 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -173,25 +173,33 @@ else local exit_signals = "hup, int, quit, term, chld" local signalfd = S.signalfd(exit_signals) S.sigprocmask("block", exit_signals) - -- Read signals from signalfd. Only process the first signal because any - -- signal causes shutdown. - local signals, err = S.util.signalfd_read(signalfd) - assert(signals, tostring(err)) - local exit_status - if signals[1].chld then - -- SIGCHILD means worker state changed: retrieve its status using waitpid - -- and set exit status accordingly. - local status, err, worker = S.waitpid(worker_pid) - assert(status, tostring(err)) - if worker.WIFEXITED then exit_status = worker.EXITSTATUS - else exit_status = 128 + worker.WTERMSIG end - else - -- Supervisor received exit signal: kill worker by sending SIGHUP and - -- and set exit status accordingly. - S.kill(worker_pid, "hup") - exit_status = 128 + signals[1].signo + while true do + -- Read signals from signalfd. Only process the first signal because any + -- signal causes shutdown. + local signals, err = S.util.signalfd_read(signalfd) + assert(signals, tostring(err)) + for i = 1, #signals do + local exit_status + if signals[i].chld then + -- SIGCHILD means worker state changed: retrieve its status using + -- waitpid and set exit status accordingly. + local status, err, worker = + S.waitpid(worker_pid, "stopped,continued") + assert(status, tostring(err)) + if worker.WIFEXITED then exit_status = worker.EXITSTATUS + elseif worker.WIFSIGNALED then exit_status = 128 + worker.WTERMSIG + -- WIFSTOPPED and WIFCONTINUED are ignored. + else goto ignore_signal end + else + -- Supervisor received exit signal: kill worker by sending SIGHUP + -- and and set exit status accordingly. + S.kill(worker_pid, "hup") + exit_status = 128 + signals[i].signo + end + -- Run shutdown routine and exit. + shutdown(worker_pid) + os.exit(exit_status) + ::ignore_signal:: + end end - -- Clean up and exit. - shutdown(worker_pid) - os.exit(exit_status) end From 76a5d6e4391bc1d8560b82616b25cb69f0fdecb3 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 16:01:14 +0200 Subject: [PATCH 52/53] main: allow setting developer_debug/debug_on_error through environment. --- src/core/main.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 0054890f77..cc31d26285 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -29,8 +29,8 @@ ffi.cdef[[ extern char** argv; ]] -_G.developer_debug = false -debug_on_error = false +_G.developer_debug = lib.getenv("SNABB_DEBUG") and true +debug_on_error = _G.developer_debug function main () zone("startup") From 127d21fa6e6dccc0cd00ceb9b11dd24ba838b723 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 15 Jun 2016 16:01:46 +0200 Subject: [PATCH 53/53] main: do not unlink runtime directory in developer_debug mode. --- src/core/main.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index cc31d26285..f736e25458 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -133,7 +133,9 @@ end -- Cleanup after Snabb process. function shutdown (pid) - shm.unlink("//"..pid) + if not _G.developer_debug then + shm.unlink("//"..pid) + end end function selftest ()