From 0249f8b8310c80c487b0d92e09649d27d672002b Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Wed, 29 Mar 2017 11:09:32 +0200 Subject: [PATCH] Feature/remove multi processing (#510) * remove metaclass * first step in removing the metaclass * fix: Remove background * fix: remove alt-docs about monitor * remove signal queue * fix: More backgroudn removal * remove data_manager stuff * feature: Remove tests, yolo * feature: Rremove more mocks * finally eradicate mp * remove Widgets * chore: Remove benchmarking and reloading Benchmarking should go if anything to its own repo reloading is dangerous, and it can trick many new users. * chore: Add todo * chore: Use py.test move faster without reinventing the wheel at all times * fix: Remove nested attrs as unused * Fix remove try/Interrupt, missing variable imax * fix: Remove trailing number from test name * fix: Use unit * chore: Update documentation * fix: Remove toymodel, and simlify examples * docs: Remove more MP from docs * chore: Remove untested example adaptive sweep * docs: Update docs * fix: remove mp from config * fix: Remove server references * chore: Add todo * fix: Remove server and metaclass * add stuff * fix: use right default for _instances * bugs * feature: add more tests * Remove find_component * fix vim typo * fix: Remove array getter * fix: remove timing * feature: Allow for legacy code to run This allows for no exception but just a warning. The deprecated argument is ignored. --- CONTRIBUTING.rst | 114 +- README.rst | 4 +- benchmarking/mptest.py | 156 -- benchmarking/mptest_results_mac.txt | 117 -- benchmarking/mptest_results_windows.txt | 88 - benchmarking/thread_test.py | 55 - docs/api/private.rst | 3 - docs/api/public.rst | 9 - docs/community/index.rst | 1 - docs/community/testing.rst | 80 - .../Creating Instrument Drivers.ipynb | 72 +- docs/examples/Datasaving examples.ipynb | 476 +---- docs/examples/Measure without a Loop.ipynb | 475 +---- docs/examples/Metadata with instruments.ipynb | 33 +- docs/examples/Parameters.ipynb | 71 +- docs/examples/Tutorial.ipynb | 1567 ++--------------- docs/examples/adaptive_sweep.py | 103 -- .../Keithley_example.ipynb | 104 +- .../Qcodes example ATS_ONWORK.ipynb | 0 .../Qcodes example with Agilent 34400A.ipynb | 0 .../Qcodes example with Decadac.ipynb | 0 .../Qcodes example with Ithaco.ipynb | 0 .../Qcodes example with Keithley 2600.ipynb | 0 ...es example with Mercury IPS (Magnet).ipynb | 0 .../Qcodes example with QDac.ipynb | 0 ...odes example with Rohde Schwarz ZN20.ipynb | 0 ...odes example with Tektronix AWG5014C.ipynb | 0 .../Qcodes example with Triton.ipynb | 36 +- .../Triton1_thermometry.reg | 0 docs/examples/testsweep/gates_chan0_set.dat | 404 ----- docs/examples/testsweep/snapshot.json | 216 --- docs/examples/toymodel.py | 149 -- docs/user/tutorial.rst | 18 +- qcodes/__init__.py | 17 +- qcodes/config/qcodesrc.json | 1 - qcodes/config/qcodesrc_schema.json | 7 +- qcodes/data/data_set.py | 290 +-- qcodes/data/manager.py | 163 -- qcodes/instrument/base.py | 190 +- qcodes/instrument/ip.py | 19 - qcodes/instrument/metaclass.py | 45 - qcodes/instrument/mock.py | 316 ---- qcodes/instrument/remote.py | 529 ------ qcodes/instrument/server.py | 190 -- qcodes/instrument/visa.py | 28 - qcodes/loops.py | 257 +-- qcodes/measure.py | 19 +- qcodes/plots/base.py | 10 +- qcodes/process/__init__.py | 0 qcodes/process/helpers.py | 63 - qcodes/process/qcodes_process.py | 72 - qcodes/process/server.py | 399 ----- qcodes/process/stream_queue.py | 152 -- qcodes/station.py | 9 +- qcodes/test.py | 119 -- qcodes/tests/data_mocks.py | 24 - qcodes/tests/instrument_mocks.py | 201 +-- qcodes/tests/test_data.py | 159 +- qcodes/tests/test_driver_testcase.py | 58 - qcodes/tests/test_format.py | 4 +- qcodes/tests/test_hdf5formatter.py | 6 +- qcodes/tests/test_helpers.py | 4 + qcodes/tests/test_instrument.py | 998 +---------- qcodes/tests/test_instrument_server.py | 157 -- qcodes/tests/test_loop.py | 359 +--- qcodes/tests/test_measure.py | 2 - qcodes/tests/test_multiprocessing.py | 471 ----- qcodes/tests/test_nested_attrs.py | 100 -- qcodes/tests/test_visa.py | 52 +- qcodes/utils/nested_attrs.py | 175 -- qcodes/utils/reload_code.py | 98 -- qcodes/utils/timing.py | 89 - qcodes/widgets/__init__.py | 0 qcodes/widgets/display.py | 47 - qcodes/widgets/widgets.css | 77 - qcodes/widgets/widgets.js | 237 --- qcodes/widgets/widgets.py | 215 --- 77 files changed, 609 insertions(+), 10170 deletions(-) delete mode 100644 benchmarking/mptest.py delete mode 100644 benchmarking/mptest_results_mac.txt delete mode 100644 benchmarking/mptest_results_windows.txt delete mode 100644 benchmarking/thread_test.py delete mode 100644 docs/community/testing.rst delete mode 100644 docs/examples/adaptive_sweep.py rename docs/examples/{ => driver_examples}/Keithley_example.ipynb (85%) rename docs/examples/{ => driver_examples}/Qcodes example ATS_ONWORK.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Agilent 34400A.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Decadac.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Ithaco.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Keithley 2600.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Mercury IPS (Magnet).ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with QDac.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Rohde Schwarz ZN20.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Tektronix AWG5014C.ipynb (100%) rename docs/examples/{ => driver_examples}/Qcodes example with Triton.ipynb (95%) rename docs/examples/{ => driver_examples}/Triton1_thermometry.reg (100%) delete mode 100644 docs/examples/testsweep/gates_chan0_set.dat delete mode 100644 docs/examples/testsweep/snapshot.json delete mode 100644 docs/examples/toymodel.py delete mode 100644 qcodes/data/manager.py delete mode 100644 qcodes/instrument/metaclass.py delete mode 100644 qcodes/instrument/mock.py delete mode 100644 qcodes/instrument/remote.py delete mode 100644 qcodes/instrument/server.py delete mode 100644 qcodes/process/__init__.py delete mode 100644 qcodes/process/helpers.py delete mode 100644 qcodes/process/qcodes_process.py delete mode 100644 qcodes/process/server.py delete mode 100644 qcodes/process/stream_queue.py delete mode 100644 qcodes/test.py delete mode 100644 qcodes/tests/test_driver_testcase.py delete mode 100644 qcodes/tests/test_instrument_server.py delete mode 100644 qcodes/tests/test_multiprocessing.py delete mode 100644 qcodes/tests/test_nested_attrs.py delete mode 100644 qcodes/utils/nested_attrs.py delete mode 100644 qcodes/utils/reload_code.py delete mode 100644 qcodes/utils/timing.py delete mode 100644 qcodes/widgets/__init__.py delete mode 100644 qcodes/widgets/display.py delete mode 100644 qcodes/widgets/widgets.css delete mode 100644 qcodes/widgets/widgets.js delete mode 100644 qcodes/widgets/widgets.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 63aef9e7e31..0657ab003c0 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -59,13 +59,18 @@ Setup Running Tests ~~~~~~~~~~~~~ -The core test runner is in ``qcodes/test.py``: +We don't want to reinvent the wheel, and thus use py.test. +It's easy to install: :: - python qcodes/test.py - # optional extra verbosity and fail fast - python qcodes/test.py -v -f + pip install coverage pytest-cov pytest + +Then to test and view the coverage: + +:: + py.test --cov=qcodes --cov-report xml --cov-config=.coveragerc + You can also run single tests with: @@ -78,92 +83,6 @@ You can also run single tests with: # or python -m unittest qcodes.tests.test_metadata.TestMetadatable.test_snapshot -If you run the core test runner, you should see output that looks -something like this: - -:: - - .........***** found one MockMock, testing ***** - ............................................Timing resolution: - startup time: 0.000e+00 - min/med/avg/max dev: 9.260e-07, 9.670e-07, 1.158e-06, 2.109e-03 - async sleep delays: - startup time: 2.069e-04 - min/med/avg/max dev: 3.372e-04, 6.376e-04, 6.337e-04, 1.007e-03 - multiprocessing startup delay and regular sleep delays: - startup time: 1.636e-02 - min/med/avg/max dev: 3.063e-05, 2.300e-04, 2.232e-04, 1.743e-03 - should go to stdout;should go to stderr;.stdout stderr stdout stderr ..[10:44:09.063 A Queue] should get printed - ................................... - ---------------------------------------------------------------------- - Ran 91 tests in 4.192s - - OK - Name Stmts Miss Cover Missing - ---------------------------------------------------------- - data/data_array.py 104 0 100% - data/data_set.py 179 140 22% 38-55, 79-94, 99-104, 123-135, 186-212, 215-221, 224-244, 251-254, 257-264, 272, 280-285, 300-333, 347-353, 360-384, 395-399, 405-407, 414-420, 426-427, 430, 433-438 - data/format.py 225 190 16% 44-55, 61-62, 70, 78-97, 100, 114-148, 157-188, 232, 238, 246, 258-349, 352, 355-358, 361-368, 375-424, 427-441, 444, 447-451 - data/io.py 76 50 34% 71-84, 90-91, 94, 97, 103, 109-110, 119-148, 154-161, 166, 169, 172, 175-179, 182, 185-186 - data/manager.py 124 89 28% 15-20, 31, 34, 48-62, 65-67, 70, 76-77, 80-84, 90-102, 108-110, 117-121, 142-151, 154-182, 185, 188, 207-208, 215-221, 227-229, 237, 243, 249 - instrument/base.py 74 0 100% - instrument/function.py 45 1 98% 77 - instrument/ip.py 20 12 40% 10-16, 19-20, 24-25, 29-38 - instrument/mock.py 63 0 100% - instrument/parameter.py 200 2 99% 467, 470 - instrument/sweep_values.py 107 33 69% 196-207, 220-227, 238-252, 255-277 - instrument/visa.py 36 24 33% 10-25, 28-32, 35-36, 40-41, 47-48, 57-58, 62-64, 68 - loops.py 285 239 16% 65-74, 81-91, 120-122, 133-141, 153-165, 172-173, 188-207, 216-240, 243-313, 316-321, 324-350, 354-362, 371-375, 378-381, 414-454, 457-474, 477-484, 487-491, 510-534, 537-543, 559-561, 564, 577, 580, 590-608, 611-618, 627-628, 631 - station.py 35 24 31% 17-32, 35, 45-50, 60, 67-82, 88 - utils/helpers.py 95 0 100% - utils/metadata.py 13 0 100% - utils/multiprocessing.py 95 2 98% 125, 134 - utils/sync_async.py 114 8 93% 166, 171-173, 176, 180, 184, 189-191 - utils/timing.py 72 0 100% - utils/validators.py 110 0 100% - ---------------------------------------------------------- - TOTAL 2072 814 61% - -The key is ``OK`` in the middle (that means all the tests passed), and -the presence of the coverage report after it. If any tests fail, we do -not show a coverage report, and the end of the output will contain -tracebacks and messages about what failed, for example: - -:: - - ====================================================================== - FAIL: test_sweep_steps_edge_case (tests.test_instrument.TestParameters) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "/Users/alex/qdev/Qcodes/qcodes/tests/test_instrument.py", line 360, in test_sweep_steps_edge_case - self.check_set_amplitude2('Off', log_count=1, history_count=2) - File "/Users/alex/qdev/Qcodes/qcodes/tests/test_instrument.py", line 345, in check_set_amplitude2 - self.assertTrue(line.startswith('negative delay'), line) - AssertionError: False is not true : cannot sweep amplitude2 from 0.1 to Off - jumping. - - ---------------------------------------------------------------------- - Ran 91 tests in 4.177s - - FAILED (failures=1) - -The coverage report is only useful if you have been adding new code, to -see whether your tests visit all of your code. Look at the file(s) you -have been working on, and ensure that the "missing" section does not -contain the line numbers of any of the blocks you have touched. -Currently the core still has a good deal of untested code - eventually -we will have all of this tested, but for now you can ignore all the rest -of the missing coverage. - -You can also run these tests from inside python. The output is similar -except that a) you don't get coverage reporting, and b) one test has to -be skipped because it does not apply within a notebook, so the output -will end ``OK (skipped=1)``: - -.. code:: python - - import qcodes - qcodes.test_core() # optional verbosity = 1 (default) or 2 - If the tests pass, you should be ready to start developing! To tests actual instruments, first instantiate them in an interactive @@ -314,15 +233,6 @@ and then unit testing should be run on pull-request, using CI. Maybe simplify to a one command that says: if there's enough cover, and all good or fail and where it fails. -- The standard test commands are listed above under - :ref:`runnningtests`. More notes on different test runners can - be found in :ref:`testing`. - -- Core tests live in - `qcodes/tests `__ - and instrument tests live in the same directories as the instrument - drivers. - - We should have a *few* high-level "integration" tests, but simple unit tests (that just depend on code in one module) are more valuable for several reasons: @@ -330,7 +240,6 @@ good or fail and where it fails. - When features change it is likely that more tests will need to change - Unit tests can cover many scenarios much faster than integration tests. - - If you're having difficulty making unit tests, first consider whether your code could be restructured to make it less dependent on other modules. Often, however, extra techniques are needed to break down a @@ -339,9 +248,8 @@ good or fail and where it fails. - Patching, one of the most useful parts of the `unittest.mock `__ library. This lets you specify exactly how other functions/objects - should behave when they're called by the code you are testing. For a - simple example, see - `test\_multiprocessing.py `__ + should behave when they're called by the code you are testing. + - Supporting files / data: Lets say you have a test of data acquisition and analysis. You can break that up into an acquisition test and an analysis by saving the intermediate state, namely the data file, in diff --git a/README.rst b/README.rst index 423dfab85e8..c17387b7d92 100644 --- a/README.rst +++ b/README.rst @@ -70,9 +70,11 @@ $QCODES_INSTALL_DIR is the folder where you want to have the source code. cd $QCODES_INSTALL_DIR pyenv install 3.5.2 pyenv virtualenv 3.5.2 qcodes-dev + pyenv activate qcodes-dev pip install -r requirements.txt + pip install coverage pytest-cov pytest --upgrade pip install -e . - python qcodes/test.py -f + py.test --cov=qcodes --cov-config=.coveragerc If the tests pass you are ready to hack! This is the reference setup one needs to have to contribute, otherwise diff --git a/benchmarking/mptest.py b/benchmarking/mptest.py deleted file mode 100644 index 4072ef92169..00000000000 --- a/benchmarking/mptest.py +++ /dev/null @@ -1,156 +0,0 @@ -# stress test multiprocessing -# run with: python mptest.py -# proc_count: the number of processes to spin up and load qcodes -# period: milliseconds to wait between calling each process -# repetitions: how many times to call each one -# start_method: multiprocessing method to use (fork, forkserver, spawn) - -import multiprocessing as mp -import sys -import os -import psutil -import time - -import qcodes as qc - - -timer = time.perf_counter - - -def print_perf(): - print(time.perf_counter()) - - -def mp_test(name, qin, qout, qglobal): - ''' - simple test that keeps a process running until asked to stop, - and looks for an attribute within qcodes just to ensure it has loaded it - ''' - delays = [] - first = True - while True: - item, qtime = qin.get() - if first: # ignore the first one... process is still starting - first = False - else: - delays.append(timer() - qtime) - if item == 'break': - qglobal.put({ - 'name': name, - 'avg': sum(delays) / len(delays), - 'max': max(delays) - }) - break - qout.put(repr(getattr(qc, item))) - - -def get_memory(pid): - mi = psutil.Process(pid).memory_info() - return mi.rss, mi.vms - - -def get_all_memory(processes, title): - main_memory = get_memory(os.getpid()) - proc_memory = [0, 0] - for proc in processes: - for i, v in enumerate(get_memory(proc.pid)): - proc_memory[i] += v - # print(v) - - return { - 'main_physical': main_memory[0]/1e6, - 'main_virtual': main_memory[1]/1e6, - 'proc_physical': proc_memory[0]/1e6, - 'proc_virtual': proc_memory[1]/1e6, - 'title': title - } - - -def format_memory(mem): - return ('{title}\n' - ' main: {main_physical:.0f} MB phys, ' - '{main_virtual:.0f} MB virt\n' - ' procs: {proc_physical:.0f} MB phys, ' - '{proc_virtual:.0f} MB virt\n' - '').format(**mem) - - -if __name__ == '__main__': - proc_count = int(sys.argv[-4]) - period = float(sys.argv[-3]) - reps = int(sys.argv[-2]) - method = sys.argv[-1] - mp.set_start_method(method) - - qglobal = mp.Queue() - - mem = [get_all_memory([], 'on startup')] - - queues = [] - processes = [] - resp_delays = [] - - t_before_start = timer() - - for proc_num in range(proc_count): - qin = mp.Queue() - qout = mp.Queue() - queues.append((qin, qout)) - p = mp.Process(target=mp_test, - args=('p{}'.format(proc_num), qin, qout, qglobal)) - processes.append(p) - p.start() - - start_delay = (timer() - t_before_start) * 1000 - - mem.append(get_all_memory(processes, 'procs started')) - - for i in range(reps): - for qin, qout in queues: - t1 = timer() - qin.put(('Loop', timer())) - loop_repr = qout.get() - if i: - # ignore the first one, process is still starting - resp_delays.append((timer() - t1) * 1000) - if(loop_repr != repr(qc.Loop)): - raise RuntimeError('{} != {}'.format(loop_repr, repr(qc.Loop))) - print('.', end='', flush=True) - time.sleep(period / 1000) - print('') - - mem.append(get_all_memory(processes, 'procs done working')) - - for qin, qout in queues: - qin.put(('break', timer())) - - t_before_join = timer() - for proc in processes: - proc.join() - join_delay = (timer() - t_before_join) * 1000 - - delays = [qglobal.get() for proc in processes] - avg_delay = sum([d['avg'] for d in delays]) * 1000 / len(delays) - max_delay = max([d['max'] for d in delays]) * 1000 - - avg_resp_delay = sum(resp_delays) / len(resp_delays) - max_resp_delay = max(resp_delays) - - print(('Ran {} procs using "{}" method\n' - 'sent messages every {} milliseconds, {} times\n' - ).format(proc_count, method, period, reps)) - - print('Milliseconds to start all processes: {:.3f}'.format(start_delay)) - print('Final join delay: {:.3f}\n'.format(join_delay)) - - print('Milliseconds to receive to queue request') - print(' avg: {:.6f}'.format(avg_delay)) - print(' max: {:.6f}\n'.format(max_delay)) - - print('Milliseconds to respond to queue request') - print(' avg: {:.6f}'.format(avg_resp_delay)) - print(' max: {:.6f}\n'.format(max_resp_delay)) - - # report on the memory results - for m in mem: - print(format_memory(m)) diff --git a/benchmarking/mptest_results_mac.txt b/benchmarking/mptest_results_mac.txt deleted file mode 100644 index ec8e2a7765d..00000000000 --- a/benchmarking/mptest_results_mac.txt +++ /dev/null @@ -1,117 +0,0 @@ -Mac OS X -------------------- - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 30.601 -Final join delay: 130.124 - -Milliseconds to receive to queue request - avg: 0.321102 - max: 1.061535 - -Milliseconds to respond to queue request - avg: 0.607558 - max: 1.465819 - -on startup - main: 29 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 8 MB phys, 12327 MB virt - -procs done working - main: 29 MB phys, 2524 MB virt - procs: 142 MB phys, 12509 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 512.364 -Final join delay: 456.155 - -Milliseconds to receive to queue request - avg: 0.302726 - max: 2.754109 - -Milliseconds to respond to queue request - avg: 0.555251 - max: 1.123662 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 150 MB phys, 49561 MB virt - -procs done working - main: 29 MB phys, 2603 MB virt - procs: 571 MB phys, 50029 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 5242.341 -Final join delay: 3806.779 - -Milliseconds to receive to queue request - avg: 0.527770 - max: 59.421732 - -Milliseconds to respond to queue request - avg: 0.416881 - max: 4.560305 - -on startup - main: 28 MB phys, 2497 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 30 MB phys, 2499 MB virt - procs: 1234 MB phys, 248003 MB virt - -procs done working - main: 11 MB phys, 3026 MB virt - procs: 2144 MB phys, 250162 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 fork -.................................................. -Ran 100 procs using "fork" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 447.375 -Final join delay: 113.426 - -Milliseconds to receive to queue request - avg: 0.351694 - max: 11.889809 - -Milliseconds to respond to queue request - avg: 0.494252 - max: 2.525026 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2499 MB virt - procs: 444 MB phys, 249616 MB virt - -procs done working - main: 32 MB phys, 3025 MB virt - procs: 525 MB phys, 250142 MB virt diff --git a/benchmarking/mptest_results_windows.txt b/benchmarking/mptest_results_windows.txt deleted file mode 100644 index e267edeb280..00000000000 --- a/benchmarking/mptest_results_windows.txt +++ /dev/null @@ -1,88 +0,0 @@ -Windows -------------------- - -c:\Qcodes>python mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 15.406 -Final join delay: 92.359 - -Milliseconds to receive to queue request - avg: -510.603209 - max: -509.684951 - -Milliseconds to respond to queue request - avg: 0.360488 - max: 0.659211 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 14 MB phys, 9 MB virt - -procs done working - main: 36 MB phys, 32 MB virt - procs: 179 MB phys, 155 MB virt - - -c:\Qcodes>python mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 126.134 -Final join delay: 228.260 - -Milliseconds to receive to queue request - avg: -1591.011659 - max: -1563.003352 - -Milliseconds to respond to queue request - avg: 0.365974 - max: 0.794690 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 98 MB phys, 60 MB virt - -procs done working - main: 37 MB phys, 32 MB virt - procs: 716 MB phys, 620 MB virt - - -c:\Qcodes>python mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 761.556 -Final join delay: 927.847 - -Milliseconds to receive to queue request - avg: -8560.453780 - max: -8470.848083 - -Milliseconds to respond to queue request - avg: 0.423033 - max: 1.299948 - -on startup - main: 35 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 37 MB phys, 32 MB virt - procs: 872 MB phys, 572 MB virt - -procs done working - main: 40 MB phys, 36 MB virt - procs: 3575 MB phys, 3094 MB virt diff --git a/benchmarking/thread_test.py b/benchmarking/thread_test.py deleted file mode 100644 index 40dac28bc69..00000000000 --- a/benchmarking/thread_test.py +++ /dev/null @@ -1,55 +0,0 @@ -import threading -import time - - -def sleeper(t, n, out, make_error): - time.sleep(t) - out[n] = n - if make_error: - raise RuntimeError('hello from # {}!'.format(n)) - - -def runmany(n, t, error_nums): - out = [None] * n - - t0 = time.perf_counter() - - threads = [ - CatchingThread(target=sleeper, args=(t, i, out, i in error_nums)) - for i in range(n)] - - # start threads backward - [t.start() for t in reversed(threads)] - [t.join() for t in threads] - - t1 = time.perf_counter() - - out_ok = [] - for i in range(n): - if out[i] != i: - out_ok += ['ERROR! out[{}] = {}'.format(i, out[i])] - - if not out_ok: - out_ok += ['all output correct'] - - print('{} parallel threads sleeping\n'.format(n) + - 'given time: {}\n'.format(t) + - 'resulting time: {}\n'.format(t1 - t0) + - '\n'.join(out_ok)) - - -class CatchingThread(threading.Thread): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.exception = None - - def run(self): - try: - super().run() - except Exception as e: - self.exception = e - - def join(self): - super().join() - if self.exception: - raise self.exception diff --git a/docs/api/private.rst b/docs/api/private.rst index 8fffb7cb2fa..f5e38caebc9 100644 --- a/docs/api/private.rst +++ b/docs/api/private.rst @@ -12,8 +12,5 @@ Classes and Functions utils.command utils.deferred_operations - utils.nested_attrs utils.helpers - utils.timing utils.metadata - process.qcodes_process diff --git a/docs/api/public.rst b/docs/api/public.rst index 12acc2e9464..04521a6f0d5 100644 --- a/docs/api/public.rst +++ b/docs/api/public.rst @@ -42,8 +42,6 @@ Loops .. autosummary:: :toctree: generated/ - get_bg - halt_bg Loop Measure @@ -70,9 +68,6 @@ Data .. autosummary:: :toctree: generated/ - get_data_manager - qcodes.data.manager.DataManager - DataMode DataSet new_data load_data @@ -102,8 +97,6 @@ Instrument Instrument IPInstrument VisaInstrument - MockInstrument - MockModel Plot @@ -123,7 +116,5 @@ Utils & misc :toctree: generated/ qcodes.utils.validators - qcodes.process.helpers.set_mp_method qcodes.utils.helpers.in_notebook - qcodes.widgets.widgets.show_subprocess_widget diff --git a/docs/community/index.rst b/docs/community/index.rst index 0d42653900d..c0cd05f6962 100644 --- a/docs/community/index.rst +++ b/docs/community/index.rst @@ -11,4 +11,3 @@ This is the guide for you, the developer that wants to maintain/expand QCoDes. install contributing objects - testing diff --git a/docs/community/testing.rst b/docs/community/testing.rst deleted file mode 100644 index dc0ef7d2210..00000000000 --- a/docs/community/testing.rst +++ /dev/null @@ -1,80 +0,0 @@ -.. _testing: - - -Notes on test runners compatible with Qcodes -============================================ - -There is now a test script `test.py `__ in the qcodes -directory that uses the standard ``unittest`` machinery to run all the -core tests (does not include instrument drivers). It has been tested on -Mac (terminal), and Windows (cmd, git bash, and PowerShell). It includes -coverage testing, but will only print a coverage report if tests pass. - -The biggest difficulty with testing Qcodes is windows multiprocessing. -The spawn method restricts execution in ways that are annoying for -regular users (no class/function definitions in the notebook, no -closures) but seem to be completely incompatible with some test runners -(and/or coverage tracking) - -I considered the following test runners: - **nose**: works well, but it -has a `note on its homepage `__ -that it is no longer being actively maintained (in favor of nose2 -development), so we should not use it long-term. - -- **unittest**: the standard, built-in python tester. The only thing we - really need to add to this is coverage testing, so now the question - is what's the easiest way to do this? On Windows just using unittest - wrapped in coverage fails. - -- **nose2**: has a broken coverage plugin - it reports all the - unindented lines, ie everything # that executes on import, as - uncovered - but can be used by wrapping it inside coverage instead, - just like unittest. - -- **py.test**: seems to add lots of features, but it's not clear they - are useful for us? Has a good coverage plugin but seems to require - tons of command-line options. Requires both ``pytest`` and - ``pytest-cov`` packages - -on Mac terminal: - -:: - - # the following work with coverage: - nosetests - python setup.py nosetests - py.test --cov-config .coveragerc --cov qcodes --cov-report term-missing - coverage run -m nose2 && coverage report -m - # both of these run unittest: - coverage run setup.py test && coverage report -m - coverage run -m unittest && coverage report -m - - # nose2's coverage plugin is broken - it reports all the unindented lines (everything - # that executes on import) as uncovered - nose2 -C --coverage-report term-missing - -Windows cmd shell and git bash behave identically, PowerShell has -different chain syntax (commands with &&): - -:: - - # the following work with coverage: - nosetests - py.test --cov-config .coveragerc --cov qcodes --cov-report term-missing - coverage run -m nose2 && coverage report -m # cmd or bash - (coverage run -m nose2) -and (coverage report -m) # PowerShell - - # the following work without coverage: - python -m unittest discover - python -m unittest # discover is unnecessary now, perhaps because I put test_suite in setup.py? - - # the following do not work: - - # fails on relative import in unittest inside separate process (why is it importing that anyway?) - coverage run -m unittest discover && coverage report -m # cmd or bash - (coverage run -m unittest discover) -and (coverage report -m) # PowerShell - # these fail asking for freeze_support() but nothing I do with that seems to help - python setup.py test - coverage run setup.py test && coverage report -m # cmd or bash - (coverage run setup.py test) -and (coverage report -m) # PowerShell - python setup.py nosetests diff --git a/docs/examples/Creating Instrument Drivers.ipynb b/docs/examples/Creating Instrument Drivers.ipynb index 807a5dc6eac..138e33d71f2 100644 --- a/docs/examples/Creating Instrument Drivers.ipynb +++ b/docs/examples/Creating Instrument Drivers.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Creating QCoDeS instrument drivers" ] @@ -11,7 +14,9 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -28,7 +33,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Base Classes\n", "\n", @@ -42,7 +50,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## VisaInstrument: Simple example\n", "The Weinschel 8320 driver is about as basic a driver as you can get. It only defines one parameter, \"attenuation\". All the comments here are my additions to describe what's happening." @@ -52,7 +63,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -100,7 +113,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## VisaInstrument: a more involved example\n", "the K2600 breaks one physical instrument into two software instruments, one for each channel. It:\n", @@ -117,7 +133,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -195,7 +213,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## DLL-based instruments\n", "The Alazar cards use their own DLL. C interfaces tend to need a lot of boilerplate, so I'm not going to include it all. The key is: use `Instrument` directly, load the DLL, and have parameters interact with it." @@ -205,7 +226,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -243,7 +266,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Manual instruments\n", "A totally manual instrument (like the ithaco 1211) will contain only `ManualParameter`s. Some instruments may have a mix of manual and standard parameters. Here we also define a new `CurrentParameter` class that uses the ithaco parameters to convert a measured voltage to a current." @@ -253,7 +279,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -360,7 +388,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Custom Parameter classes\n", "\n", @@ -383,7 +414,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Dynamically adding and removing parameters\n", "\n", @@ -401,7 +435,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Functions\n", "\n", @@ -410,7 +447,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Organization\n", "\n", @@ -445,7 +485,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.0" + "version": "3.5.2" } }, "nbformat": 4, diff --git a/docs/examples/Datasaving examples.ipynb b/docs/examples/Datasaving examples.ipynb index 1117b4b695b..9aa2b7746b1 100644 --- a/docs/examples/Datasaving examples.ipynb +++ b/docs/examples/Datasaving examples.ipynb @@ -4,346 +4,11 @@ "cell_type": "code", "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/*\n", - " * Qcodes Jupyter/IPython widgets\n", - " */\n", - "require([\n", - " 'nbextensions/widgets/widgets/js/widget',\n", - " 'nbextensions/widgets/widgets/js/manager'\n", - "], function (widget, manager) {\n", - "\n", - " var UpdateView = widget.DOMWidgetView.extend({\n", - " render: function() {\n", - " window.MYWIDGET = this;\n", - " this._interval = 0;\n", - " this.update();\n", - " },\n", - " update: function() {\n", - " this.display(this.model.get('_message'));\n", - " this.setInterval();\n", - " },\n", - " display: function(message) {\n", - " /*\n", - " * display method: override this for custom display logic\n", - " */\n", - " this.el.innerHTML = message;\n", - " },\n", - " remove: function() {\n", - " clearInterval(this._updater);\n", - " },\n", - " setInterval: function(newInterval) {\n", - " var me = this;\n", - " if(newInterval===undefined) newInterval = me.model.get('interval');\n", - " if(newInterval===me._interval) return;\n", - "\n", - " me._interval = newInterval;\n", - "\n", - " if(me._updater) clearInterval(me._updater);\n", - "\n", - " if(me._interval) {\n", - " me._updater = setInterval(function() {\n", - " me.send({myupdate: true});\n", - " if(!me.model.comm_live) {\n", - " console.log('missing comm, canceling widget updates', me);\n", - " clearInterval(me._updater);\n", - " }\n", - " }, me._interval * 1000);\n", - " }\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('UpdateView', UpdateView);\n", - "\n", - " var HiddenUpdateView = UpdateView.extend({\n", - " display: function(message) {\n", - " this.$el.hide();\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView);\n", - "\n", - " var SubprocessView = UpdateView.extend({\n", - " render: function() {\n", - " var me = this;\n", - " me._interval = 0;\n", - " me._minimize = '';\n", - " me._restore = '';\n", - "\n", - " // max lines of output to show\n", - " me.maxOutputLength = 500;\n", - "\n", - " // in case there is already an outputView present,\n", - " // like from before restarting the kernel\n", - " $('.qcodes-output-view').not(me.$el).remove();\n", - "\n", - " me.$el\n", - " .addClass('qcodes-output-view')\n", - " .attr('qcodes-state', 'docked')\n", - " .html(\n", - " '
' +\n", - " '
' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '
' +\n", - " '
'\n",
-       "                );\n",
-       "\n",
-       "            me.clearButton = me.$el.find('.qcodes-clear-output');\n",
-       "            me.minButton = me.$el.find('.qcodes-minimize');\n",
-       "            me.outputArea = me.$el.find('pre');\n",
-       "            me.subprocessList = me.$el.find('.qcodes-process-list');\n",
-       "            me.abortButton = me.$el.find('.qcodes-abort-loop');\n",
-       "            me.processLinesButton = me.$el.find('.qcodes-processlines')\n",
-       "\n",
-       "            me.outputLines = [];\n",
-       "\n",
-       "            me.clearButton.click(function() {\n",
-       "                me.outputArea.html('');\n",
-       "                me.clearButton.addClass('disabled');\n",
-       "            });\n",
-       "\n",
-       "            me.abortButton.click(function() {\n",
-       "                me.send({abort: true});\n",
-       "            });\n",
-       "\n",
-       "            me.processLinesButton.click(function() {\n",
-       "                // toggle multiline process list display\n",
-       "                me.subprocessesMultiline = !me.subprocessesMultiline;\n",
-       "                me.showSubprocesses();\n",
-       "            });\n",
-       "\n",
-       "            me.$el.find('.js-state').click(function() {\n",
-       "                var state = this.className.substr(this.className.indexOf('qcodes'))\n",
-       "                        .split('-')[1].split(' ')[0];\n",
-       "                me.model.set('_state', state);\n",
-       "            });\n",
-       "\n",
-       "            $(window)\n",
-       "                .off('resize.qcodes')\n",
-       "                .on('resize.qcodes', function() {me.clipBounds();});\n",
-       "\n",
-       "            me.update();\n",
-       "        },\n",
-       "\n",
-       "        updateState: function() {\n",
-       "            var me = this,\n",
-       "                oldState = me.$el.attr('qcodes-state'),\n",
-       "                state = me.model.get('_state');\n",
-       "\n",
-       "            if(state === oldState) return;\n",
-       "\n",
-       "            setTimeout(function() {\n",
-       "                // not sure why I can't pop it out of the widgetarea in render, but it seems that\n",
-       "                // some other bit of code resets the parent after render if I do it there.\n",
-       "                // To be safe, just do it on every state click.\n",
-       "                me.$el.appendTo('body');\n",
-       "\n",
-       "                if(oldState === 'floated') {\n",
-       "                    console.log('here');\n",
-       "                    me.$el.draggable('destroy').css({left:'', top: ''});\n",
-       "                }\n",
-       "\n",
-       "                me.$el.attr('qcodes-state', state);\n",
-       "\n",
-       "                if(state === 'floated') {\n",
-       "                    me.$el\n",
-       "                        .draggable({stop: function() { me.clipBounds(); }})\n",
-       "                        .css({\n",
-       "                            left: window.innerWidth - me.$el.width() - 15,\n",
-       "                            top: window.innerHeight - me.$el.height() - 10\n",
-       "                        });\n",
-       "                }\n",
-       "\n",
-       "                // any previous highlighting is now moot\n",
-       "                me.$el.removeClass('qcodes-highlight');\n",
-       "            }, 0);\n",
-       "\n",
-       "        },\n",
-       "\n",
-       "        clipBounds: function() {\n",
-       "            var me = this;\n",
-       "            if(me.$el.attr('qcodes-state') === 'floated') {\n",
-       "                var bounds = me.$el[0].getBoundingClientRect(),\n",
-       "                    minVis = 40,\n",
-       "                    maxLeft = window.innerWidth - minVis,\n",
-       "                    minLeft = minVis - bounds.width,\n",
-       "                    maxTop = window.innerHeight - minVis;\n",
-       "\n",
-       "                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);\n",
-       "                else if(bounds.left < minLeft) me.$el.css('left', minLeft);\n",
-       "\n",
-       "                if(bounds.top > maxTop) me.$el.css('top', maxTop);\n",
-       "                else if(bounds.top < 0) me.$el.css('top', 0);\n",
-       "            }\n",
-       "        },\n",
-       "\n",
-       "        display: function(message) {\n",
-       "            var me = this;\n",
-       "            if(message) {\n",
-       "                var initialScroll = me.outputArea.scrollTop();\n",
-       "                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));\n",
-       "                var scrollBottom = me.outputArea.scrollTop();\n",
-       "\n",
-       "                if(me.$el.attr('qcodes-state') === 'minimized') {\n",
-       "                    // if we add text and the box is minimized, highlight the\n",
-       "                    // title bar to alert the user that there are new messages.\n",
-       "                    // remove then add the class, so we get the animation again\n",
-       "                    // if it's already highlighted\n",
-       "                    me.$el.removeClass('qcodes-highlight');\n",
-       "                    setTimeout(function(){\n",
-       "                        me.$el.addClass('qcodes-highlight');\n",
-       "                    }, 0);\n",
-       "                }\n",
-       "\n",
-       "                var newLines = message.split('\\n'),\n",
-       "                    out = me.outputLines,\n",
-       "                    outLen = out.length;\n",
-       "                if(outLen) out[outLen - 1] += newLines[0];\n",
-       "                else out.push(newLines[0]);\n",
-       "\n",
-       "                for(var i = 1; i < newLines.length; i++) {\n",
-       "                    out.push(newLines[i]);\n",
-       "                }\n",
-       "\n",
-       "                if(out.length > me.maxOutputLength) {\n",
-       "                    out.splice(0, out.length - me.maxOutputLength + 1,\n",
-       "                        '<<< Output clipped >>>');\n",
-       "                }\n",
-       "\n",
-       "                me.outputArea.text(out.join('\\n'));\n",
-       "                me.clearButton.removeClass('disabled');\n",
-       "\n",
-       "                // if we were scrolled to the bottom initially, make sure\n",
-       "                // we stay that way.\n",
-       "                me.outputArea.scrollTop(initialScroll === scrollBottom ?\n",
-       "                    me.outputArea.prop('scrollHeight') : initialScroll);\n",
-       "            }\n",
-       "\n",
-       "            me.showSubprocesses();\n",
-       "            me.updateState();\n",
-       "        },\n",
-       "\n",
-       "        showSubprocesses: function() {\n",
-       "            var me = this,\n",
-       "                replacer = me.subprocessesMultiline ? '
' : ', ',\n",
-       "                processes = (me.model.get('_processes') || '')\n",
-       "                    .replace(/\\n/g, '>' + replacer + '<');\n",
-       "\n",
-       "            if(processes) processes = '<' + processes + '>';\n",
-       "            else processes = 'No subprocesses';\n",
-       "\n",
-       "            me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1);\n",
-       "\n",
-       "            me.subprocessList.html(processes);\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView);\n",
-       "});\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       ""
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "%matplotlib nbagg\n",
     "import numpy as np\n",
@@ -355,7 +20,9 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -381,7 +48,9 @@
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -392,7 +61,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Start a loop and generate data from dummy instruments "
    ]
@@ -401,7 +73,9 @@
    "cell_type": "code",
    "execution_count": 4,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -412,7 +86,9 @@
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -440,10 +116,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "metadata": {
     "collapsed": false,
-    "scrolled": true
+    "deletable": true,
+    "editable": true,
+    "scrolled": false
    },
    "outputs": [
     {
@@ -451,27 +129,27 @@
      "output_type": "stream",
      "text": [
       "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '2016-09-04/13-24-02_MockParabola_run'\n",
+      "   location = 'data/2017-03-10/#017_MockParabola_run_15-44-02'\n",
       "      |                    |     | \n",
       "   Setpoint | MockParabola_x_set           | x               | (10,)\n",
       "   Measured | MockParabola_skewed_parabola | skewed_parabola | (10,)\n",
-      "started at 2016-09-04 13:24:02\n"
+      "started at 2017-03-10 15:44:02\n"
      ]
     }
    ],
    "source": [
     "loop = qc.Loop(station.MockParabola.x[-100:100:20]).each(station.MockParabola.skewed_parabola)\n",
-    "data_l = loop.run(name='MockParabola_run', formatter=qc.data.gnuplot_format.GNUPlotFormat(), \n",
-    "                  background=False, data_manager=False)\n",
+    "data_l = loop.run(name='MockParabola_run', formatter=qc.data.gnuplot_format.GNUPlotFormat())\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -479,13 +157,12 @@
      "output_type": "stream",
      "text": [
       "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '2016-09-04/13-24-02_MockParabola_run_002'\n",
+      "   location = 'data/2017-03-10/#018_MockParabola_run_15-44-07'\n",
       "      |                    |     | \n",
       "   Setpoint | MockParabola_x_set           | x               | (10,)\n",
       "   Setpoint | MockParabola_y_set           | y               | (10, 15)\n",
       "   Measured | MockParabola_skewed_parabola | skewed_parabola | (10, 15)\n",
-      "started at 2016-09-04 13:24:03\n"
+      "started at 2017-03-10 15:44:07\n"
      ]
     }
    ],
@@ -494,14 +171,16 @@
     "h5fmt = hdf5_format.HDF5Format()\n",
     "loop = qc.Loop(station.MockParabola.x[-100:100:20]).loop(\n",
     "    station.MockParabola.y[-100:50:10]).each(station.MockParabola.skewed_parabola)\n",
-    "data_l = loop.run(name='MockParabola_run', formatter=h5fmt, background=False, data_manager=False)\n"
+    "data_l = loop.run(name='MockParabola_run', formatter=h5fmt)\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -515,16 +194,21 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Run the tests for the dataformat"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "metadata": {
     "collapsed": false,
+    "deletable": true,
+    "editable": true,
     "scrolled": false
    },
    "outputs": [
@@ -545,7 +229,22 @@
       "test_incremental_write (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
       "test_loop_writing (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... root - WARNING - Error getting or interpreting *IDN?: ''\n",
       "ok\n",
-      "test_loop_writing_2D (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... root - WARNING - Error getting or interpreting *IDN?: ''\n"
+      "test_loop_writing_2D (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... root - WARNING - Error getting or interpreting *IDN?: ''\n",
+      "ok\n",
+      "test_metadata_write_read (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
+      "test_read_writing_dicts_withlists_to_hdf5 (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
+      "test_reading_into_existing_data_array (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
+      "test_str_to_bool (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
+      "test_writing_metadata (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
+      "test_writing_unsupported_types_to_hdf5 (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... root - WARNING - List of type \"\" for \"list_of_dataset\":\"[DataSet:\n",
+      "   location = '/Users/unga/src/Qcodes/qcodes/unittest_data/2017-03-10/#016_test_missing_attr_15-44-09'\n",
+      "    |  |  | , DataSet:\n",
+      "   location = '/Users/unga/src/Qcodes/qcodes/unittest_data/2017-03-10/#016_test_missing_attr_15-44-09'\n",
+      "    |  |  | ]\" not supported, storing as string\n",
+      "root - WARNING - Type \"\" for \"nested_dataset\":\"DataSet:\n",
+      "   location = '/Users/unga/src/Qcodes/qcodes/unittest_data/2017-03-10/#016_test_missing_attr_15-44-09'\n",
+      "    |  |  | \" not supported, storing as string\n",
+      "root - WARNING - List of mixed type for \"\":\"list_of_mixed_type\" not supported, storing as string\n"
      ]
     },
     {
@@ -553,60 +252,28 @@
      "output_type": "stream",
      "text": [
       "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '/Users/Adriaan/GitHubRepos/DiCarloLab_Repositories/Qcodes/qcodes/unittest_data/2016-09-04/#631_MockLoop_hdf5_test_13-24-03'\n",
+      "   location = '/Users/unga/src/Qcodes/qcodes/unittest_data/2017-03-10/#010_MockLoop_hdf5_test_15-44-09'\n",
       "      |                         |     | \n",
       "   Setpoint | Loop_writing_test_x_set           | x               | (10,)\n",
       "   Measured | Loop_writing_test_skewed_parabola | skewed_parabola | (10,)\n",
-      "started at 2016-09-04 13:24:04\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "ok\n",
-      "test_metadata_write_read (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
-      "test_read_writing_dicts_withlists_to_hdf5 (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
-      "test_reading_into_existing_data_array (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... "
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "started at 2017-03-10 15:44:09\n",
       "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '/Users/Adriaan/GitHubRepos/DiCarloLab_Repositories/Qcodes/qcodes/unittest_data/2016-09-04/#632_MockLoop_hdf5_test_13-24-04'\n",
+      "   location = '/Users/unga/src/Qcodes/qcodes/unittest_data/2017-03-10/#011_MockLoop_hdf5_test_15-44-09'\n",
       "      |                            |     | \n",
       "   Setpoint | Loop_writing_test_2D_x_set           | x               | (10,)\n",
       "   Setpoint | Loop_writing_test_2D_y_set           | y               | (10, 10)\n",
       "   Measured | Loop_writing_test_2D_skewed_parabola | skewed_parabola | (10, 10)\n",
-      "started at 2016-09-04 13:24:04\n"
+      "started at 2017-03-10 15:44:09\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "ok\n",
-      "test_str_to_bool (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... ok\n",
-      "test_writing_unsupported_types_to_hdf5 (qcodes.tests.test_hdf5formatter.TestHDF5_Format) ... root - WARNING - List of mixed type for \"\":\"list_of_mixed_type\" not supported, storing as string\n",
-      "root - WARNING - List of type \"\" for \"list_of_dataset\":\"[DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '/Users/Adriaan/GitHubRepos/DiCarloLab_Repositories/Qcodes/qcodes/unittest_data/2016-09-04/#636_test_missing_attr_13-24-04'\n",
-      "    |  |  | , DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '/Users/Adriaan/GitHubRepos/DiCarloLab_Repositories/Qcodes/qcodes/unittest_data/2016-09-04/#636_test_missing_attr_13-24-04'\n",
-      "    |  |  | ]\" not supported, storing as string\n",
-      "root - WARNING - Type \"\" for \"nested_dataset\":\"DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '/Users/Adriaan/GitHubRepos/DiCarloLab_Repositories/Qcodes/qcodes/unittest_data/2016-09-04/#636_test_missing_attr_13-24-04'\n",
-      "    |  |  | \" not supported, storing as string\n",
       "ok\n",
       "\n",
       "----------------------------------------------------------------------\n",
-      "Ran 16 tests in 1.970s\n",
+      "Ran 17 tests in 0.348s\n",
       "\n",
       "OK\n"
      ]
@@ -624,6 +291,15 @@
     "suite = unittest.defaultTestLoader.loadTestsFromTestCase(tst)\n",
     "result = unittest.TextTestRunner(verbosity=2).run(suite)\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -642,7 +318,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "version": "3.5.2"
   },
   "widgets": {
    "state": {},
diff --git a/docs/examples/Measure without a Loop.ipynb b/docs/examples/Measure without a Loop.ipynb
index 828db49fce4..f59aeebab11 100644
--- a/docs/examples/Measure without a Loop.ipynb	
+++ b/docs/examples/Measure without a Loop.ipynb	
@@ -2,7 +2,10 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Measure without a Loop\n",
     "\n",
@@ -13,358 +16,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [
-    {
-     "data": {
-      "application/javascript": [
-       "/*\n",
-       " * Qcodes Jupyter/IPython widgets\n",
-       " */\n",
-       "require([\n",
-       "    'nbextensions/widgets/widgets/js/widget',\n",
-       "    'nbextensions/widgets/widgets/js/manager'\n",
-       "], function (widget, manager) {\n",
-       "\n",
-       "    var UpdateView = widget.DOMWidgetView.extend({\n",
-       "        render: function() {\n",
-       "            window.MYWIDGET = this;\n",
-       "            this._interval = 0;\n",
-       "            this.update();\n",
-       "        },\n",
-       "        update: function() {\n",
-       "            this.display(this.model.get('_message'));\n",
-       "            this.setInterval();\n",
-       "        },\n",
-       "        display: function(message) {\n",
-       "            /*\n",
-       "             * display method: override this for custom display logic\n",
-       "             */\n",
-       "            this.el.innerHTML = message;\n",
-       "        },\n",
-       "        remove: function() {\n",
-       "            clearInterval(this._updater);\n",
-       "        },\n",
-       "        setInterval: function(newInterval) {\n",
-       "            var me = this;\n",
-       "            if(newInterval===undefined) newInterval = me.model.get('interval');\n",
-       "            if(newInterval===me._interval) return;\n",
-       "\n",
-       "            me._interval = newInterval;\n",
-       "\n",
-       "            if(me._updater) clearInterval(me._updater);\n",
-       "\n",
-       "            if(me._interval) {\n",
-       "                me._updater = setInterval(function() {\n",
-       "                    me.send({myupdate: true});\n",
-       "                    if(!me.model.comm_live) {\n",
-       "                        console.log('missing comm, canceling widget updates', me);\n",
-       "                        clearInterval(me._updater);\n",
-       "                    }\n",
-       "                }, me._interval * 1000);\n",
-       "            }\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('UpdateView', UpdateView);\n",
-       "\n",
-       "    var HiddenUpdateView = UpdateView.extend({\n",
-       "        display: function(message) {\n",
-       "            this.$el.hide();\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView);\n",
-       "\n",
-       "    var SubprocessView = UpdateView.extend({\n",
-       "        render: function() {\n",
-       "            var me = this;\n",
-       "            me._interval = 0;\n",
-       "            me._minimize = '';\n",
-       "            me._restore = '';\n",
-       "\n",
-       "            // max lines of output to show\n",
-       "            me.maxOutputLength = 500;\n",
-       "\n",
-       "            // in case there is already an outputView present,\n",
-       "            // like from before restarting the kernel\n",
-       "            $('.qcodes-output-view').not(me.$el).remove();\n",
-       "\n",
-       "            me.$el\n",
-       "                .addClass('qcodes-output-view')\n",
-       "                .attr('qcodes-state', 'docked')\n",
-       "                .html(\n",
-       "                    '
' +\n",
-       "                        '
' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                    '
' +\n",
-       "                    '
'\n",
-       "                );\n",
-       "\n",
-       "            me.clearButton = me.$el.find('.qcodes-clear-output');\n",
-       "            me.minButton = me.$el.find('.qcodes-minimize');\n",
-       "            me.outputArea = me.$el.find('pre');\n",
-       "            me.subprocessList = me.$el.find('.qcodes-process-list');\n",
-       "            me.abortButton = me.$el.find('.qcodes-abort-loop');\n",
-       "            me.processLinesButton = me.$el.find('.qcodes-processlines')\n",
-       "\n",
-       "            me.outputLines = [];\n",
-       "\n",
-       "            me.clearButton.click(function() {\n",
-       "                me.outputArea.html('');\n",
-       "                me.clearButton.addClass('disabled');\n",
-       "            });\n",
-       "\n",
-       "            me.abortButton.click(function() {\n",
-       "                me.send({abort: true});\n",
-       "            });\n",
-       "\n",
-       "            me.processLinesButton.click(function() {\n",
-       "                // toggle multiline process list display\n",
-       "                me.subprocessesMultiline = !me.subprocessesMultiline;\n",
-       "                me.showSubprocesses();\n",
-       "            });\n",
-       "\n",
-       "            me.$el.find('.js-state').click(function() {\n",
-       "                var state = this.className.substr(this.className.indexOf('qcodes'))\n",
-       "                        .split('-')[1].split(' ')[0];\n",
-       "                me.model.set('_state', state);\n",
-       "            });\n",
-       "\n",
-       "            $(window)\n",
-       "                .off('resize.qcodes')\n",
-       "                .on('resize.qcodes', function() {me.clipBounds();});\n",
-       "\n",
-       "            me.update();\n",
-       "        },\n",
-       "\n",
-       "        updateState: function() {\n",
-       "            var me = this,\n",
-       "                oldState = me.$el.attr('qcodes-state'),\n",
-       "                state = me.model.get('_state');\n",
-       "\n",
-       "            if(state === oldState) return;\n",
-       "\n",
-       "            setTimeout(function() {\n",
-       "                // not sure why I can't pop it out of the widgetarea in render, but it seems that\n",
-       "                // some other bit of code resets the parent after render if I do it there.\n",
-       "                // To be safe, just do it on every state click.\n",
-       "                me.$el.appendTo('body');\n",
-       "\n",
-       "                if(oldState === 'floated') {\n",
-       "                    console.log('here');\n",
-       "                    me.$el.draggable('destroy').css({left:'', top: ''});\n",
-       "                }\n",
-       "\n",
-       "                me.$el.attr('qcodes-state', state);\n",
-       "\n",
-       "                if(state === 'floated') {\n",
-       "                    me.$el\n",
-       "                        .draggable({stop: function() { me.clipBounds(); }})\n",
-       "                        .css({\n",
-       "                            left: window.innerWidth - me.$el.width() - 15,\n",
-       "                            top: window.innerHeight - me.$el.height() - 10\n",
-       "                        });\n",
-       "                }\n",
-       "\n",
-       "                // any previous highlighting is now moot\n",
-       "                me.$el.removeClass('qcodes-highlight');\n",
-       "            }, 0);\n",
-       "\n",
-       "        },\n",
-       "\n",
-       "        clipBounds: function() {\n",
-       "            var me = this;\n",
-       "            if(me.$el.attr('qcodes-state') === 'floated') {\n",
-       "                var bounds = me.$el[0].getBoundingClientRect(),\n",
-       "                    minVis = 40,\n",
-       "                    maxLeft = window.innerWidth - minVis,\n",
-       "                    minLeft = minVis - bounds.width,\n",
-       "                    maxTop = window.innerHeight - minVis;\n",
-       "\n",
-       "                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);\n",
-       "                else if(bounds.left < minLeft) me.$el.css('left', minLeft);\n",
-       "\n",
-       "                if(bounds.top > maxTop) me.$el.css('top', maxTop);\n",
-       "                else if(bounds.top < 0) me.$el.css('top', 0);\n",
-       "            }\n",
-       "        },\n",
-       "\n",
-       "        display: function(message) {\n",
-       "            var me = this;\n",
-       "            if(message) {\n",
-       "                var initialScroll = me.outputArea.scrollTop();\n",
-       "                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));\n",
-       "                var scrollBottom = me.outputArea.scrollTop();\n",
-       "\n",
-       "                if(me.$el.attr('qcodes-state') === 'minimized') {\n",
-       "                    // if we add text and the box is minimized, highlight the\n",
-       "                    // title bar to alert the user that there are new messages.\n",
-       "                    // remove then add the class, so we get the animation again\n",
-       "                    // if it's already highlighted\n",
-       "                    me.$el.removeClass('qcodes-highlight');\n",
-       "                    setTimeout(function(){\n",
-       "                        me.$el.addClass('qcodes-highlight');\n",
-       "                    }, 0);\n",
-       "                }\n",
-       "\n",
-       "                var newLines = message.split('\\n'),\n",
-       "                    out = me.outputLines,\n",
-       "                    outLen = out.length;\n",
-       "                if(outLen) out[outLen - 1] += newLines[0];\n",
-       "                else out.push(newLines[0]);\n",
-       "\n",
-       "                for(var i = 1; i < newLines.length; i++) {\n",
-       "                    out.push(newLines[i]);\n",
-       "                }\n",
-       "\n",
-       "                if(out.length > me.maxOutputLength) {\n",
-       "                    out.splice(0, out.length - me.maxOutputLength + 1,\n",
-       "                        '<<< Output clipped >>>');\n",
-       "                }\n",
-       "\n",
-       "                me.outputArea.text(out.join('\\n'));\n",
-       "                me.clearButton.removeClass('disabled');\n",
-       "\n",
-       "                // if we were scrolled to the bottom initially, make sure\n",
-       "                // we stay that way.\n",
-       "                me.outputArea.scrollTop(initialScroll === scrollBottom ?\n",
-       "                    me.outputArea.prop('scrollHeight') : initialScroll);\n",
-       "            }\n",
-       "\n",
-       "            me.showSubprocesses();\n",
-       "            me.updateState();\n",
-       "        },\n",
-       "\n",
-       "        showSubprocesses: function() {\n",
-       "            var me = this,\n",
-       "                replacer = me.subprocessesMultiline ? '
' : ', ',\n",
-       "                processes = (me.model.get('_processes') || '')\n",
-       "                    .replace(/\\n/g, '>' + replacer + '<');\n",
-       "\n",
-       "            if(processes) processes = '<' + processes + '>';\n",
-       "            else processes = 'No subprocesses';\n",
-       "\n",
-       "            me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1);\n",
-       "\n",
-       "            me.subprocessList.html(processes);\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView);\n",
-       "});\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       ""
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "# setup\n",
     "%matplotlib nbagg\n",
-    "import numpy as np\n",
-    "import qcodes as qc"
+    "import qcodes as qc\n",
+    "# import dummy driver for the tutorial\n",
+    "from qcodes.tests.instrument_mocks import DummyInstrument\n",
+    "from qcodes.instrument.mock import ArrayGetter\n",
+    "\n",
+    "dac1 = DummyInstrument(name=\"dac\")\n",
+    "dac2 = DummyInstrument(name=\"dac2\")\n",
+    "# the default dummy instrument returns always a constant value, in the following line we make it random \n",
+    "# just for the looks 💅\n",
+    "import random\n",
+    "dac2.dac2.get =  lambda: random.randint(0,100)\n",
+    "\n",
+    "# The station is a container for all instruments that makes it easy \n",
+    "# to log meta-data\n",
+    "station = qc.Station(dac1, dac2)"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## Instantiates all the instruments needed for the demo\n",
     "\n",
@@ -372,34 +55,11 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 2,
+   "cell_type": "markdown",
    "metadata": {
-    "collapsed": true
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [],
-   "source": [
-    "from toymodel import AModel, MockGates, MockSource, MockMeter, AverageAndRaw\n",
-    "from qcodes.instrument.mock import ArrayGetter\n",
-    "\n",
-    "# now create this \"experiment\", note that all these are instruments \n",
-    "model = AModel()\n",
-    "gates = MockGates('gates', model=model)\n",
-    "source = MockSource('source', model=model)\n",
-    "meter = MockMeter('meter', model=model)\n",
-    "\n",
-    "# The station is a container for all instruments that makes it easy \n",
-    "# to log meta-data\n",
-    "station = qc.Station(gates, source, meter)\n",
-    "\n",
-    "# it's nice to have the key parameters be part of the global namespace\n",
-    "# that way they're objects that we can easily set, get, and slice\n",
-    "c0, c1, c2, vsd = gates.chan0, gates.chan1, gates.chan2, source.amplitude"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "### Only array output\n",
     "The arguments to Measure are all the same actions you use in a Loop.\n",
@@ -408,74 +68,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '2016-10-20/12-14-31'\n",
-      "      |           |  | \n",
-      "   Measured | chan2_1             | chan2        | (100,)\n",
-      "   Measured | meter_amplitude_1_0 | amplitude    | (100,)\n",
-      "   Measured | chan2_3             | chan2        | (100,)\n",
-      "   Measured | meter_amplitude_3_0 | amplitude    | (100,)\n",
-      "acquired at 2016-10-20 12:14:31\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "data = qc.Measure(\n",
-    "    qc.Task(c0.set, 0),\n",
-    "    ArrayGetter(meter.amplitude, c2[-10:10:0.2], 0.001),\n",
-    "    qc.Task(c0.set, 2),\n",
-    "    ArrayGetter(meter.amplitude, c2[-10:10:0.2], 0.001)\n",
+    "    qc.Task(dac1.dac1.set, 0),\n",
+    "    ArrayGetter(dac1.dac2, dac1.dac3.sweep(-10,10,0.2), 0.001),\n",
+    "    qc.Task(dac1.dac1.set, 2),\n",
+    "    ArrayGetter(dac1.dac2, dac1.dac3.sweep(-10,10,0.2), 0.001),\n",
     ").run()"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Output with scalars\n",
-    "Scalars still get treated as data, but it's not exactly natural:\n",
-    "- They still get put into arrays, just 1D 1-element arrays.\n",
-    "- They need a setpoint array, at least for now, so we make a fake one \"single_set.\" There's something nice about this, actually, in that it makes it absolutely clear that these items are not really arrays."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "collapsed": false
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = '2016-10-20/12-14-35'\n",
-      "      |           |   | \n",
-      "   Setpoint | single_set          | single        | (1,)\n",
-      "   Measured | gates_chan0         | chan0         | (1,)\n",
-      "   Measured | gates_chan1         | chan1         | (1,)\n",
-      "   Measured | chan2               | chan2         | (100,)\n",
-      "   Measured | meter_amplitude     | amplitude     | (100,)\n",
-      "   Measured | meter_avg_amplitude | avg_amplitude | (1,)\n",
-      "acquired at 2016-10-20 12:14:35\n"
-     ]
-    }
-   ],
-   "source": [
-    "data = qc.Measure(c0, c1, AverageAndRaw(meter.amplitude, c2[-10:10:0.2], 0.001)).run()"
-   ]
   }
  ],
  "metadata": {
@@ -494,7 +101,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.0"
+   "version": "3.5.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/examples/Metadata with instruments.ipynb b/docs/examples/Metadata with instruments.ipynb
index 8b5905412aa..21cf0b24cb1 100644
--- a/docs/examples/Metadata with instruments.ipynb	
+++ b/docs/examples/Metadata with instruments.ipynb	
@@ -4,7 +4,9 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -247,19 +249,16 @@
    ],
    "source": [
     "from pprint import pprint\n",
-    "\n",
-    "import qcodes as qc\n",
-    "\n",
-    "qc.set_mp_method('spawn')\n",
-    "\n",
-    "qc.show_subprocess_widget()"
+    "import qcodes as qc"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -294,6 +293,8 @@
    "execution_count": 3,
    "metadata": {
     "collapsed": false,
+    "deletable": true,
+    "editable": true,
     "scrolled": false
    },
    "outputs": [
@@ -647,6 +648,8 @@
    "execution_count": 4,
    "metadata": {
     "collapsed": false,
+    "deletable": true,
+    "editable": true,
     "scrolled": false
    },
    "outputs": [],
@@ -662,7 +665,9 @@
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -684,7 +689,9 @@
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -711,7 +718,9 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": []
@@ -733,7 +742,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "version": "3.5.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/examples/Parameters.ipynb b/docs/examples/Parameters.ipynb
index b279762f077..722840217ba 100644
--- a/docs/examples/Parameters.ipynb
+++ b/docs/examples/Parameters.ipynb
@@ -2,7 +2,10 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Parameters in QCoDeS"
    ]
@@ -11,7 +14,9 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
@@ -21,7 +26,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "QCoDeS provides 5 classes of parameter built in. Three base classes (which must be subclassed to use):\n",
     "- `Parameter` represents a single value at a time\n",
@@ -40,7 +48,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## Parameter\n",
     "Most of the time you can use `StandardParameter` directly; even if you have custom `get`/`set` functions, but sometimes it's useful to subclass `Parameter`:"
@@ -50,7 +61,9 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -94,7 +107,9 @@
    "cell_type": "code",
    "execution_count": 3,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -130,7 +145,9 @@
    "cell_type": "code",
    "execution_count": 4,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -166,7 +183,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## ArrayParameter\n",
     "For actions that create a whole array of values at once. When you use it in a `Loop`, it makes a single `DataArray` with the array returned by `get` nested inside extra dimension(s) for the loop.\n",
@@ -178,7 +198,9 @@
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -223,7 +245,9 @@
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -261,7 +285,9 @@
    "cell_type": "code",
    "execution_count": 7,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -336,7 +362,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## MultiParameter\n",
     "Return multiple items at once, where each item can be a single value or an array. When you use it in a `Loop`, it makes a separate `DataArray` for each item.\n",
@@ -350,7 +379,9 @@
    "cell_type": "code",
    "execution_count": 8,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -391,7 +422,9 @@
    "cell_type": "code",
    "execution_count": 9,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -427,7 +460,9 @@
    "cell_type": "code",
    "execution_count": 10,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -467,7 +502,9 @@
    "cell_type": "code",
    "execution_count": 11,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [
     {
@@ -546,7 +583,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.0"
+   "version": "3.5.2"
   }
  },
  "nbformat": 4,
diff --git a/docs/examples/Tutorial.ipynb b/docs/examples/Tutorial.ipynb
index 97a4a71dc4a..11464a3a71a 100644
--- a/docs/examples/Tutorial.ipynb
+++ b/docs/examples/Tutorial.ipynb
@@ -2,7 +2,10 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# QCoDeS tutorial \n",
     "Basic overview of QCoDeS"
@@ -10,7 +13,10 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## Typical QCodes workflow \n",
     "1. Start up an interactive python session (e.g. using jupyter) \n",
@@ -21,415 +27,84 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "collapsed": false,
+    "deletable": true,
+    "editable": true,
     "scrolled": false
    },
-   "outputs": [
-    {
-     "data": {
-      "application/javascript": [
-       "/*\n",
-       " * Qcodes Jupyter/IPython widgets\n",
-       " */\n",
-       "require([\n",
-       "    'nbextensions/widgets/widgets/js/widget',\n",
-       "    'nbextensions/widgets/widgets/js/manager'\n",
-       "], function (widget, manager) {\n",
-       "\n",
-       "    var UpdateView = widget.DOMWidgetView.extend({\n",
-       "        render: function() {\n",
-       "            window.MYWIDGET = this;\n",
-       "            this._interval = 0;\n",
-       "            this.update();\n",
-       "        },\n",
-       "        update: function() {\n",
-       "            this.display(this.model.get('_message'));\n",
-       "            this.setInterval();\n",
-       "        },\n",
-       "        display: function(message) {\n",
-       "            /*\n",
-       "             * display method: override this for custom display logic\n",
-       "             */\n",
-       "            this.el.innerHTML = message;\n",
-       "        },\n",
-       "        remove: function() {\n",
-       "            clearInterval(this._updater);\n",
-       "        },\n",
-       "        setInterval: function(newInterval) {\n",
-       "            var me = this;\n",
-       "            if(newInterval===undefined) newInterval = me.model.get('interval');\n",
-       "            if(newInterval===me._interval) return;\n",
-       "\n",
-       "            me._interval = newInterval;\n",
-       "\n",
-       "            if(me._updater) clearInterval(me._updater);\n",
-       "\n",
-       "            if(me._interval) {\n",
-       "                me._updater = setInterval(function() {\n",
-       "                    me.send({myupdate: true});\n",
-       "                    if(!me.model.comm_live) {\n",
-       "                        console.log('missing comm, canceling widget updates', me);\n",
-       "                        clearInterval(me._updater);\n",
-       "                    }\n",
-       "                }, me._interval * 1000);\n",
-       "            }\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('UpdateView', UpdateView);\n",
-       "\n",
-       "    var HiddenUpdateView = UpdateView.extend({\n",
-       "        display: function(message) {\n",
-       "            this.$el.hide();\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView);\n",
-       "\n",
-       "    var SubprocessView = UpdateView.extend({\n",
-       "        render: function() {\n",
-       "            var me = this;\n",
-       "            me._interval = 0;\n",
-       "            me._minimize = '';\n",
-       "            me._restore = '';\n",
-       "\n",
-       "            // max lines of output to show\n",
-       "            me.maxOutputLength = 500;\n",
-       "\n",
-       "            // in case there is already an outputView present,\n",
-       "            // like from before restarting the kernel\n",
-       "            $('.qcodes-output-view').not(me.$el).remove();\n",
-       "\n",
-       "            me.$el\n",
-       "                .addClass('qcodes-output-view')\n",
-       "                .attr('qcodes-state', 'docked')\n",
-       "                .html(\n",
-       "                    '
' +\n",
-       "                        '
' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                        '' +\n",
-       "                    '
' +\n",
-       "                    '
'\n",
-       "                );\n",
-       "\n",
-       "            me.clearButton = me.$el.find('.qcodes-clear-output');\n",
-       "            me.minButton = me.$el.find('.qcodes-minimize');\n",
-       "            me.outputArea = me.$el.find('pre');\n",
-       "            me.subprocessList = me.$el.find('.qcodes-process-list');\n",
-       "            me.abortButton = me.$el.find('.qcodes-abort-loop');\n",
-       "            me.processLinesButton = me.$el.find('.qcodes-processlines')\n",
-       "\n",
-       "            me.outputLines = [];\n",
-       "\n",
-       "            me.clearButton.click(function() {\n",
-       "                me.outputArea.html('');\n",
-       "                me.clearButton.addClass('disabled');\n",
-       "            });\n",
-       "\n",
-       "            me.abortButton.click(function() {\n",
-       "                me.send({abort: true});\n",
-       "            });\n",
-       "\n",
-       "            me.processLinesButton.click(function() {\n",
-       "                // toggle multiline process list display\n",
-       "                me.subprocessesMultiline = !me.subprocessesMultiline;\n",
-       "                me.showSubprocesses();\n",
-       "            });\n",
-       "\n",
-       "            me.$el.find('.js-state').click(function() {\n",
-       "                var state = this.className.substr(this.className.indexOf('qcodes'))\n",
-       "                        .split('-')[1].split(' ')[0];\n",
-       "                me.model.set('_state', state);\n",
-       "            });\n",
-       "\n",
-       "            $(window)\n",
-       "                .off('resize.qcodes')\n",
-       "                .on('resize.qcodes', function() {me.clipBounds();});\n",
-       "\n",
-       "            me.update();\n",
-       "        },\n",
-       "\n",
-       "        updateState: function() {\n",
-       "            var me = this,\n",
-       "                oldState = me.$el.attr('qcodes-state'),\n",
-       "                state = me.model.get('_state');\n",
-       "\n",
-       "            if(state === oldState) return;\n",
-       "\n",
-       "            setTimeout(function() {\n",
-       "                // not sure why I can't pop it out of the widgetarea in render, but it seems that\n",
-       "                // some other bit of code resets the parent after render if I do it there.\n",
-       "                // To be safe, just do it on every state click.\n",
-       "                me.$el.appendTo('body');\n",
-       "\n",
-       "                if(oldState === 'floated') {\n",
-       "                    console.log('here');\n",
-       "                    me.$el.draggable('destroy').css({left:'', top: ''});\n",
-       "                }\n",
-       "\n",
-       "                me.$el.attr('qcodes-state', state);\n",
-       "\n",
-       "                if(state === 'floated') {\n",
-       "                    me.$el\n",
-       "                        .draggable({stop: function() { me.clipBounds(); }})\n",
-       "                        .css({\n",
-       "                            left: window.innerWidth - me.$el.width() - 15,\n",
-       "                            top: window.innerHeight - me.$el.height() - 10\n",
-       "                        });\n",
-       "                }\n",
-       "\n",
-       "                // any previous highlighting is now moot\n",
-       "                me.$el.removeClass('qcodes-highlight');\n",
-       "            }, 0);\n",
-       "\n",
-       "        },\n",
-       "\n",
-       "        clipBounds: function() {\n",
-       "            var me = this;\n",
-       "            if(me.$el.attr('qcodes-state') === 'floated') {\n",
-       "                var bounds = me.$el[0].getBoundingClientRect(),\n",
-       "                    minVis = 40,\n",
-       "                    maxLeft = window.innerWidth - minVis,\n",
-       "                    minLeft = minVis - bounds.width,\n",
-       "                    maxTop = window.innerHeight - minVis;\n",
-       "\n",
-       "                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);\n",
-       "                else if(bounds.left < minLeft) me.$el.css('left', minLeft);\n",
-       "\n",
-       "                if(bounds.top > maxTop) me.$el.css('top', maxTop);\n",
-       "                else if(bounds.top < 0) me.$el.css('top', 0);\n",
-       "            }\n",
-       "        },\n",
-       "\n",
-       "        display: function(message) {\n",
-       "            var me = this;\n",
-       "            if(message) {\n",
-       "                var initialScroll = me.outputArea.scrollTop();\n",
-       "                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));\n",
-       "                var scrollBottom = me.outputArea.scrollTop();\n",
-       "\n",
-       "                if(me.$el.attr('qcodes-state') === 'minimized') {\n",
-       "                    // if we add text and the box is minimized, highlight the\n",
-       "                    // title bar to alert the user that there are new messages.\n",
-       "                    // remove then add the class, so we get the animation again\n",
-       "                    // if it's already highlighted\n",
-       "                    me.$el.removeClass('qcodes-highlight');\n",
-       "                    setTimeout(function(){\n",
-       "                        me.$el.addClass('qcodes-highlight');\n",
-       "                    }, 0);\n",
-       "                }\n",
-       "\n",
-       "                var newLines = message.split('\\n'),\n",
-       "                    out = me.outputLines,\n",
-       "                    outLen = out.length;\n",
-       "                if(outLen) out[outLen - 1] += newLines[0];\n",
-       "                else out.push(newLines[0]);\n",
-       "\n",
-       "                for(var i = 1; i < newLines.length; i++) {\n",
-       "                    out.push(newLines[i]);\n",
-       "                }\n",
-       "\n",
-       "                if(out.length > me.maxOutputLength) {\n",
-       "                    out.splice(0, out.length - me.maxOutputLength + 1,\n",
-       "                        '<<< Output clipped >>>');\n",
-       "                }\n",
-       "\n",
-       "                me.outputArea.text(out.join('\\n'));\n",
-       "                me.clearButton.removeClass('disabled');\n",
-       "\n",
-       "                // if we were scrolled to the bottom initially, make sure\n",
-       "                // we stay that way.\n",
-       "                me.outputArea.scrollTop(initialScroll === scrollBottom ?\n",
-       "                    me.outputArea.prop('scrollHeight') : initialScroll);\n",
-       "            }\n",
-       "\n",
-       "            me.showSubprocesses();\n",
-       "            me.updateState();\n",
-       "        },\n",
-       "\n",
-       "        showSubprocesses: function() {\n",
-       "            var me = this,\n",
-       "                replacer = me.subprocessesMultiline ? '
' : ', ',\n",
-       "                processes = (me.model.get('_processes') || '')\n",
-       "                    .replace(/\\n/g, '>' + replacer + '<');\n",
-       "\n",
-       "            if(processes) processes = '<' + processes + '>';\n",
-       "            else processes = 'No subprocesses';\n",
-       "\n",
-       "            me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1);\n",
-       "\n",
-       "            me.subprocessList.html(processes);\n",
-       "        }\n",
-       "    });\n",
-       "    manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView);\n",
-       "});\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/html": [
-       ""
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "%matplotlib nbagg\n",
-    "import matplotlib.pyplot as plt\n",
-    "from pprint import pprint\n",
-    "import time\n",
-    "import numpy as np\n",
     "import qcodes as qc"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "## Instantiates all the instruments needed for the demo"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
-    "# spawn doesn't like function or class definitions in the interpreter\n",
-    "# session - had to move them to a file.\n",
-    "from toymodel import AModel, MockGates, MockSource, MockMeter, AverageGetter, AverageAndRaw\n",
+    "# import dummy driver for the tutorial\n",
+    "from qcodes.tests.instrument_mocks import DummyInstrument\n",
+    "from qcodes.instrument.mock import ArrayGetter\n",
     "\n",
-    "# now create this \"experiment\", note that all these are instruments \n",
-    "model = AModel()\n",
-    "gates = MockGates('gates', model=model)\n",
-    "source = MockSource('source', model=model)\n",
-    "meter = MockMeter('meter', model=model)\n",
+    "dac1 = DummyInstrument(name=\"dac\")\n",
+    "dac2 = DummyInstrument(name=\"dac2\")\n",
+    "# the default dummy instrument returns always a constant value, in the following line we make it random \n",
+    "# just for the looks 💅\n",
+    "import random\n",
+    "dac2.dac2.get =  lambda: random.randint(0,100)\n",
     "\n",
     "# The station is a container for all instruments that makes it easy \n",
     "# to log meta-data\n",
-    "station = qc.Station(gates, source, meter)\n",
-    "\n",
-    "# it's nice to have the key parameters be part of the global namespace\n",
-    "# that way they're objects that we can easily set, get, and slice\n",
-    "c0, c1, c2, vsd = gates.chan0, gates.chan1, gates.chan2, source.amplitude"
+    "station = qc.Station(dac1, dac2)"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "### The location provider can be set globally "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
-    "# dm = qc.data.manager.get_data_manager()\n",
     "loc_provider = qc.data.location.FormatLocation(fmt='data/{date}/#{counter}_{name}_{time}')\n",
     "qc.data.data_set.DataSet.location_provider=loc_provider"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
     "# Running an experiment \n",
     "\n",
@@ -437,148 +112,40 @@
     "\n",
     "Before you run a measurement loop you do two things:\n",
     "1. You describe what parameter(s) to vary and how. This is the creation of a `Loop` object: `loop = Loop(sweep_values, ...)`\n",
-    "2. You describe what to do at each step in the loop. This is `loop.each(*actions)` which converts the `Loop` object into an `ActiveLoop` object. Actions can be:\n",
+    "2. You describe what to do at each step in the loop. This is `loop.each(*actions)` \n",
     "  - measurements (any object with a `.get` method will be interpreted as a measurement)\n",
     "  - `Task`: some callable (which can have arguments with it) to be executed each time through the loop. Does not generate data.\n",
     "  - `Wait`: a specialized `Task` just to wait a certain time.\n",
-    "  - `BreakIf`: some condition that, if it returns truthy, breaks (this level of) the loop\n",
-    "  - Another `ActiveLoop` to nest inside the outer one.\n",
-    "\n",
-    "For more details, see issue #232 docs: Write bigger picture"
+    "  - `BreakIf`: some condition that, if it returns truthy, breaks (this level of) the loop"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "WARNING:root:negative delay -0.001849 sec\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "DataSet:\n",
-      "   mode     = DataMode.LOCAL\n",
-      "   location = 'data/2016-11-08/#024_testsweep_13-29-04'\n",
-      "      |       |  | \n",
-      "   Setpoint | gates_chan0_set | chan0        | (201,)\n",
-      "   Measured | meter_amplitude | amplitude    | (201,)\n",
-      "started at 2016-11-08 13:29:23\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Notice that one can use an explicit location and `overwrite=True` here so that\n",
-    "# running this notebook over and over won't result in extra files.\n",
-    "# If you leave these out, you get a new timestamped DataSet each time.\n",
-    "\n",
-    "\n",
-    "loop = qc.Loop(c0.sweep(0,20,0.1), delay=0.001).each(meter.amplitude)\n",
-    "data = loop.get_data_set(name='testsweep')\n",
-    "plot = qc.QtPlot()\n",
-    "plot.add(data.meter_amplitude)\n",
-    "_ = loop.with_bg_task(plot.update, plot.save).run()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Output of the loop\n",
-    "Notice the **\"DataSet\"**.  \n",
-    "A loop returns a dataset.  \n",
-    "The representation of the dataset shows what arrays it contains and where it is saved.   \n",
-    "The dataset initially starts out empty (filled with NAN's) and get's filled while the Loop get's executed. "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
+   "outputs": [],
    "source": [
-    "Once the measurement is done, take a look at the file in finder/explorer (the dataset.location should give you the relative path). \n",
-    "Note also the snapshot that captures the settings of all instruments at the start of the Loop. \n",
-    "This metadata is also accesible from the dataset and captures a snapshot of each instrument listed in the station. "
+    "loop = qc.Loop(dac1.dac1.sweep(0,20,0.1), delay=0.001).each(dac2.dac2)\n",
+    "data = loop.get_data_set(name='testsweep')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'__class__': 'toymodel.MockMeter',\n",
-       " 'functions': {},\n",
-       " 'name': 'meter',\n",
-       " 'parameters': {'IDN': {'__class__': 'qcodes.instrument.parameter.StandardParameter',\n",
-       "   'instrument': 'toymodel.MockMeter',\n",
-       "   'instrument_name': 'meter',\n",
-       "   'label': 'IDN',\n",
-       "   'name': 'IDN',\n",
-       "   'ts': '2016-11-08 13:29:02',\n",
-       "   'units': '',\n",
-       "   'value': {'firmware': None,\n",
-       "    'model': 'MockMeter',\n",
-       "    'serial': 'meter',\n",
-       "    'vendor': None}},\n",
-       "  'amplitude': {'__class__': 'qcodes.instrument.parameter.StandardParameter',\n",
-       "   'instrument': 'toymodel.MockMeter',\n",
-       "   'instrument_name': 'meter',\n",
-       "   'label': 'Current (nA)',\n",
-       "   'name': 'amplitude',\n",
-       "   'ts': '2016-11-08 13:29:23',\n",
-       "   'units': '',\n",
-       "   'value': 0.117}}}"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "meter.snapshot()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Plotting the loop  II\n",
-    "\n",
-    "QCodes supports both matplotlib inline plotting and pyqtgraph for plotting. \n",
-    "For a comparison see http://pyqtgraph.org/ (actually not that biased)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The same API works for plotting a measured dataset or an old  dataset."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
+   "outputs": [],
    "source": [
-    "`DataSet` objects are not intended to be instantiated directly, but\n",
-    "rather through the helper functions:\n",
-    "- `load_data` for existing data sets, including the data currently\n",
-    "  being acquired.\n",
-    "- `new_data` to make an empty data set to be populated with new\n",
-    "  measurements or simulation data. `new_data` is called internally by\n",
-    "  `Loop.run()` so is also generally not needed directly."
+    "plot = qc.QtPlot()\n",
+    "plot.add(data.dac2_dac2)\n",
+    "_ = loop.with_bg_task(plot.update, plot.save).run()"
    ]
   },
   {
@@ -586,1024 +153,130 @@
    "execution_count": null,
    "metadata": {
     "collapsed": false,
-    "scrolled": false
+    "deletable": true,
+    "editable": true
    },
    "outputs": [],
    "source": [
-    "loaded_data = qc.load_data(\"data/2016-10-10/#002_testsweep_10-08-32\")\n",
-    "plot = qc.MatPlot(loaded_data.meter_amplitude)"
+    "plot"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Example: multiple 2D measurements with live plotting"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
    "metadata": {
-    "collapsed": true
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [],
    "source": [
-    "loop = qc.Loop(c1[-15:15:1], 0.01).loop(c0[-15:12:.5], 0.001).each(\n",
-    "    meter.amplitude, # first measurement, at c2=0 -> amplitude_0 bcs it's action 0\n",
-    "    qc.Task(c2.set, 1), # action 1 -> c2.set(1)\n",
-    "    qc.Wait(0.001),\n",
-    "    meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n",
-    "    qc.Task(c2.set, 0)\n",
-    "    )\n",
-    "data = loop.get_data_set(name='2D_test')"
+    "## Output of the loop\n",
+    "A loop returns a dataset.  \n",
+    "The representation of the dataset shows what arrays it contains and where it is saved.   \n",
+    "The dataset initially starts out empty (filled with NAN's) and get's filled while the Loop get's executed. "
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "deletable": true,
+    "editable": true
+   },
    "source": [
-    "### Plot with matplotlib "
+    "Once the measurement is done, take a look at the file in finder/explorer (the dataset.location should give you the relative path). \n",
+    "Note also the snapshot that captures the settings of all instruments at the start of the Loop. \n",
+    "This metadata is also accesible from the dataset and captures a snapshot of each instrument listed in the station. "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {
     "collapsed": false,
-    "scrolled": false
+    "deletable": true,
+    "editable": true
    },
-   "outputs": [
-    {
-     "data": {
-      "application/javascript": [
-       "/* Put everything inside the global mpl namespace */\n",
-       "window.mpl = {};\n",
-       "\n",
-       "mpl.get_websocket_type = function() {\n",
-       "    if (typeof(WebSocket) !== 'undefined') {\n",
-       "        return WebSocket;\n",
-       "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
-       "        return MozWebSocket;\n",
-       "    } else {\n",
-       "        alert('Your browser does not have WebSocket support.' +\n",
-       "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
-       "              'Firefox 4 and 5 are also supported but you ' +\n",
-       "              'have to enable WebSockets in about:config.');\n",
-       "    };\n",
-       "}\n",
-       "\n",
-       "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
-       "    this.id = figure_id;\n",
-       "\n",
-       "    this.ws = websocket;\n",
-       "\n",
-       "    this.supports_binary = (this.ws.binaryType != undefined);\n",
-       "\n",
-       "    if (!this.supports_binary) {\n",
-       "        var warnings = document.getElementById(\"mpl-warnings\");\n",
-       "        if (warnings) {\n",
-       "            warnings.style.display = 'block';\n",
-       "            warnings.textContent = (\n",
-       "                \"This browser does not support binary websocket messages. \" +\n",
-       "                    \"Performance may be slow.\");\n",
-       "        }\n",
-       "    }\n",
-       "\n",
-       "    this.imageObj = new Image();\n",
-       "\n",
-       "    this.context = undefined;\n",
-       "    this.message = undefined;\n",
-       "    this.canvas = undefined;\n",
-       "    this.rubberband_canvas = undefined;\n",
-       "    this.rubberband_context = undefined;\n",
-       "    this.format_dropdown = undefined;\n",
-       "\n",
-       "    this.image_mode = 'full';\n",
-       "\n",
-       "    this.root = $('
');\n",
-       "    this._root_extra_style(this.root)\n",
-       "    this.root.attr('style', 'display: inline-block');\n",
-       "\n",
-       "    $(parent_element).append(this.root);\n",
-       "\n",
-       "    this._init_header(this);\n",
-       "    this._init_canvas(this);\n",
-       "    this._init_toolbar(this);\n",
-       "\n",
-       "    var fig = this;\n",
-       "\n",
-       "    this.waiting = false;\n",
-       "\n",
-       "    this.ws.onopen =  function () {\n",
-       "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
-       "            fig.send_message(\"send_image_mode\", {});\n",
-       "            fig.send_message(\"refresh\", {});\n",
-       "        }\n",
-       "\n",
-       "    this.imageObj.onload = function() {\n",
-       "            if (fig.image_mode == 'full') {\n",
-       "                // Full images could contain transparency (where diff images\n",
-       "                // almost always do), so we need to clear the canvas so that\n",
-       "                // there is no ghosting.\n",
-       "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
-       "            }\n",
-       "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
-       "        };\n",
-       "\n",
-       "    this.imageObj.onunload = function() {\n",
-       "        this.ws.close();\n",
-       "    }\n",
-       "\n",
-       "    this.ws.onmessage = this._make_on_message_function(this);\n",
-       "\n",
-       "    this.ondownload = ondownload;\n",
-       "}\n",
-       "\n",
-       "mpl.figure.prototype._init_header = function() {\n",
-       "    var titlebar = $(\n",
-       "        '
');\n",
-       "    var titletext = $(\n",
-       "        '
');\n",
-       "    titlebar.append(titletext)\n",
-       "    this.root.append(titlebar);\n",
-       "    this.header = titletext[0];\n",
-       "}\n",
-       "\n",
-       "\n",
-       "\n",
-       "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
-       "\n",
-       "}\n",
-       "\n",
-       "\n",
-       "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
-       "\n",
-       "}\n",
-       "\n",
-       "mpl.figure.prototype._init_canvas = function() {\n",
-       "    var fig = this;\n",
-       "\n",
-       "    var canvas_div = $('
');\n",
-       "\n",
-       "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
-       "\n",
-       "    function canvas_keyboard_event(event) {\n",
-       "        return fig.key_event(event, event['data']);\n",
-       "    }\n",
-       "\n",
-       "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
-       "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
-       "    this.canvas_div = canvas_div\n",
-       "    this._canvas_extra_style(canvas_div)\n",
-       "    this.root.append(canvas_div);\n",
-       "\n",
-       "    var canvas = $('');\n",
-       "    canvas.addClass('mpl-canvas');\n",
-       "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
-       "\n",
-       "    this.canvas = canvas[0];\n",
-       "    this.context = canvas[0].getContext(\"2d\");\n",
-       "\n",
-       "    var rubberband = $('');\n",
-       "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
-       "\n",
-       "    var pass_mouse_events = true;\n",
-       "\n",
-       "    canvas_div.resizable({\n",
-       "        start: function(event, ui) {\n",
-       "            pass_mouse_events = false;\n",
-       "        },\n",
-       "        resize: function(event, ui) {\n",
-       "            fig.request_resize(ui.size.width, ui.size.height);\n",
-       "        },\n",
-       "        stop: function(event, ui) {\n",
-       "            pass_mouse_events = true;\n",
-       "            fig.request_resize(ui.size.width, ui.size.height);\n",
-       "        },\n",
-       "    });\n",
-       "\n",
-       "    function mouse_event_fn(event) {\n",
-       "        if (pass_mouse_events)\n",
-       "            return fig.mouse_event(event, event['data']);\n",
-       "    }\n",
-       "\n",
-       "    rubberband.mousedown('button_press', mouse_event_fn);\n",
-       "    rubberband.mouseup('button_release', mouse_event_fn);\n",
-       "    // Throttle sequential mouse events to 1 every 20ms.\n",
-       "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
-       "\n",
-       "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
-       "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
-       "\n",
-       "    canvas_div.on(\"wheel\", function (event) {\n",
-       "        event = event.originalEvent;\n",
-       "        event['data'] = 'scroll'\n",
-       "        if (event.deltaY < 0) {\n",
-       "            event.step = 1;\n",
-       "        } else {\n",
-       "            event.step = -1;\n",
-       "        }\n",
-       "        mouse_event_fn(event);\n",
-       "    });\n",
-       "\n",
-       "    canvas_div.append(canvas);\n",
-       "    canvas_div.append(rubberband);\n",
-       "\n",
-       "    this.rubberband = rubberband;\n",
-       "    this.rubberband_canvas = rubberband[0];\n",
-       "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
-       "    this.rubberband_context.strokeStyle = \"#000000\";\n",
-       "\n",
-       "    this._resize_canvas = function(width, height) {\n",
-       "        // Keep the size of the canvas, canvas container, and rubber band\n",
-       "        // canvas in synch.\n",
-       "        canvas_div.css('width', width)\n",
-       "        canvas_div.css('height', height)\n",
-       "\n",
-       "        canvas.attr('width', width);\n",
-       "        canvas.attr('height', height);\n",
-       "\n",
-       "        rubberband.attr('width', width);\n",
-       "        rubberband.attr('height', height);\n",
-       "    }\n",
-       "\n",
-       "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
-       "    // upon first draw.\n",
-       "    this._resize_canvas(600, 600);\n",
-       "\n",
-       "    // Disable right mouse context menu.\n",
-       "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
-       "        return false;\n",
-       "    });\n",
-       "\n",
-       "    function set_focus () {\n",
-       "        canvas.focus();\n",
-       "        canvas_div.focus();\n",
-       "    }\n",
-       "\n",
-       "    window.setTimeout(set_focus, 100);\n",
-       "}\n",
-       "\n",
-       "mpl.figure.prototype._init_toolbar = function() {\n",
-       "    var fig = this;\n",
-       "\n",
-       "    var nav_element = $('
')\n",
-       "    nav_element.attr('style', 'width: 100%');\n",
-       "    this.root.append(nav_element);\n",
-       "\n",
-       "    // Define a callback function for later on.\n",
-       "    function toolbar_event(event) {\n",
-       "        return fig.toolbar_button_onclick(event['data']);\n",
-       "    }\n",
-       "    function toolbar_mouse_event(event) {\n",
-       "        return fig.toolbar_button_onmouseover(event['data']);\n",
-       "    }\n",
-       "\n",
-       "    for(var toolbar_ind in mpl.toolbar_items) {\n",
-       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
-       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
-       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
-       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
-       "\n",
-       "        if (!name) {\n",
-       "            // put a spacer in here.\n",
-       "            continue;\n",
-       "        }\n",
-       "        var button = $('