From f3567ec50ae6ea6563ee6d5cf1d51b38bd0640dd Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Sun, 5 Mar 2017 22:51:49 +0100 Subject: [PATCH 01/36] remove metaclass --- qcodes/instrument/base.py | 1 + qcodes/instrument/metaclass.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/qcodes/instrument/base.py b/qcodes/instrument/base.py index 0a2ac8a4299..db7dfa0c269 100644 --- a/qcodes/instrument/base.py +++ b/qcodes/instrument/base.py @@ -95,6 +95,7 @@ def __init__(self, name, server_name=None, **kwargs): self._meta_attrs = ['name'] self._no_proxy_methods = {'__getstate__'} + self.record_instance(self) def get_idn(self): """ diff --git a/qcodes/instrument/metaclass.py b/qcodes/instrument/metaclass.py index 641028ab8df..2d5ceca65d7 100644 --- a/qcodes/instrument/metaclass.py +++ b/qcodes/instrument/metaclass.py @@ -40,6 +40,6 @@ def __call__(cls, *args, server_name=None, **kwargs): # for RemoteInstrument, we want to record this instance with the # class that it proxies, not with RemoteInstrument itself - cls.record_instance(instrument) + # cls.record_instance(instrument) return instrument From ff7b8799891e7edd052a457d9bc603c551e81ee3 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 00:21:34 +0100 Subject: [PATCH 02/36] first step in removing the metaclass --- qcodes/instrument/base.py | 1 - qcodes/instrument/metaclass.py | 7 ++++--- qcodes/tests/test_instrument.py | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/qcodes/instrument/base.py b/qcodes/instrument/base.py index db7dfa0c269..c131f7b9663 100644 --- a/qcodes/instrument/base.py +++ b/qcodes/instrument/base.py @@ -220,7 +220,6 @@ def record_instance(cls, instance): """ wr = weakref.ref(instance) name = instance.name - # First insert this instrument in the record of *all* instruments # making sure its name is unique existing_wr = cls._all_instruments.get(name) diff --git a/qcodes/instrument/metaclass.py b/qcodes/instrument/metaclass.py index 2d5ceca65d7..3e4593a29d2 100644 --- a/qcodes/instrument/metaclass.py +++ b/qcodes/instrument/metaclass.py @@ -35,11 +35,12 @@ def __call__(cls, *args, server_name=None, **kwargs): else: warnings.warn('Multiprocessing is in beta, use at own risk', UserWarning) + instrument = RemoteInstrument(*args, instrument_class=cls, server_name=server_name, **kwargs) - # for RemoteInstrument, we want to record this instance with the - # class that it proxies, not with RemoteInstrument itself - # cls.record_instance(instrument) + # for RemoteInstrument, we want to record this instance with the + # class that it proxies, not with RemoteInstrument itself + cls.record_instance(instrument) return instrument diff --git a/qcodes/tests/test_instrument.py b/qcodes/tests/test_instrument.py index 434f0ae1148..8dc5703de62 100644 --- a/qcodes/tests/test_instrument.py +++ b/qcodes/tests/test_instrument.py @@ -1,6 +1,7 @@ """ Test suite for instument.* """ +import gc from datetime import datetime, timedelta from unittest import TestCase import time @@ -211,6 +212,13 @@ def test_creation_failure(self): name = 'gatesFailing2' with self.assertRaises(ValueError): GatesBadDelayValue(model=self.model, name=name, server_name=None) + # gc is confused by the context manager we just used + # we want to make sure the ojbect we just tried to create + # but it threw an exception is properly gc'ed. Which + # is what happens wihtout the context manager AND + # it's how the __del__ method is supposed to work. + + gc.collect() # this instrument should not be in the instance list with self.assertRaises(KeyError): From 0676d7aaa62332aff2c94e162efd8ff4a52062e3 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 12:01:33 +0100 Subject: [PATCH 03/36] fix: Remove background --- qcodes/__init__.py | 2 +- qcodes/loops.py | 144 ++------------------- qcodes/tests/test_loop.py | 265 +------------------------------------- 3 files changed, 13 insertions(+), 398 deletions(-) diff --git a/qcodes/__init__.py b/qcodes/__init__.py index 859bcaf45dd..4243cf3c2cb 100644 --- a/qcodes/__init__.py +++ b/qcodes/__init__.py @@ -40,7 +40,7 @@ from qcodes.widgets.widgets import show_subprocess_widget from qcodes.station import Station -from qcodes.loops import get_bg, halt_bg, Loop +from qcodes.loops import Loop from qcodes.measure import Measure from qcodes.actions import Task, Wait, BreakIf diff --git a/qcodes/loops.py b/qcodes/loops.py index ebec726a46c..5c13f553728 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -48,18 +48,15 @@ from datetime import datetime import logging -import multiprocessing as mp import time import numpy as np import warnings -from qcodes import config from qcodes.station import Station from qcodes.data.data_set import new_data, DataMode from qcodes.data.data_array import DataArray from qcodes.data.manager import get_data_manager from qcodes.utils.helpers import wait_secs, full_class, tprint -from qcodes.process.qcodes_process import QcodesProcess from qcodes.utils.metadata import Metadatable from .actions import (_actions_snapshot, Task, Wait, _Measure, _Nest, @@ -67,71 +64,7 @@ log = logging.getLogger(__name__) -# Switches off multiprocessing by default, cant' be altered after module -USE_MP = config.core.legacy_mp -MP_NAME = 'Measurement' - - -def get_bg(return_first=False): - """ - Find the active background measurement process, if any - returns None otherwise. - - Todo: - RuntimeError message is really hard to understand. - Args: - return_first(bool): if there are multiple loops running return the - first anyway. - Raises: - RuntimeError: if multiple loops are active and return_first is False. - Returns: - Union[loop, None]: active loop or none if no loops are active - """ - processes = mp.active_children() - loops = [p for p in processes if getattr(p, 'name', '') == MP_NAME] - - if len(loops) > 1 and not return_first: - raise RuntimeError('Oops, multiple loops are running???') - - if loops: - return loops[0] - - # if we got here, there shouldn't be a loop running. Make sure the - # data manager, if there is one, agrees! - _clear_data_manager() - return None - - -def halt_bg(timeout=5, traceback=True): - """ - Stop the active background measurement process, if any. - - Args: - timeout (int): seconds to wait for a clean exit before forcibly - terminating. - - traceback (bool): whether to print a traceback at the point of - interrupt, for debugging purposes. - """ - loop = get_bg(return_first=True) - if not loop: - print('No loop running') - return - - if traceback: - signal_ = ActiveLoop.HALT_DEBUG - else: - signal_ = ActiveLoop.HALT - - loop.signal_queue.put(signal_) - loop.join(timeout) - - if loop.is_alive(): - loop.terminate() - loop.join(timeout/2) - print('Background loop did not respond to halt signal, terminated') - - _clear_data_manager() +USE_MP=False def _clear_data_manager(): @@ -139,13 +72,6 @@ def _clear_data_manager(): if dm and dm.ask('get_measuring'): dm.ask('finalize_data') -# TODO(giulioungaretti) remove dead code -# def measure(*actions): -# # measure has been moved into Station -# # TODO - for all-at-once parameters we want to be able to -# # store the output into a DataSet without making a Loop. -# pass - class Loop(Metadatable): """ @@ -314,7 +240,7 @@ def run_temp(self, *args, **kwargs): shortcut to run a loop in the foreground as a temporary dataset using the default measurement set """ - return self.run(*args, background=False, quiet=True, + return self.run(*args, quiet=True, data_manager=False, location=False, **kwargs) def then(self, *actions, overwrite=False): @@ -426,22 +352,13 @@ def __init__(self, sweep_values, delay, *actions, then_actions=(), self.bg_min_delay = bg_min_delay self.data_set = None - # compile now, but don't save the results - # just used for preemptive error checking - # if we saved the results, we wouldn't capture nesting - # nor would we be able to reuse an ActiveLoop multiple times - # within one outer Loop. - # TODO: this doesn't work, because _Measure needs the data_set, - # which doesn't exist yet - do we want to make a special "dry run" - # mode, or is it sufficient to let errors wait until .run()? - # self._compile_actions(actions) - # if the first action is another loop, it changes how delays # happen - the outer delay happens *after* the inner var gets # set to its initial value self._nest_first = hasattr(actions[0], 'containers') # for sending halt signals to the loop + import multiprocessing as mp self.signal_queue = mp.Queue() self._monitor = None # TODO: how to specify this? @@ -754,18 +671,16 @@ def run_temp(self, **kwargs): especially for use in composite parameters that need to run a Loop as part of their get method """ - return self.run(background=False, quiet=True, - data_manager=False, location=False, **kwargs) + return self.run(quiet=True, data_manager=USE_MP, location=False, + **kwargs) - def run(self, background=USE_MP, use_threads=False, quiet=False, + def run(self, use_threads=False, quiet=False, data_manager=USE_MP, station=None, progress_interval=False, *args, **kwargs): """ Execute this loop. Args: - background: (default False) run this sweep in a separate process - so we can have live plotting and other analysis in the main process use_threads: (default False): whenever there are multiple `get` calls back-to-back, execute them in separate threads so they run in parallel (as long as they don't block each other) @@ -803,21 +718,8 @@ def run(self, background=USE_MP, use_threads=False, quiet=False, if progress_interval is not False: self.progress_interval = progress_interval - prev_loop = get_bg() - if prev_loop: - if not quiet: - print('Waiting for the previous background Loop to finish...', - flush=True) - prev_loop.join() - data_set = self.get_data_set(data_manager, *args, **kwargs) - if background and not getattr(data_set, 'data_manager', None): - warnings.warn( - 'With background=True you must also set data_manager=True ' - 'or you will not be able to sync your DataSet.', - UserWarning) - self.set_common_attrs(data_set=data_set, use_threads=use_threads, signal_queue=self.signal_queue) @@ -831,47 +733,17 @@ def run(self, background=USE_MP, use_threads=False, quiet=False, ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') data_set.add_metadata({'loop': { 'ts_start': ts, - 'background': background, 'use_threads': use_threads, 'use_data_manager': (data_manager is not False) }}) data_set.save_metadata() - if prev_loop and not quiet: - print('...done. Starting ' + (data_set.location or 'new loop'), - flush=True) - try: - if background: - warnings.warn("Multiprocessing is in beta, use at own risk", - UserWarning) - p = QcodesProcess(target=self._run_wrapper, name=MP_NAME) - p.is_sweep = True - p.signal_queue = self.signal_queue - p.start() - self.process = p - - # now that the data_set we created has been put in the loop - # process, this copy turns into a reader - # if you're not using a DataManager, it just stays local - # and sync() reads from disk - if self.data_set.mode == DataMode.PUSH_TO_SERVER: - self.data_set.mode = DataMode.PULL_FROM_SERVER - self.data_set.sync() - else: - if hasattr(self, 'process'): - # in case this ActiveLoop was run before in the background - del self.process - - self._run_wrapper() - - if self.data_set.mode != DataMode.LOCAL: - self.data_set.sync() - + self._run_wrapper() ds = self.data_set - finally: + if not quiet: print(repr(self.data_set)) print(datetime.now().strftime('started at %Y-%m-%d %H:%M:%S')) diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index fff3414ba82..a50aeb356f4 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -1,262 +1,17 @@ from datetime import datetime -import logging -import multiprocessing as mp -import numpy as np import time from unittest import TestCase from unittest.mock import patch -from qcodes.loops import (Loop, MP_NAME, get_bg, halt_bg, ActiveLoop, - _DebugInterrupt) +from qcodes.loops import Loop, ActiveLoop, _DebugInterrupt from qcodes.actions import Task, Wait, BreakIf from qcodes.station import Station -from qcodes.data.io import DiskIO from qcodes.data.data_array import DataArray -from qcodes.data.manager import get_data_manager -from qcodes.instrument.mock import ArrayGetter -from qcodes.instrument.parameter import Parameter, ManualParameter -from qcodes.process.helpers import kill_processes -from qcodes.process.qcodes_process import QcodesProcess +from qcodes.instrument.parameter import ManualParameter from qcodes.utils.validators import Numbers from qcodes.utils.helpers import LogCapture -from .instrument_mocks import (AMockModel, MockGates, MockSource, MockMeter, - MultiGetter) - - -class TestMockInstLoop(TestCase): - def setUp(self): - get_data_manager().restart(force=True) - kill_processes() - # TODO: figure out what's leaving DataManager in a weird state - # and fix it - get_data_manager().restart(force=True) - time.sleep(0.1) - - self.model = AMockModel() - - self.gates = MockGates(model=self.model, server_name='') - self.source = MockSource(model=self.model, server_name='') - self.meter = MockMeter(model=self.model, server_name='') - self.location = '_loop_test_' - self.location2 = '_loop_test2_' - self.io = DiskIO('.') - - c1 = self.gates.chan1 - self.loop = Loop(c1[1:5:1], 0.001).each(c1) - self.loop_progress = Loop(c1[1:5:1], 0.001, - progress_interval=1).each(c1) - - self.assertFalse(self.io.list(self.location)) - self.assertFalse(self.io.list(self.location2)) - - def tearDown(self): - for instrument in [self.gates, self.source, self.meter]: - instrument.close() - - get_data_manager().close() - self.model.close() - - self.io.remove_all(self.location) - self.io.remove_all(self.location2) - - def check_empty_data(self, data): - expected = repr([float('nan')] * 4) - self.assertEqual(repr(data.gates_chan1.tolist()), expected) - self.assertEqual(repr(data.gates_chan1_set.tolist()), expected) - - def check_loop_data(self, data): - self.assertEqual(data.gates_chan1.tolist(), [1, 2, 3, 4]) - self.assertEqual(data.gates_chan1_set.tolist(), [1, 2, 3, 4]) - - self.assertTrue(self.io.list(self.location)) - - def test_background_and_datamanager(self): - # make sure that an unpicklable instrument can indeed run in a loop - # because the instrument itself is in a server - - # TODO: if we don't save the dataset (location=False) then we can't - # sync it when we're done. Should fix that - for now that just means - # you can only do in-memory loops if you set data_manager=False - # TODO: this is the one place we don't do quiet=True - test that we - # really print stuff? - data = self.loop.run(location=self.location, background=True, data_manager=True) - self.check_empty_data(data) - - # wait for process to finish (ensures that this was run in the bg, - # because otherwise there *is* no loop.process) - self.loop.process.join() - - data.sync() - self.check_loop_data(data) - - def test_local_instrument(self): - # a local instrument should work in a foreground loop, but - # not in a background loop (should give a RuntimeError) - self.gates.close() # so we don't have two gates with same name - gates_local = MockGates(model=self.model, server_name=None) - self.gates = gates_local - c1 = gates_local.chan1 - loop_local = Loop(c1[1:5:1], 0.001).each(c1) - - # if spawn, pickle will happen - if mp.get_start_method() == "spawn": - with self.assertRaises(RuntimeError): - loop_local.run(location=self.location, - quiet=True, - background=True) - # allow for *nix - # TODO(giulioungaretti) see what happens ? - # what is the expected beavhiour ? - # The RunimError will never be raised here, as the forkmethod - # won't try to pickle anything at all. - else: - logging.error("this should not be allowed, but for now we let it be") - loop_local.run(location=self.location, quiet=True) - - data = loop_local.run(location=self.location2, background=False, - quiet=True) - self.check_loop_data(data) - - def test_background_no_datamanager(self): - # We don't support syncing data from a background process - # if not using a datamanager. See warning in ActiveLoop.run() - # So we expect the data to be empty even after running. - data = self.loop.run(location=self.location, - background=True, - data_manager=False, - quiet=True) - self.check_empty_data(data) - - self.loop.process.join() - - data.sync() - self.check_empty_data(data) - - def test_foreground_and_datamanager(self): - data = self.loop.run(location=self.location, background=False, - quiet=True) - self.assertFalse(hasattr(self.loop, 'process')) - - self.check_loop_data(data) - - def test_foreground_no_datamanager_progress(self): - data = self.loop_progress.run(location=self.location, background=False, - data_manager=False, quiet=True) - self.assertFalse(hasattr(self.loop, 'process')) - - self.check_loop_data(data) - - @patch('qcodes.loops.tprint') - def test_progress_calls(self, tprint_mock): - data = self.loop_progress.run(location=self.location, background=False, - data_manager=False, quiet=True) - self.assertFalse(hasattr(self.loop, 'process')) - - self.check_loop_data(data) - expected_calls = len(self.loop_progress.sweep_values) + 1 - self.assertEqual(tprint_mock.call_count, expected_calls) - - # now run again with no progress interval and check that we get no - # additional calls - data = self.loop_progress.run(location=False, background=False, - data_manager=False, quiet=True, - progress_interval=None) - self.assertFalse(hasattr(self.loop, 'process')) - - self.check_loop_data(data) - self.assertEqual(tprint_mock.call_count, expected_calls) - - def test_foreground_no_datamanager(self): - data = self.loop.run(location=self.location, background=False, - data_manager=False, quiet=True) - self.assertFalse(hasattr(self.loop, 'process')) - - self.check_loop_data(data) - - def test_enqueue(self): - c1 = self.gates.chan1 - loop = Loop(c1[1:5:1], 0.01).each(c1) - data1 = loop.run(location=self.location, - quiet=True, - background=True, - data_manager=True) - - # second running of the loop should be enqueued, blocks until - # the first one finishes. - # TODO: check what it prints? - data2 = loop.run(location=self.location2, - quiet=True, - background=True, - data_manager=True) - - data1.sync() - data2.sync() - self.assertEqual(data1.gates_chan1.tolist(), [1, 2, 3, 4]) - for v in data2.gates_chan1: - self.assertTrue(np.isnan(v)) - - loop.process.join() - data2.sync() - self.assertEqual(data2.gates_chan1.tolist(), [1, 2, 3, 4]) - - # and while we're here, check that running a loop in the - # foreground *after* the background clears its .process - self.assertTrue(hasattr(loop, 'process')) - loop.run_temp() - self.assertFalse(hasattr(loop, 'process')) - - def test_sync_no_overwrite(self): - # Test fix for 380, this tests that the setpoints are not incorrectly - # overwritten by data_set.sync() for this to happen with the original code - # the delay must be larger than the write period otherwise sync is a no opt. - - loop = Loop(self.gates.chan1.sweep(0, 1, 1), delay=0.1).each(ArrayGetter(self.meter.amplitude, - self.gates.chan2[0:1:1], 0.000001)) - data = loop.get_data_set(name='testsweep', write_period=0.01) - _ = loop.with_bg_task(data.sync).run() - assert not np.isnan(data.chan2_set).any() - -def sleeper(t): - time.sleep(t) - - -class TestBG(TestCase): - def test_get_halt(self): - kill_processes() - self.assertIsNone(get_bg()) - - p1 = QcodesProcess(name=MP_NAME, target=sleeper, args=(10, )) - p1.start() - p2 = QcodesProcess(name=MP_NAME, target=sleeper, args=(10, )) - p2.start() - p1.signal_queue = p2.signal_queue = mp.Queue() - qcodes_processes = [p for p in mp.active_children() - if isinstance(p, QcodesProcess)] - self.assertEqual(len(qcodes_processes), 2, mp.active_children()) - - with self.assertRaises(RuntimeError): - get_bg() - bg1 = get_bg(return_first=True) - self.assertIn(bg1, [p1, p2]) - - halt_bg(timeout=0.05) - bg2 = get_bg() - self.assertIn(bg2, [p1, p2]) - # is this robust? requires that active_children always returns the same - # order, even if it's not the order you started processes in - self.assertNotEqual(bg1, bg2) - - self.assertEqual(len(mp.active_children()), 1) - - halt_bg(timeout=0.05) - self.assertIsNone(get_bg()) - - self.assertEqual(len(mp.active_children()), 0) - - # TODO - test that we print "no loops running"? - # at least this shows that it won't raise an error - halt_bg() +from .instrument_mocks import MultiGetter class FakeMonitor: @@ -279,9 +34,6 @@ def setUpClass(cls): cls.p3 = ManualParameter('p3', vals=Numbers(-10, 10)) Station().set_measurement(cls.p2, cls.p3) - def setUp(self): - kill_processes() - def test_nesting(self): loop = Loop(self.p1[1:3:1], 0.001).loop( self.p2[3:5:1], 0.001).loop( @@ -402,15 +154,7 @@ def test_tasks_waits(self): self.p2) delay_array = [] loop._monitor = FakeMonitor(delay_array) - - # give it a "process" as if it was run in the bg before, - # check that this gets cleared - loop.process = 'TDD' - data = loop.run_temp() - - self.assertFalse(hasattr(loop, 'process')) - self.assertEqual(data.p1_set.tolist(), [1, 2]) self.assertEqual(data.p2_2.tolist(), [-1, -1]) self.assertEqual(data.p2_4.tolist(), [1, 1]) @@ -690,7 +434,6 @@ def g(): 'default_measurement': [p2snap, p3snap] }, 'loop': { - 'background': False, 'use_threads': False, 'use_data_manager': False, '__class__': 'qcodes.loops.ActiveLoop', @@ -781,7 +524,7 @@ def test_halt(self): # need to use explicit loop.run rather than run_temp # so we can avoid providing location=False twice, which # is an error. - loop.run(background=False, data_manager=False, quiet=True) + loop.run(data_manager=False, quiet=True) self.check_data(data) From 700e150a7ffde2613906e4a2d92e8e0a6e09555c Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 12:38:47 +0100 Subject: [PATCH 04/36] fix: remove alt-docs about monitor --- qcodes/loops.py | 8 -------- qcodes/tests/test_loop.py | 34 ---------------------------------- 2 files changed, 42 deletions(-) diff --git a/qcodes/loops.py b/qcodes/loops.py index 5c13f553728..86898cc204a 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -361,8 +361,6 @@ def __init__(self, sweep_values, delay, *actions, then_actions=(), import multiprocessing as mp self.signal_queue = mp.Queue() - self._monitor = None # TODO: how to specify this? - def then(self, *actions, overwrite=False): """ Attach actions to be performed after the loop completes. @@ -913,12 +911,6 @@ def _wait(self, delay): if delay: finish_clock = time.perf_counter() + delay - if self._monitor: - # TODO - perhpas pass self._check_signal in here - # so that we can halt within monitor.call if it - # lasts a very long time? - self._monitor.call(finish_by=finish_clock) - while True: self._check_signal() t = wait_secs(finish_clock) diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index a50aeb356f4..0c44557b17f 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -14,18 +14,6 @@ from .instrument_mocks import MultiGetter -class FakeMonitor: - ''' - when attached to an ActiveLoop as _monitor, records how long - the monitor was given to measure - ''' - def __init__(self, delay_array): - self.delay_array = delay_array - - def call(self, finish_by=None): - self.delay_array.append(finish_by - time.perf_counter()) - - class TestLoop(TestCase): @classmethod def setUpClass(cls): @@ -143,28 +131,6 @@ def test_func(*args, **kwargs): self.assertEqual(data.p2.tolist(), [2]) - def test_tasks_waits(self): - delay0 = 0.01 - delay1 = 0.03 - loop = Loop(self.p1[1:3:1], delay0).each( - Task(self.p2.set, -1), - Wait(delay1), - self.p2, - Task(self.p2.set, 1), - self.p2) - delay_array = [] - loop._monitor = FakeMonitor(delay_array) - data = loop.run_temp() - self.assertEqual(data.p1_set.tolist(), [1, 2]) - self.assertEqual(data.p2_2.tolist(), [-1, -1]) - self.assertEqual(data.p2_4.tolist(), [1, 1]) - - self.assertEqual(len(delay_array), 4) - for i, delay in enumerate(delay_array): - target = delay1 if i % 2 else delay0 - self.assertLessEqual(delay, target) - self.assertGreater(delay, target - 0.001) - @patch('time.sleep') def test_delay0(self, sleep_mock): self.p2.set(3) From 01e066ecb518e0a2cb080c7c601b65593cc6afa2 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 13:19:27 +0100 Subject: [PATCH 05/36] remove signal queue --- qcodes/loops.py | 203 ++++++++++++++++---------------------- qcodes/tests/test_loop.py | 60 +++-------- 2 files changed, 98 insertions(+), 165 deletions(-) diff --git a/qcodes/loops.py b/qcodes/loops.py index 86898cc204a..c20ce7430f3 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -330,12 +330,6 @@ class ActiveLoop(Metadatable): The *ActiveLoop* determines what *DataArray*\s it will need to hold the data it collects, and it creates a *DataSet* holding these *DataArray*\s """ - # constants for signal_queue - HALT = 'HALT LOOP' - HALT_DEBUG = 'HALT AND DEBUG' - - # maximum sleep time (secs) between checking the signal_queue for a HALT - signal_period = 1 def __init__(self, sweep_values, delay, *actions, then_actions=(), station=None, progress_interval=None, bg_task=None, @@ -357,10 +351,6 @@ def __init__(self, sweep_values, delay, *actions, then_actions=(), # set to its initial value self._nest_first = hasattr(actions[0], 'containers') - # for sending halt signals to the loop - import multiprocessing as mp - self.signal_queue = mp.Queue() - def then(self, *actions, overwrite=False): """ Attach actions to be performed after the loop completes. @@ -582,7 +572,7 @@ def _default_setpoints(self, shape): return sp - def set_common_attrs(self, data_set, use_threads, signal_queue): + def set_common_attrs(self, data_set, use_threads): """ set a couple of common attributes that the main and nested loops all need to have: @@ -590,21 +580,10 @@ def set_common_attrs(self, data_set, use_threads, signal_queue): - a queue for communicating with the main process """ self.data_set = data_set - self.signal_queue = signal_queue self.use_threads = use_threads for action in self.actions: if hasattr(action, 'set_common_attrs'): - action.set_common_attrs(data_set, use_threads, signal_queue) - - def _check_signal(self): - while not self.signal_queue.empty(): - signal_ = self.signal_queue.get() - if signal_ == self.HALT: - raise _QuietInterrupt('sweep was halted') - elif signal_ == self.HALT_DEBUG: - raise _DebugInterrupt('sweep was halted') - else: - raise ValueError('unknown signal', signal_) + action.set_common_attrs(data_set, use_threads) def get_data_set(self, data_manager=USE_MP, *args, **kwargs): """ @@ -718,8 +697,7 @@ def run(self, use_threads=False, quiet=False, data_set = self.get_data_set(data_manager, *args, **kwargs) - self.set_common_attrs(data_set=data_set, use_threads=use_threads, - signal_queue=self.signal_queue) + self.set_common_attrs(data_set=data_set, use_threads=use_threads) station = station or self.station or Station.default if station: @@ -785,17 +763,16 @@ def _compile_one(self, action, new_action_indices): return action def _run_wrapper(self, *args, **kwargs): - try: - self._run_loop(*args, **kwargs) - except _QuietInterrupt: - pass - finally: - if hasattr(self, 'data_set'): - # somehow this does not show up in the data_set returned by - # run(), but it is saved to the metadata - ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - self.data_set.add_metadata({'loop': {'ts_end': ts}}) - self.data_set.finalize() + # try: + self._run_loop(*args, **kwargs) + # finally: + if hasattr(self, 'data_set'): + # TODO (giulioungaretti) WTF? + # somehow this does not show up in the data_set returned by + # run(), but it is saved to the metadata + ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + self.data_set.add_metadata({'loop': {'ts_end': ts}}) + self.data_set.finalize() def _run_loop(self, first_delay=0, action_indices=(), loop_indices=(), current_values=(), @@ -824,77 +801,74 @@ def _run_loop(self, first_delay=0, action_indices=(), self.last_task_failed = False - for i, value in enumerate(self.sweep_values): - if self.progress_interval is not None: - tprint('loop %s: %d/%d (%.1f [s])' % ( - self.sweep_values.name, i, imax, time.time() - t0), - dt=self.progress_interval, tag='outerloop') - - set_val = self.sweep_values.set(value) - - new_indices = loop_indices + (i,) - new_values = current_values + (value,) - data_to_store = {} - - if hasattr(self.sweep_values, "parameters"): - set_name = self.data_set.action_id_map[action_indices] - if hasattr(self.sweep_values, 'aggregate'): - value = self.sweep_values.aggregate(*set_val) - self.data_set.store(new_indices, {set_name: value}) - for j, val in enumerate(set_val): - set_index = action_indices + (j+1, ) - set_name = (self.data_set.action_id_map[set_index]) - data_to_store[set_name] = val - else: - set_name = self.data_set.action_id_map[action_indices] - data_to_store[set_name] = value - - self.data_set.store(new_indices, data_to_store) - - if not self._nest_first: - # only wait the delay time if an inner loop will not inherit it - self._wait(delay) - - try: - for f in callables: - f(first_delay=delay, - loop_indices=new_indices, - current_values=new_values) - - # after the first action, no delay is inherited - delay = 0 - except _QcodesBreak: - break - - # after the first setpoint, delay reverts to the loop delay - delay = self.delay - - # now check for a background task and execute it if it's - # been long enough since the last time - # don't let exceptions in the background task interrupt - # the loop - # if the background task fails twice consecutively, stop - # executing it - if self.bg_task is not None: - t = time.time() - if t - last_task >= self.bg_min_delay: - try: - self.bg_task() - except Exception: - if self.last_task_failed: - self.bg_task = None - self.last_task_failed = True - log.exception("Failed to execute bg task") - - last_task = t - - if self.progress_interval is not None: - # final progress note: set dt=-1 so it *always* prints - tprint('loop %s DONE: %d/%d (%.1f [s])' % ( - - self.sweep_values.name, i + 1, imax, time.time() - t0), - dt=-1, tag='outerloop') + try: + for i, value in enumerate(self.sweep_values): + if self.progress_interval is not None: + tprint('loop %s: %d/%d (%.1f [s])' % ( + self.sweep_values.name, i, imax, time.time() - t0), + dt=self.progress_interval, tag='outerloop') + + set_val = self.sweep_values.set(value) + + new_indices = loop_indices + (i,) + new_values = current_values + (value,) + data_to_store = {} + + if hasattr(self.sweep_values, "parameters"): + set_name = self.data_set.action_id_map[action_indices] + if hasattr(self.sweep_values, 'aggregate'): + value = self.sweep_values.aggregate(*set_val) + self.data_set.store(new_indices, {set_name: value}) + for j, val in enumerate(set_val): + set_index = action_indices + (j+1, ) + set_name = (self.data_set.action_id_map[set_index]) + data_to_store[set_name] = val + else: + set_name = self.data_set.action_id_map[action_indices] + data_to_store[set_name] = value + + self.data_set.store(new_indices, data_to_store) + + if not self._nest_first: + # only wait the delay time if an inner loop will not inherit it + self._wait(delay) + + try: + for f in callables: + f(first_delay=delay, + loop_indices=new_indices, + current_values=new_values) + + # after the first action, no delay is inherited + delay = 0 + except _QcodesBreak: + break + # after the first setpoint, delay reverts to the loop delay + delay = self.delay + + # now check for a background task and execute it if it's + # been long enough since the last time + # don't let exceptions in the background task interrupt + # the loop + # if the background task fails twice consecutively, stop + # executing it + if self.bg_task is not None: + t = time.time() + if t - last_task >= self.bg_min_delay: + try: + self.bg_task() + except Exception: + if self.last_task_failed: + self.bg_task = None + self.last_task_failed = True + log.exception("Failed to execute bg task") + + last_task = t + + except Interrupt: + log.debug("Stopping loop cleanly") + return # run the background task one last time to catch the last setpoint(s) if self.bg_task is not None: self.bg_task() @@ -910,20 +884,9 @@ def _run_loop(self, first_delay=0, action_indices=(), def _wait(self, delay): if delay: finish_clock = time.perf_counter() + delay - - while True: - self._check_signal() - t = wait_secs(finish_clock) - time.sleep(min(t, self.signal_period)) - if t <= self.signal_period: - break - else: - self._check_signal() - - -class _QuietInterrupt(Exception): - pass + t = wait_secs(finish_clock) + time.sleep(t) -class _DebugInterrupt(Exception): +class Interrupt(Exception): pass diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index 0c44557b17f..cbe082bc7ff 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -1,9 +1,10 @@ from datetime import datetime import time from unittest import TestCase +import numpy as np from unittest.mock import patch -from qcodes.loops import Loop, ActiveLoop, _DebugInterrupt +from qcodes.loops import Loop, Interrupt from qcodes.actions import Task, Wait, BreakIf from qcodes.station import Station from qcodes.data.data_array import DataArray @@ -450,70 +451,39 @@ def g(): class AbortingGetter(ManualParameter): - ''' - A manual parameter that can only be measured a couple of times + """ + A manual parameter that can only be measured n times before it aborts the loop that's measuring it. - - You have to attach the queue after construction with set_queue - so you can grab it from the loop that uses the parameter. - ''' + """ def __init__(self, *args, count=1, msg=None, **kwargs): self._count = self._initial_count = count - self.msg = msg # also need a _signal_queue, but that has to be added later super().__init__(*args, **kwargs) def get(self): self._count -= 1 if self._count <= 0: - self._signal_queue.put(self.msg) + raise Interrupt return super().get() - def set_queue(self, queue): - self._signal_queue = queue - def reset(self): self._count = self._initial_count -class TestSignal(TestCase): +class Test_halt(TestCase): def test_halt(self): - p1 = AbortingGetter('p1', count=2, vals=Numbers(-10, 10), - msg=ActiveLoop.HALT_DEBUG) - loop = Loop(p1[1:6:1], 0.005).each(p1) + abort_after = 3 + self.res = list(np.arange(0, abort_after-1, 1.)) + [self.res.append(float('nan')) for i in range(0, abort_after-1)] + + p1 = AbortingGetter('p1', count=abort_after, vals=Numbers(-10, 10)) + loop = Loop(p1.sweep(0, abort_after, 1), 0.005).each(p1) # we want to test what's in data, so get it ahead of time # because loop.run will not return. data = loop.get_data_set(location=False) - p1.set_queue(loop.signal_queue) - - with self.assertRaises(_DebugInterrupt): - # need to use explicit loop.run rather than run_temp - # so we can avoid providing location=False twice, which - # is an error. - loop.run(data_manager=False, quiet=True) - - self.check_data(data) - def test_halt_quiet(self): - p1 = AbortingGetter('p1', count=2, vals=Numbers(-10, 10), - msg=ActiveLoop.HALT) - loop = Loop(p1[1:6:1], 0.005).each(p1) - p1.set_queue(loop.signal_queue) - - # does not raise, just quits, but the data set looks the same - # as in test_halt - data = loop.run_temp() - self.check_data(data) - - def check_data(self, data): - nan = float('nan') - self.assertEqual(data.p1.tolist()[:2], [1, 2]) - # when NaN is involved, I'll just compare reprs, because NaN!=NaN - self.assertEqual(repr(data.p1.tolist()[-2:]), repr([nan, nan])) - # because of the way the waits work out, we can get an extra - # point measured before the interrupt is registered. But the - # test would be valid either way. - self.assertIn(repr(data.p1[2]), (repr(nan), repr(3), repr(3.0))) + loop.run(data_manager=False, quiet=True) + self.assertEqual(repr(data.p1.tolist()), repr(self.res)) class TestMetaData(TestCase): From 5bf22e3b38ddadec23461767492fea0d835e5acb Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 14:08:56 +0100 Subject: [PATCH 06/36] fix: More backgroudn removal --- qcodes/measure.py | 7 ++----- qcodes/tests/test_hdf5formatter.py | 4 ++-- qcodes/tests/test_measure.py | 1 - 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/qcodes/measure.py b/qcodes/measure.py index 17063fd38c4..782e07bc223 100644 --- a/qcodes/measure.py +++ b/qcodes/measure.py @@ -71,9 +71,6 @@ def run(self, use_threads=False, quiet=False, data_manager=USE_MP, a DataSet object containing the results of the measurement """ - # background is not configurable, would be weird to run this in the bg - background = False - data_set = self._dummyLoop.get_data_set(data_manager=data_manager, **kwargs) @@ -83,7 +80,7 @@ def run(self, use_threads=False, quiet=False, data_manager=USE_MP, data_set.location = False # run the measurement as if it were a Loop - self._dummyLoop.run(background=background, use_threads=use_threads, + self._dummyLoop.run(use_threads=use_threads, station=station, quiet=True) # look for arrays that are unnecessarily nested, and un-nest them @@ -128,7 +125,7 @@ def run(self, use_threads=False, quiet=False, data_manager=USE_MP, # puts in a 'loop' section that we need to replace with 'measurement' # but we use the info from 'loop' to ensure consistency and avoid # duplication. - LOOP_SNAPSHOT_KEYS = ['background', 'ts_start', 'ts_end', + LOOP_SNAPSHOT_KEYS = ['ts_start', 'ts_end', 'use_data_manager', 'use_threads'] data_set.add_metadata({'measurement': { k: data_set.metadata['loop'][k] for k in LOOP_SNAPSHOT_KEYS diff --git a/qcodes/tests/test_hdf5formatter.py b/qcodes/tests/test_hdf5formatter.py index d1506998f18..4f759f1d8bf 100644 --- a/qcodes/tests/test_hdf5formatter.py +++ b/qcodes/tests/test_hdf5formatter.py @@ -137,7 +137,7 @@ def test_loop_writing(self): loop = Loop(MockPar.x[-100:100:20]).each(MockPar.skewed_parabola) data1 = loop.run(name='MockLoop_hdf5_test', formatter=self.formatter, - background=False, data_manager=False) + data_manager=False) data2 = DataSet(location=data1.location, formatter=self.formatter) data2.read() for key in data2.arrays.keys(): @@ -159,7 +159,7 @@ def test_loop_writing_2D(self): MockPar.y[-50:50:10]).each(MockPar.skewed_parabola) data1 = loop.run(name='MockLoop_hdf5_test', formatter=self.formatter, - background=False, data_manager=False) + data_manager=False) data2 = DataSet(location=data1.location, formatter=self.formatter) data2.read() for key in data2.arrays.keys(): diff --git a/qcodes/tests/test_measure.py b/qcodes/tests/test_measure.py index 71d3c45d4c4..8563c34559e 100644 --- a/qcodes/tests/test_measure.py +++ b/qcodes/tests/test_measure.py @@ -25,7 +25,6 @@ def test_simple_scalar(self): meta = data.metadata['measurement'] self.assertEqual(meta['__class__'], 'qcodes.measure.Measure') self.assertEqual(len(meta['actions']), 1) - self.assertFalse(meta['background']) self.assertFalse(meta['use_data_manager']) self.assertFalse(meta['use_threads']) From f320b5d6cabfdfa3e5bbf5ccb48c750048d77009 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 15:24:41 +0100 Subject: [PATCH 07/36] remove data_manager stuff --- qcodes/__init__.py | 2 +- qcodes/data/data_set.py | 290 +++-------------------------- qcodes/loops.py | 42 +---- qcodes/measure.py | 14 +- qcodes/tests/test_data.py | 140 +------------- qcodes/tests/test_format.py | 4 +- qcodes/tests/test_hdf5formatter.py | 6 +- qcodes/tests/test_loop.py | 4 +- qcodes/tests/test_measure.py | 1 - 9 files changed, 55 insertions(+), 448 deletions(-) diff --git a/qcodes/__init__.py b/qcodes/__init__.py index 4243cf3c2cb..b1b2644ccd9 100644 --- a/qcodes/__init__.py +++ b/qcodes/__init__.py @@ -45,7 +45,7 @@ from qcodes.actions import Task, Wait, BreakIf from qcodes.data.manager import get_data_manager -from qcodes.data.data_set import DataMode, DataSet, new_data, load_data +from qcodes.data.data_set import DataSet, new_data, load_data from qcodes.data.location import FormatLocation from qcodes.data.data_array import DataArray from qcodes.data.format import Formatter diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 98b3fa60c3a..d5407807b5d 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -1,35 +1,19 @@ """DataSet class and factory functions.""" -from enum import Enum import time import logging from traceback import format_exc from copy import deepcopy from collections import OrderedDict -from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat from .io import DiskIO from .location import FormatLocation from qcodes.utils.helpers import DelegateAttributes, full_class, deep_update -class DataMode(Enum): - - """Server connection modes supported by a DataSet.""" - - LOCAL = 1 - PUSH_TO_SERVER = 2 - PULL_FROM_SERVER = 3 - - -SERVER_MODES = set((DataMode.PULL_FROM_SERVER, DataMode.PUSH_TO_SERVER)) - - def new_data(location=None, loc_record=None, name=None, overwrite=False, - io=None, data_manager=False, mode=DataMode.LOCAL, **kwargs): - # NOTE(giulioungaretti): leave this docstrings as it is, because - # documenting the types is silly in this case. + io=None, **kwargs): """ Create a new DataSet. @@ -61,24 +45,6 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False, says the root data directory is the current working directory, ie where you started the python session. - data_manager (Optional[bool]): use a manager for the - ``DataServer`` that offloads storage and syncing of this Defaults - to ``False`` i.e. this ``DataSet`` will store itself without extra - processes. Set to ``True`` to use the default from - ``get_data_manager()``. - - mode (DataMode, optional): connection type to the ``DataServer``. - - - ``DataMode.LOCAL``: this DataSet doesn't communicate across - processes. - - ``DataMode.PUSH_TO_SERVER``: no local copy of data, just pushes - each measurement to a ``DataServer``. - - ``DataMode.PULL_FROM_SERVER``: pulls changes from the - ``DataServer`` on calling ``self.sync()``. Reverts to local if - and when it stops being the live measurement. - - Default ``DataMode.LOCAL``. - arrays (Optional[List[qcodes.DataArray]): arrays to add to the DataSet. Can be added later with ``self.add_array(array)``. @@ -86,11 +52,8 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False, write (and read) with. Default ``DataSet.default_formatter`` which is initially ``GNUPlotFormat()``. - write_period (float or None, optional): Only if ``mode=LOCAL``, seconds - between saves to disk. If not ``LOCAL``, the ``DataServer`` handles - this and generally writes more often. Use None to disable writing - from calls to ``self.store``. Default 5. - + write_period (float or None, optional):seconds + between saves to disk. Returns: A new ``DataSet`` object ready for storing new data in. """ @@ -111,23 +74,13 @@ def new_data(location=None, loc_record=None, name=None, overwrite=False, if location and (not overwrite) and io.list(location): raise FileExistsError('"' + location + '" already has data') - if data_manager is True: - data_manager = get_data_manager() - else: - if mode != DataMode.LOCAL: - raise ValueError('DataSets without a data_manager must be local') + return DataSet(location=location, io=io, **kwargs) - return DataSet(location=location, io=io, data_manager=data_manager, - mode=mode, **kwargs) - -def load_data(location=None, data_manager=None, formatter=None, io=None): +def load_data(location=None, formatter=None, io=None): """ Load an existing DataSet. - The resulting ``DataSet.mode`` is determined automatically from location: - PULL_FROM_SERVER if this is the live DataSet, otherwise LOCAL - Args: location (str, optional): the location to load from. Default is the current live DataSet. @@ -135,13 +88,6 @@ def load_data(location=None, data_manager=None, formatter=None, io=None): combination of io + location. the default ``DiskIO`` sets the base directory, which this location is a relative path inside. - data_manager (DataManager or False, optional): manager for the - ``DataServer`` that offloads storage and syncing of this - ``DataSet``. Usually omitted (default None) to use the default - from ``get_data_manager()``. If ``False``, this ``DataSet`` will - store itself. ``load_data`` will not start a DataManager but may - query an existing one to determine (and pull) the live data. - formatter (Formatter, optional): sets the file format/structure to read with. Default ``DataSet.default_formatter`` which is initially ``GNUPlotFormat()``. @@ -154,39 +100,14 @@ def load_data(location=None, data_manager=None, formatter=None, io=None): Returns: A new ``DataSet`` object loaded with pre-existing data. """ - if data_manager is None: - data_manager = get_data_manager(only_existing=True) - - if location is None: - if not data_manager: - raise RuntimeError('Live data requested but DataManager does ' - 'not exist or was requested not to be used') - - return _get_live_data(data_manager) - - elif location is False: + if location is False: raise ValueError('location=False means a temporary DataSet, ' 'which is incompatible with load_data') - elif (data_manager and - location == data_manager.ask('get_data', 'location')): - return _get_live_data(data_manager) - - else: - data = DataSet(location=location, formatter=formatter, io=io, - mode=DataMode.LOCAL) - data.read_metadata() - data.read() - return data - - -def _get_live_data(data_manager): - live_data = data_manager.ask('get_data') - if live_data is None or isinstance(live_data, NoData): - raise RuntimeError('DataManager has no live data') - - live_data.mode = DataMode.PULL_FROM_SERVER - return live_data + data = DataSet(location=location, formatter=formatter, io=io) + data.read_metadata() + data.read() + return data class DataSet(DelegateAttributes): @@ -212,24 +133,6 @@ class DataSet(DelegateAttributes): says the root data directory is the current working directory, ie where you started the python session. - data_manager (Optional[bool]): use a manager for the - ``DataServer`` that offloads storage and syncing of this Defaults - to ``False`` i.e. this ``DataSet`` will store itself without extra - processes. Set to ``True`` to use the default from - ``get_data_manager()``. - - mode (DataMode, optional): connection type to the ``DataServer``. - - - ``DataMode.LOCAL``: this DataSet doesn't communicate across - processes. - - ``DataMode.PUSH_TO_SERVER``: no local copy of data, just pushes - each measurement to a ``DataServer``. - - ``DataMode.PULL_FROM_SERVER``: pulls changes from the - ``DataServer`` on calling ``self.sync()``. Reverts to local if - and when it stops being the live measurement. - - Default to ``DataMode.LOCAL``. - arrays (Optional[List[qcodes.DataArray]): arrays to add to the DataSet. Can be added later with ``self.add_array(array)``. @@ -265,8 +168,8 @@ class DataSet(DelegateAttributes): background_functions = OrderedDict() - def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, - data_manager=False, formatter=None, io=None, write_period=5): + def __init__(self, location=None, arrays=None, formatter=None, io=None, + write_period=5): if location is False or isinstance(location, str): self.location = location else: @@ -289,92 +192,10 @@ def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, for array in arrays: self.add_array(array) - if data_manager is True and mode in SERVER_MODES: - data_manager = get_data_manager() - - if mode == DataMode.LOCAL: - self._init_local() - elif mode == DataMode.PUSH_TO_SERVER: - self._init_push_to_server(data_manager) - elif mode == DataMode.PULL_FROM_SERVER: - self._init_live(data_manager) - else: - raise ValueError('unrecognized DataSet mode', mode) - - def _init_local(self): - self.mode = DataMode.LOCAL - if self.arrays: for array in self.arrays.values(): array.init_data() - def _init_push_to_server(self, data_manager): - self.mode = DataMode.PUSH_TO_SERVER - - # If some code was not available when data_manager was started, - # we can't unpickle it on the other end. - # So we'll try, then restart if this error occurs, then try again. - # - # This still has a pitfall, if code has been *changed* since - # starting the server, it will still have the old version and - # everything will look fine but it won't have the new behavior. - # If the user does that, they need to manually restart the server, - # using: - # data_manager.restart() - try: - data_manager.ask('new_data', self) - except AttributeError: - data_manager.restart() - data_manager.ask('new_data', self) - - # need to set data_manager *after* sending to data_manager because - # we can't (and shouldn't) send data_manager itself through a queue - self.data_manager = data_manager - - def init_on_server(self): - """ - Configure this DataSet as the DataServer copy. - - Should be run only by the DataServer itself. - """ - if not self.arrays: - raise RuntimeError('A server-side DataSet needs DataArrays.') - - self._init_local() - - def _init_live(self, data_manager): - self.mode = DataMode.PULL_FROM_SERVER - self.data_manager = data_manager - with data_manager.query_lock: - if self.is_on_server: - live_obj = data_manager.ask('get_data') - self.arrays = live_obj.arrays - else: - self._init_local() - - @property - def is_live_mode(self): - """ - Indicate whether this DataSet thinks it is live in the DataServer. - - Does not actually talk to the DataServer or sync with it. - """ - return self.mode in SERVER_MODES and self.data_manager and True - - @property - def is_on_server(self): - """ - Check whether this DataSet is actually live in the DataServer. - - If it thought it was but isn't, convert it to mode=LOCAL - """ - if not self.is_live_mode or self.location is False: - return False - - with self.data_manager.query_lock: - live_location = self.data_manager.ask('get_data', 'location') - return self.location == live_location - def sync(self): """ Synchronize this DataSet with the DataServer or storage. @@ -391,39 +212,9 @@ def sync(self): # changed (and I guess throw an error if both did? Would be cool if we # could find a robust and intuitive way to make modifications to the # version on the DataServer from the main copy) - if not self.is_live_mode: - # LOCAL DataSet - no need to sync just use local data - return False - # TODO - for remote live plotting, maybe set some timestamp - # threshold and call it static after it's been dormant a long time? - # I'm thinking like a minute, or ten? Maybe it's configurable? - - with self.data_manager.query_lock: - if self.is_on_server: - synced_indices = { - array_id: array.get_synced_index() - for array_id, array in self.arrays.items() - } - - changes = self.data_manager.ask('get_changes', synced_indices) - - for array_id, array_changes in changes.items(): - self.arrays[array_id].apply_changes(**array_changes) - - measuring = self.data_manager.ask('get_measuring') - if not measuring: - # we must have *just* stopped measuring - # but the DataSet is still on the server, - # so we got the data, and don't need to read. - self.mode = DataMode.LOCAL - return False - return True - else: - # this DataSet *thought* it was on the server, but it wasn't, - # so we haven't synced yet and need to read from storage - self.mode = DataMode.LOCAL - self.read() - return False + + # LOCAL DataSet - no need to sync just use local data + return False def fraction_complete(self): """ @@ -582,9 +373,6 @@ def store(self, loop_indices, ids_values): """ Insert data into one or more of our DataArrays. - If in ``PUSH_TO_SERVER`` mode, this is where we do that! - Otherwise we also periodically trigger a write to storage. - Args: loop_indices (tuple): the indices within whatever loops we are inside. May have fewer dimensions than some of the arrays @@ -594,23 +382,13 @@ def store(self, loop_indices, ids_values): array_ids, and values are single numbers or entire slices to insert into that array. """ - if self.mode == DataMode.PUSH_TO_SERVER: - # Defers to the copy on the dataserver to call this identical - # function - self.data_manager.write('store_data', loop_indices, ids_values) - elif self.mode == DataMode.LOCAL: - # You will always end up in this block, either in the copy - # on the server (if you hit the if statement above) or else here - for array_id, value in ids_values.items(): - self.arrays[array_id][loop_indices] = value - self.last_store = time.time() - if (self.write_period is not None and - time.time() > self.last_write + self.write_period): - self.write() - self.last_write = time.time() - else: # in PULL_FROM_SERVER mode; store() isn't legal - raise RuntimeError('This object is pulling from a DataServer, ' - 'so data insertion is not allowed.') + for array_id, value in ids_values.items(): + self.arrays[array_id][loop_indices] = value + self.last_store = time.time() + if (self.write_period is not None and + time.time() > self.last_write + self.write_period): + self.write() + self.last_write = time.time() def default_parameter_name(self, paramname='amplitude'): """ Return name of default parameter for plotting @@ -692,10 +470,6 @@ def write(self, write_metadata=False): Args: write_metadata (bool): write the metadata to disk """ - if self.mode != DataMode.LOCAL: - raise RuntimeError('This object is connected to a DataServer, ' - 'which handles writing automatically.') - if self.location is False: return @@ -783,20 +557,11 @@ def finalize(self): Also closes the data file(s), if the ``Formatter`` we're using supports that. """ - if self.mode == DataMode.PUSH_TO_SERVER: - # Just like .store, if this DataSet is on the DataServer, - # we defer to the copy there and execute this same method. - self.data_manager.ask('finalize_data') - elif self.mode == DataMode.LOCAL: - # You will always end up in this block, either in the copy - # on the server (if you hit the if statement above) or else here - self.write() + self.write() + + if hasattr(self.formatter, 'close_file'): + self.formatter.close_file(self) - if hasattr(self.formatter, 'close_file'): - self.formatter.close_file(self) - else: - raise RuntimeError('This mode does not allow finalizing', - self.mode) self.save_metadata() def snapshot(self, update=False): @@ -833,8 +598,7 @@ def __repr__(self): """Rich information about the DataSet and contained arrays.""" out = type(self).__name__ + ':' - attrs = [['mode', self.mode], - ['location', repr(self.location)]] + attrs = [['location', repr(self.location)]] attr_template = '\n {:8} = {}' for var, val in attrs: out += attr_template.format(var, val) diff --git a/qcodes/loops.py b/qcodes/loops.py index c20ce7430f3..f00a35b6aa6 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -50,12 +50,10 @@ import logging import time import numpy as np -import warnings from qcodes.station import Station -from qcodes.data.data_set import new_data, DataMode +from qcodes.data.data_set import new_data from qcodes.data.data_array import DataArray -from qcodes.data.manager import get_data_manager from qcodes.utils.helpers import wait_secs, full_class, tprint from qcodes.utils.metadata import Metadatable @@ -64,13 +62,6 @@ log = logging.getLogger(__name__) -USE_MP=False - - -def _clear_data_manager(): - dm = get_data_manager(only_existing=True) - if dm and dm.ask('get_measuring'): - dm.ask('finalize_data') class Loop(Metadatable): @@ -240,8 +231,7 @@ def run_temp(self, *args, **kwargs): shortcut to run a loop in the foreground as a temporary dataset using the default measurement set """ - return self.run(*args, quiet=True, - data_manager=False, location=False, **kwargs) + return self.run(*args, quiet=True, location=False, **kwargs) def then(self, *actions, overwrite=False): """ @@ -585,7 +575,7 @@ def set_common_attrs(self, data_set, use_threads): if hasattr(action, 'set_common_attrs'): action.set_common_attrs(data_set, use_threads) - def get_data_set(self, data_manager=USE_MP, *args, **kwargs): + def get_data_set(self, *args, **kwargs): """ Return the data set for this loop. @@ -618,22 +608,12 @@ def get_data_set(self, data_manager=USE_MP, *args, **kwargs): a DataSet object that we can use to plot """ if self.data_set is None: - if data_manager is False: - data_mode = DataMode.LOCAL - else: - warnings.warn("Multiprocessing is in beta, use at own risk", - UserWarning) - data_mode = DataMode.PUSH_TO_SERVER - - data_set = new_data(arrays=self.containers(), mode=data_mode, - data_manager=data_manager, *args, **kwargs) - + data_set = new_data(arrays=self.containers(), *args, **kwargs) self.data_set = data_set else: has_args = len(kwargs) or len(args) - uses_data_manager = (self.data_set.mode != DataMode.LOCAL) - if has_args or (uses_data_manager != data_manager): + if has_args: raise RuntimeError( 'The DataSet for this loop already exists. ' 'You can only provide DataSet attributes, such as ' @@ -648,12 +628,10 @@ def run_temp(self, **kwargs): especially for use in composite parameters that need to run a Loop as part of their get method """ - return self.run(quiet=True, data_manager=USE_MP, location=False, - **kwargs) + return self.run(quiet=True, location=False, **kwargs) - def run(self, use_threads=False, quiet=False, - data_manager=USE_MP, station=None, progress_interval=False, - *args, **kwargs): + def run(self, use_threads=False, quiet=False, station=None, + progress_interval=False, *args, **kwargs): """ Execute this loop. @@ -662,7 +640,6 @@ def run(self, use_threads=False, quiet=False, back-to-back, execute them in separate threads so they run in parallel (as long as they don't block each other) quiet: (default False): set True to not print anything except errors - data_manager: set to True to use a DataManager. Default to False. station: a Station instance for snapshots (omit to use a previously provided Station, or the default Station) progress_interval (default None): show progress of the loop every x @@ -695,7 +672,7 @@ def run(self, use_threads=False, quiet=False, if progress_interval is not False: self.progress_interval = progress_interval - data_set = self.get_data_set(data_manager, *args, **kwargs) + data_set = self.get_data_set(*args, **kwargs) self.set_common_attrs(data_set=data_set, use_threads=use_threads) @@ -710,7 +687,6 @@ def run(self, use_threads=False, quiet=False, data_set.add_metadata({'loop': { 'ts_start': ts, 'use_threads': use_threads, - 'use_data_manager': (data_manager is not False) }}) data_set.save_metadata() diff --git a/qcodes/measure.py b/qcodes/measure.py index 782e07bc223..74443d3b587 100644 --- a/qcodes/measure.py +++ b/qcodes/measure.py @@ -1,7 +1,7 @@ from datetime import datetime from qcodes.instrument.parameter import ManualParameter -from qcodes.loops import Loop, USE_MP +from qcodes.loops import Loop from qcodes.actions import _actions_snapshot from qcodes.utils.helpers import full_class from qcodes.utils.metadata import Metadatable @@ -29,11 +29,9 @@ def run_temp(self, **kwargs): """ Wrapper to run this measurement as a temporary data set """ - return self.run(quiet=True, data_manager=False, location=False, - **kwargs) + return self.run(quiet=True, location=False, **kwargs) - def run(self, use_threads=False, quiet=False, data_manager=USE_MP, - station=None, **kwargs): + def run(self, use_threads=False, quiet=False, station=None, **kwargs): """ Run the actions in this measurement and return their data as a DataSet @@ -71,8 +69,7 @@ def run(self, use_threads=False, quiet=False, data_manager=USE_MP, a DataSet object containing the results of the measurement """ - data_set = self._dummyLoop.get_data_set(data_manager=data_manager, - **kwargs) + data_set = self._dummyLoop.get_data_set(**kwargs) # set the DataSet to local for now so we don't save it, since # we're going to massage it afterward @@ -125,8 +122,7 @@ def run(self, use_threads=False, quiet=False, data_manager=USE_MP, # puts in a 'loop' section that we need to replace with 'measurement' # but we use the info from 'loop' to ensure consistency and avoid # duplication. - LOOP_SNAPSHOT_KEYS = ['ts_start', 'ts_end', - 'use_data_manager', 'use_threads'] + LOOP_SNAPSHOT_KEYS = ['ts_start', 'ts_end', 'use_threads'] data_set.add_metadata({'measurement': { k: data_set.metadata['loop'][k] for k in LOOP_SNAPSHOT_KEYS }}) diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index adfc19688e3..ccd7af58181 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -8,7 +8,7 @@ from qcodes.data.data_array import DataArray from qcodes.data.manager import get_data_manager, NoData from qcodes.data.io import DiskIO -from qcodes.data.data_set import load_data, new_data, DataMode, DataSet +from qcodes.data.data_set import load_data, new_data, DataSet from qcodes.process.helpers import kill_processes from qcodes.utils.helpers import LogCapture from qcodes import active_children @@ -287,19 +287,6 @@ class TestLoadData(TestCase): def setUp(self): kill_processes() - def test_no_live_data(self): - # live data with no DataManager at all - with self.assertRaises(RuntimeError): - load_data() - self.assertEqual(len(active_children()), 0) - - # now make a DataManager and try again - get_data_manager() - self.assertEqual(len(active_children()), 1) - # same result but different code path - with self.assertRaises(RuntimeError): - load_data() - def test_no_saved_data(self): with self.assertRaises(IOError): load_data('_no/such/file_') @@ -308,34 +295,11 @@ def test_load_false(self): with self.assertRaises(ValueError): load_data(False) - def test_get_live(self): - loc = 'live from New York!' - - class MockLive: - pass - - live_data = MockLive() - - dm = MockDataManager() - dm.location = loc - dm.live_data = live_data - - data = load_data(data_manager=dm, location=loc) - self.assertEqual(data, live_data) - - for nd in (None, NoData()): - dm.live_data = nd - with self.assertRaises(RuntimeError): - load_data(data_manager=dm, location=loc) - with self.assertRaises(RuntimeError): - load_data(data_manager=dm) - def test_get_read(self): dm = MockDataManager() dm.location = 'somewhere else' - data = load_data(formatter=MockFormatter(), data_manager=dm, - location='here!') + data = load_data(formatter=MockFormatter(), location='here!') self.assertEqual(data.has_read_data, True) self.assertEqual(data.has_read_metadata, True) @@ -393,16 +357,11 @@ def test_overwrite(self): io = MatchIO([1]) with self.assertRaises(FileExistsError): - new_data(location='somewhere', io=io, data_manager=False) + new_data(location='somewhere', io=io) - data = new_data(location='somewhere', io=io, overwrite=True, - data_manager=False) + data = new_data(location='somewhere', io=io, overwrite=True,) self.assertEqual(data.location, 'somewhere') - def test_mode_error(self): - with self.assertRaises(ValueError): - new_data(mode=DataMode.PUSH_TO_SERVER, data_manager=False) - def test_location_functions(self): def my_location(io, record): return 'data/{}'.format((record or {}).get('name') or 'LOOP!') @@ -413,14 +372,12 @@ def my_location2(io, record): DataSet.location_provider = my_location - self.assertEqual(new_data(data_manager=False).location, 'data/LOOP!') - self.assertEqual(new_data(data_manager=False, name='cheese').location, - 'data/cheese') + self.assertEqual(new_data().location, 'data/LOOP!') + self.assertEqual(new_data(name='cheese').location, 'data/cheese') - data = new_data(data_manager=False, location=my_location2) + data = new_data(location=my_location2) self.assertEqual(data.location, 'data/loop?/folder') - data = new_data(data_manager=False, location=my_location2, - name='iceCream') + data = new_data(location=my_location2, name='iceCream') self.assertEqual(data.location, 'data/iceCream/folder') @@ -437,87 +394,6 @@ def test_constructor_errors(self): with self.assertRaises(ValueError): DataSet(location=42) - # OK to have location=False, but wrong mode - with self.assertRaises(ValueError): - DataSet(location=False, mode='happy') - - @patch('qcodes.data.data_set.get_data_manager') - def test_from_server(self, gdm_mock): - mock_dm = MockDataManager() - gdm_mock.return_value = mock_dm - mock_dm.location = 'Mars' - mock_dm.live_data = MockLive() - - # wrong location or False location - converts to local - data = DataSet(location='Jupiter', data_manager=True, mode=DataMode.PULL_FROM_SERVER) - self.assertEqual(data.mode, DataMode.LOCAL) - - data = DataSet(location=False, data_manager=True, mode=DataMode.PULL_FROM_SERVER) - self.assertEqual(data.mode, DataMode.LOCAL) - - # location matching server - stays in server mode - data = DataSet(location='Mars', data_manager=True, mode=DataMode.PULL_FROM_SERVER, - formatter=MockFormatter()) - self.assertEqual(data.mode, DataMode.PULL_FROM_SERVER) - self.assertEqual(data.arrays, MockLive.arrays) - - # cannot write except in LOCAL mode - with self.assertRaises(RuntimeError): - data.write() - - # cannot finalize in PULL_FROM_SERVER mode - with self.assertRaises(RuntimeError): - data.finalize() - - # now test when the server says it's not there anymore - mock_dm.location = 'Saturn' - data.sync() - self.assertEqual(data.mode, DataMode.LOCAL) - self.assertEqual(data.has_read_data, True) - - # now it's LOCAL so we *can* write. - data.write() - self.assertEqual(data.has_written_data, True) - - # location=False: write, read and sync are noops. - data.has_read_data = False - data.has_written_data = False - data.location = False - data.write() - data.read() - data.sync() - self.assertEqual(data.has_read_data, False) - self.assertEqual(data.has_written_data, False) - - @patch('qcodes.data.data_set.get_data_manager') - def test_to_server(self, gdm_mock): - mock_dm = MockDataManager() - mock_dm.needs_restart = True - gdm_mock.return_value = mock_dm - - data = DataSet(location='Venus', data_manager=True, mode=DataMode.PUSH_TO_SERVER) - self.assertEqual(mock_dm.needs_restart, False, data) - self.assertEqual(mock_dm.data_set, data) - self.assertEqual(data.data_manager, mock_dm) - self.assertEqual(data.mode, DataMode.PUSH_TO_SERVER) - - # cannot write except in LOCAL mode - with self.assertRaises(RuntimeError): - data.write() - - # now do what the DataServer does with this DataSet: init_on_server - # fails until there is an array - with self.assertRaises(RuntimeError): - data.init_on_server() - - data.add_array(MockArray()) - data.init_on_server() - self.assertEqual(data.noise.ready, True) - - # we can only add a given array_id once - with self.assertRaises(ValueError): - data.add_array(MockArray()) - def test_write_copy(self): data = DataSet1D(location=False) mockbase = os.path.abspath('some_folder') diff --git a/qcodes/tests/test_format.py b/qcodes/tests/test_format.py index d20b75ebd5f..c9a635ae316 100644 --- a/qcodes/tests/test_format.py +++ b/qcodes/tests/test_format.py @@ -333,8 +333,8 @@ def test_incremental_write(self): # we wrote to a second location without the stars, so we can read # back in and make sure that we get the right last_saved_index # for the amount of data we've read. - reread_data = load_data(location=location2, data_manager=False, - formatter=formatter, io=data.io) + reread_data = load_data(location=location2, formatter=formatter, + io=data.io) self.assertEqual(repr(reread_data.x_set.tolist()), repr(data.x_set.tolist())) self.assertEqual(repr(reread_data.y.tolist()), diff --git a/qcodes/tests/test_hdf5formatter.py b/qcodes/tests/test_hdf5formatter.py index 4f759f1d8bf..ac813edc3cf 100644 --- a/qcodes/tests/test_hdf5formatter.py +++ b/qcodes/tests/test_hdf5formatter.py @@ -136,8 +136,7 @@ def test_loop_writing(self): # # added to station to test snapshot at a later stage loop = Loop(MockPar.x[-100:100:20]).each(MockPar.skewed_parabola) data1 = loop.run(name='MockLoop_hdf5_test', - formatter=self.formatter, - data_manager=False) + formatter=self.formatter) data2 = DataSet(location=data1.location, formatter=self.formatter) data2.read() for key in data2.arrays.keys(): @@ -158,8 +157,7 @@ def test_loop_writing_2D(self): loop = Loop(MockPar.x[-100:100:20]).loop( MockPar.y[-50:50:10]).each(MockPar.skewed_parabola) data1 = loop.run(name='MockLoop_hdf5_test', - formatter=self.formatter, - data_manager=False) + formatter=self.formatter) data2 = DataSet(location=data1.location, formatter=self.formatter) data2.read() for key in data2.arrays.keys(): diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index cbe082bc7ff..ae76594936a 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -83,7 +83,6 @@ def test_repr(self): active_loop = loop data = active_loop.run_temp() expected = ('DataSet:\n' - ' mode = DataMode.LOCAL\n' ' location = False\n' ' | | | \n' ' Setpoint | p1_set | p1 | (2,)\n' @@ -402,7 +401,6 @@ def g(): }, 'loop': { 'use_threads': False, - 'use_data_manager': False, '__class__': 'qcodes.loops.ActiveLoop', 'sweep_values': { 'parameter': p1snap, @@ -482,7 +480,7 @@ def test_halt(self): # because loop.run will not return. data = loop.get_data_set(location=False) - loop.run(data_manager=False, quiet=True) + loop.run(quiet=True) self.assertEqual(repr(data.p1.tolist()), repr(self.res)) diff --git a/qcodes/tests/test_measure.py b/qcodes/tests/test_measure.py index 8563c34559e..28e8953887b 100644 --- a/qcodes/tests/test_measure.py +++ b/qcodes/tests/test_measure.py @@ -25,7 +25,6 @@ def test_simple_scalar(self): meta = data.metadata['measurement'] self.assertEqual(meta['__class__'], 'qcodes.measure.Measure') self.assertEqual(len(meta['actions']), 1) - self.assertFalse(meta['use_data_manager']) self.assertFalse(meta['use_threads']) ts_start = datetime.strptime(meta['ts_start'], '%Y-%m-%d %H:%M:%S') From 496783d9e72effe4b8176ae39bac5f7b9746961e Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 16:03:06 +0100 Subject: [PATCH 08/36] feature: Remove tests, yolo --- qcodes/tests/test_instrument.py | 963 +------------------------------- 1 file changed, 2 insertions(+), 961 deletions(-) diff --git a/qcodes/tests/test_instrument.py b/qcodes/tests/test_instrument.py index 8dc5703de62..d4a94c92832 100644 --- a/qcodes/tests/test_instrument.py +++ b/qcodes/tests/test_instrument.py @@ -1,968 +1,9 @@ """ Test suite for instument.* """ -import gc -from datetime import datetime, timedelta from unittest import TestCase -import time -from qcodes.instrument.base import Instrument -from qcodes.instrument.mock import MockInstrument -from qcodes.instrument.parameter import ManualParameter -from qcodes.instrument.server import get_instrument_server_manager - -from qcodes.utils.validators import Numbers, Ints, Strings, MultiType, Enum -from qcodes.utils.command import NoCommandError -from qcodes.utils.helpers import LogCapture -from qcodes.process.helpers import kill_processes - -from .instrument_mocks import (AMockModel, MockInstTester, - MockGates, MockSource, MockMeter, - DummyInstrument) -from .common import strip_qc - - -class GatesBadDelayType(MockGates): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_parameter('chan0bad', get_cmd='c0?', - set_cmd=self.slow_neg_set, - get_parser=float, - vals=Numbers(-10, 10), step=0.2, - delay=0.01, - max_delay='forever') - - -class GatesBadDelayValue(MockGates): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_parameter('chan0bad', get_cmd='c0?', - set_cmd=self.slow_neg_set, - get_parser=float, - vals=Numbers(-10, 10), step=0.2, - delay=0.05, - max_delay=0.03) - - -class TestInstrument(TestCase): - - @classmethod - def setUpClass(cls): - cls.model = AMockModel() - - cls.gates = MockGates(model=cls.model, server_name='') - cls.source = MockSource(model=cls.model, server_name='') - cls.meter = MockMeter( - model=cls.model, keep_history=False, server_name='') - - def setUp(self): - # reset the model state via the gates function - self.gates.reset() - - # then reset each instrument's state, so we can avoid the time to - # completely reinstantiate with every test case - for inst in (self.gates, self.source, self.meter): - inst.restart() - self.init_ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - - @classmethod - def tearDownClass(cls): - try: - cls.model.close() - for instrument in [cls.gates, cls.source, cls.meter]: - instrument.close() - # do it twice - should not error, though the second is - # irrelevant - instrument.close() - except: - pass - - # TODO: when an error occurs during constructing an instrument, - # we don't have the instrument but its server doesn't know to stop. - # should figure out a way to remove it. (I thought I had but it - # doesn't seem to have worked...) - # for test_mock_instrument_errors - kill_processes() - - def test_unpicklable(self): - self.assertEqual(self.gates.add5(6), 11) - # compare docstrings to make sure we're really calling add5 - # on the server, and seeing its docstring - self.assertIn('The class copy of this should not get run', - MockInstTester.add5.__doc__) - self.assertIn('not the same function as the original method', - self.gates.add5.__doc__) - - def test_slow_set(self): - # at least for now, need a local instrument to test logging - gatesLocal = MockGates(model=self.model, server_name=None, - name='gateslocal') - for param, logcount in (('chan0slow', 2), ('chan0slow2', 2), - ('chan0slow3', 0), ('chan0slow4', 1), - ('chan0slow5', 0)): - gatesLocal.chan0.set(-0.5) - - with LogCapture() as logs: - if param in ('chan0slow', 'chan0slow2', 'chan0slow3'): - # these are the stepped parameters - gatesLocal.set(param, 0.5) - else: - # these are the non-stepped parameters that - # still have delays - gatesLocal.set(param, -1) - gatesLocal.set(param, 1) - - loglines = logs.value.split('\n')[:-1] - # TODO: occasional extra negative delays here - self.assertEqual(len(loglines), logcount, (param, logs.value)) - for line in loglines: - self.assertTrue(line.startswith('negative delay'), line) - - def test_max_delay_errors(self): - with self.assertRaises(TypeError): - # add_parameter works remotely with string commands, but - # function commands are not going to be picklable, since they - # need to talk to the hardware, so these need to be included - # from the beginning when the instrument is created on the - # server. - GatesBadDelayType(model=self.model, name='gatesBDT') - - with self.assertRaises(ValueError): - GatesBadDelayValue(model=self.model, name='gatesBDV') - - def check_ts(self, ts_str): - now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - self.assertTrue(self.init_ts <= ts_str <= now) - - def test_instances(self): - instruments = [self.gates, self.source, self.meter] - for instrument in instruments: - for other_instrument in instruments: - instances = instrument.instances() - # check that each instrument is in only its own - # instances list - # also test type checking in find_instrument, - # but we need to use find_component so it executes - # on the server - if other_instrument is instrument: - self.assertIn(instrument, instances) - - name2 = other_instrument.find_component( - instrument.name + '.name', - other_instrument._instrument_class) - self.assertEqual(name2, instrument.name) - else: - self.assertNotIn(other_instrument, instances) - - with self.assertRaises(TypeError): - other_instrument.find_component( - instrument.name + '.name', - other_instrument._instrument_class) - - # check that we can find each instrument from any other - # find_instrument is explicitly mapped in RemoteInstrument - # so this call gets executed in the main process - self.assertEqual( - instrument, - other_instrument.find_instrument(instrument.name)) - - # but find_component is not, so it executes on the server - self.assertEqual( - instrument.name, - other_instrument.find_component(instrument.name + '.name')) - - # check that we can find this instrument from the base class - self.assertEqual(instrument, - Instrument.find_instrument(instrument.name)) - - # somehow instances never go away... there are always 3 - # extra references to every instrument object, so del doesn't - # work. For this reason, instrument tests should take - # the *last* instance to test. - # so we can't test that the list of defined instruments is actually - # *only* what we want to see defined. - - def test_instance_name_uniqueness(self): - with self.assertRaises(KeyError): - MockGates(model=self.model) - - def test_remove_instance(self): - self.gates.close() - self.assertEqual(self.gates.instances(), []) - with self.assertRaises(KeyError): - Instrument.find_instrument('gates') - - type(self).gates = MockGates(model=self.model, server_name="") - self.assertEqual(self.gates.instances(), [self.gates]) - self.assertEqual(Instrument.find_instrument('gates'), self.gates) - - def test_creation_failure(self): - # this we already know should fail (see test_max_delay_errors) - name = 'gatesFailing' - with self.assertRaises(ValueError): - GatesBadDelayValue(model=self.model, name=name, server_name='') - - # this instrument should not be in the instance list - with self.assertRaises(KeyError): - Instrument.find_instrument(name) - - # now do the same with a local instrument - name = 'gatesFailing2' - with self.assertRaises(ValueError): - GatesBadDelayValue(model=self.model, name=name, server_name=None) - # gc is confused by the context manager we just used - # we want to make sure the ojbect we just tried to create - # but it threw an exception is properly gc'ed. Which - # is what happens wihtout the context manager AND - # it's how the __del__ method is supposed to work. - - gc.collect() - - # this instrument should not be in the instance list - with self.assertRaises(KeyError): - Instrument.find_instrument(name) - - def test_mock_instrument(self): - gates, source, meter = self.gates, self.source, self.meter - - # initial state - # short form of getter - self.assertEqual(gates.get('chan0'), 0) - # shortcut to the parameter, longer form of get - self.assertEqual(gates['chan0'].get(), 0) - # explicit long form of getter - self.assertEqual(gates.parameters['chan0'].get(), 0) - # all 3 should produce the same history entry - hist = gates.getattr('history') - self.assertEqual(len(hist), 3) - for item in hist: - self.assertEqual(item[1:], ('ask', 'c0')) - - # errors trying to set (or validate) invalid param values - # put here so we ensure that these errors don't make it to - # the history (ie they don't result in hardware commands) - with self.assertRaises(TypeError): - gates.set('chan1', '1') - with self.assertRaises(TypeError): - gates.parameters['chan1'].validate('1') - - # change one param at a time - gates.set('chan0', 0.5) - self.assertEqual(gates.get('chan0'), 0.5) - self.assertEqual(meter.get('amplitude'), 0.05) - - gates.set('chan1', 2) - self.assertEqual(gates.get('chan1'), 2) - self.assertEqual(meter.get('amplitude'), 0.45) - - gates.set('chan2', -3.2) - self.assertEqual(gates.get('chan2'), -3.2) - self.assertEqual(meter.get('amplitude'), -2.827) - - source.set('amplitude', 0.6) - self.assertEqual(source.get('amplitude'), 0.6) - self.assertEqual(meter.get('amplitude'), -16.961) - - gatehist = gates.getattr('history') - sourcehist = source.getattr('history') - meterhist = meter.getattr('history') - # check just the size and timestamps of histories - for entry in gatehist + sourcehist + meterhist: - self.check_ts(entry[0]) - self.assertEqual(len(gatehist), 9) - self.assertEqual(len(sourcehist), 5) - # meter does not keep history but should still have a history attr - self.assertEqual(len(meterhist), 0) - - # plus enough setters to check the parameter sweep - # first source has to get the starting value - self.assertEqual(sourcehist[0][1:], ('ask', 'ampl')) - # then it writes each - self.assertEqual(sourcehist[1][1:], ('write', 'ampl', '0.3000')) - self.assertEqual(sourcehist[2][1:], ('write', 'ampl', '0.5000')) - self.assertEqual(sourcehist[3][1:], ('write', 'ampl', '0.6000')) - - source.set('amplitude', 0.8) - self.assertEqual(source.get('amplitude'), 0.8) - gates.set('chan1', -2) - self.assertEqual(gates.get('chan1'), -2) - - # test functions - self.assertEqual(meter.call('echo', 1.2345), 1.23) # model returns .2f - # too many ways to do this... - self.assertEqual(meter.echo.call(1.2345), 1.23) - self.assertEqual(meter.echo(1.2345), 1.23) - self.assertEqual(meter['echo'].call(1.2345), 1.23) - self.assertEqual(meter['echo'](1.2345), 1.23) - with self.assertRaises(TypeError): - meter.call('echo', 1, 2) - with self.assertRaises(TypeError): - meter.call('echo', '1') - - # validating before actually trying to call - with self.assertRaises(TypeError): - meter.functions['echo'].validate(1, 2) - with self.assertRaises(TypeError): - meter.functions['echo'].validate('1') - gates.call('reset') - self.assertEqual(gates.get('chan0'), 0) - - self.assertEqual(meter.call('echo', 4.567), 4.57) - gates.set('chan0', 1) - self.assertEqual(gates.get('chan0'), 1) - gates.call('reset') - self.assertEqual(gates.get('chan0'), 0) - - def test_mock_idn(self): - self.assertEqual(self.gates.IDN(), { - 'vendor': None, - 'model': 'MockGates', - 'serial': 'gates', - 'firmware': None - }) - - def test_mock_set_sweep(self): - gates = self.gates - gates.set('chan0step', 0.5) - gatehist = gates.getattr('history') - self.assertEqual(len(gatehist), 6) - self.assertEqual( - [float(h[3]) for h in gatehist if h[1] == 'write'], - [0.1, 0.2, 0.3, 0.4, 0.5]) - - def test_mock_instrument_errors(self): - gates, meter = self.gates, self.meter - with self.assertRaises(ValueError): - gates.ask('no question') - with self.assertRaises(ValueError): - gates.ask('question?yes but more after') - - with self.assertRaises(ValueError): - gates.ask('ampl?') - - with self.assertRaises(TypeError): - MockInstrument('mockbaddelay1', delay='forever') - with self.assertRaises(TypeError): - # TODO: since this instrument didn't work, it should be OK - # to use the same name again... how do we allow that? - MockInstrument('mockbaddelay2', delay=-1) - - # TODO: when an error occurs during constructing an instrument, - # we don't have the instrument but its server doesn't know to stop. - # should figure out a way to remove it. (I thought I had but it - # doesn't seem to have worked...) - get_instrument_server_manager('MockInstruments').close() - time.sleep(0.5) - - with self.assertRaises(AttributeError): - MockInstrument('', model=None) - - with self.assertRaises(KeyError): - gates.add_parameter('chan0', get_cmd='boo') - with self.assertRaises(KeyError): - gates.add_function('reset', call_cmd='hoo') - - with self.assertRaises(NotImplementedError): - meter.set('amplitude', 0.5) - meter.add_parameter('gain', set_cmd='gain {:.3f}') - with self.assertRaises(NotImplementedError): - meter.get('gain') - - with self.assertRaises(TypeError): - gates.add_parameter('fugacity', set_cmd='f {:.4f}', vals=[1, 2, 3]) - - def check_set_amplitude2(self, val, log_count, history_count): - source = self.sourceLocal - with LogCapture() as logs: - source.amplitude2.set(val) - - loglines = logs.value.split('\n')[:-1] - - self.assertEqual(len(loglines), log_count, logs.value) - for line in loglines: - self.assertIn('cannot sweep', line.lower()) - hist = source.getattr('history') - self.assertEqual(len(hist), history_count) - - def test_sweep_steps_edge_case(self): - # MultiType with sweeping is weird - not sure why one would do this, - # but we should handle it - # at least for now, need a local instrument to check logging - source = self.sourceLocal = MockSource(model=self.model, - server_name=None, - name='sourcelocal') - source.add_parameter('amplitude2', get_cmd='ampl?', - set_cmd='ampl:{}', get_parser=float, - vals=MultiType(Numbers(0, 1), Strings()), - step=0.2, delay=0.02) - self.assertEqual(len(source.getattr('history')), 0) - - # 2 history items - get then set, and one warning (cannot sweep - # number to string value) - self.check_set_amplitude2('Off', log_count=1, history_count=2) - - # one more history item - single set, and one warning (cannot sweep - # string to number) - self.check_set_amplitude2(0.2, log_count=1, history_count=3) - - # the only real sweep (0.2 to 0.8) adds 3 set's to history and no logs - self.check_set_amplitude2(0.8, log_count=0, history_count=6) - - # single set added to history, and another sweep warning num->string - self.check_set_amplitude2('Off', log_count=1, history_count=7) - - def test_set_sweep_errors(self): - gates = self.gates - - # for reference, some add_parameter's that should work - gates.add_parameter('t0', set_cmd='{}', vals=Numbers(), - step=0.1, delay=0.01) - gates.add_parameter('t2', set_cmd='{}', vals=Ints(), - step=1, delay=0.01, - max_val_age=0) - - with self.assertRaises(TypeError): - # can't sweep non-numerics - gates.add_parameter('t1', set_cmd='{}', vals=Strings(), - step=1, delay=0.01) - with self.assertRaises(TypeError): - # need a numeric step too - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step='a skosh', delay=0.01) - with self.assertRaises(TypeError): - # Ints requires and int step - gates.add_parameter('t1', set_cmd='{}', vals=Ints(), - step=0.1, delay=0.01) - with self.assertRaises(ValueError): - # need a non-negative step - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step=-0.1, delay=0.01) - with self.assertRaises(TypeError): - # need a numeric delay - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step=0.1, delay='a tad') - with self.assertRaises(ValueError): - # need a non-negative delay - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step=0.1, delay=-0.01) - with self.assertRaises(TypeError): - # need a numeric max_val_age - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step=0.1, delay=0.01, - max_val_age='an hour') - with self.assertRaises(ValueError): - # need a non-negative max_val_age - gates.add_parameter('t1', set_cmd='{}', vals=Numbers(), - step=0.1, delay=0.01, - max_val_age=-1) - - def getmem(self, key): - return self.source.ask('mem{}?'.format(key)) - - def test_val_mapping(self): - gates = self.gates - - # memraw has no mappings - it just sets and gets what the instrument - # uses to encode this parameter - gates.add_parameter('memraw', set_cmd='mem0:{}', get_cmd='mem0?', - vals=Enum('zero', 'one')) - - # memcoded maps the instrument codes ('zero' and 'one') into nicer - # user values 0 and 1 - gates.add_parameter('memcoded', set_cmd='mem0:{}', get_cmd='mem0?', - val_mapping={0: 'zero', 1: 'one'}) - - gates.memcoded.set(0) - self.assertEqual(gates.memraw.get(), 'zero') - self.assertEqual(gates.memcoded.get(), 0) - self.assertEqual(self.getmem(0), 'zero') - - gates.memraw.set('one') - self.assertEqual(gates.memcoded.get(), 1) - self.assertEqual(gates.memraw.get(), 'one') - self.assertEqual(self.getmem(0), 'one') - - with self.assertRaises(ValueError): - gates.memraw.set(0) - - with self.assertRaises(ValueError): - gates.memcoded.set('zero') - - def test_val_mapping_ints(self): - gates = self.gates - - gates.add_parameter('moderaw', set_cmd='mem0:{}', get_cmd='mem0?', - vals=Enum('0', '1')) - - # modecoded maps the instrument codes ('0' and '1') into nicer - # user values 'AC' and 'DC' - # Here we're using integers in the mapping, rather than turning - # them into strings. - gates.add_parameter('modecoded', set_cmd='mem0:{}', get_cmd='mem0?', - val_mapping={'DC': 0, 'AC': 1}) - - gates.modecoded.set('AC') - self.assertEqual(gates.moderaw.get(), '1') - self.assertEqual(gates.modecoded.get(), 'AC') - self.assertEqual(self.getmem(0), '1') - - gates.moderaw.set('0') - self.assertEqual(gates.modecoded.get(), 'DC') - self.assertEqual(gates.moderaw.get(), '0') - self.assertEqual(self.getmem(0), '0') - - with self.assertRaises(ValueError): - gates.modecoded.set(0) - - with self.assertRaises(ValueError): - gates.modecoded.set('0') - - with self.assertRaises(ValueError): - gates.moderaw.set('DC') - - def test_val_mapping_parsers(self): - gates = self.gates - - gates.add_parameter('moderaw', set_cmd='mem0:{}', get_cmd='mem0?', - vals=Enum('0', '1')) - - with self.assertRaises(TypeError): - # set_parser is not allowed with val_mapping - gates.add_parameter('modecoded', set_cmd='mem0:{}', - get_cmd='mem0?', - val_mapping={'DC': 0, 'AC': 1}, - set_parser=float) - - gates.add_parameter('modecoded', set_cmd='mem0:{:.0f}', - get_cmd='mem0?', - val_mapping={'DC': 0.0, 'AC': 1.0}, - get_parser=float) - - gates.modecoded.set('AC') - self.assertEqual(gates.moderaw.get(), '1') - self.assertEqual(gates.modecoded.get(), 'AC') - self.assertEqual(self.getmem(0), '1') - - gates.moderaw.set('0') - self.assertEqual(gates.modecoded.get(), 'DC') - self.assertEqual(gates.moderaw.get(), '0') - self.assertEqual(self.getmem(0), '0') - - with self.assertRaises(ValueError): - gates.modecoded.set(0) - - with self.assertRaises(ValueError): - gates.modecoded.set('0') - - def test_standard_snapshot(self): - self.maxDiff = None - snap = self.meter.snapshot() - strip_qc(snap) - for psnap in snap['parameters'].values(): - strip_qc(psnap) - - self.assertEqual(snap, { - '__class__': 'tests.instrument_mocks.MockMeter', - 'name': 'meter', - 'parameters': { - 'IDN': { - '__class__': ( - 'qcodes.instrument.parameter.StandardParameter'), - 'instrument': 'tests.instrument_mocks.MockMeter', - 'instrument_name': 'meter', - 'label': 'IDN', - 'name': 'IDN', - 'ts': None, - 'unit': '', - 'value': None, - 'vals': '' - }, - 'amplitude': { - '__class__': ( - 'qcodes.instrument.parameter.StandardParameter'), - 'instrument': 'tests.instrument_mocks.MockMeter', - 'instrument_name': 'meter', - 'label': 'amplitude', - 'name': 'amplitude', - 'ts': None, - 'unit': '', - 'value': None, - 'vals': '' - } - }, - 'functions': {'echo': {}} - }) - - ampsnap = self.meter.snapshot(update=True)['parameters']['amplitude'] - amp = self.meter.get('amplitude') - self.assertEqual(ampsnap['value'], amp) - amp_ts = datetime.strptime(ampsnap['ts'], '%Y-%m-%d %H:%M:%S') - self.assertLessEqual(amp_ts, datetime.now()) - self.assertGreater(amp_ts, datetime.now() - timedelta(seconds=1.1)) - - def test_manual_snapshot(self): - self.source.add_parameter('noise', parameter_class=ManualParameter) - noise = self.source.noise - - noisesnap = self.source.snapshot()['parameters']['noise'] - strip_qc(noisesnap) - self.assertEqual(noisesnap, { - '__class__': 'qcodes.instrument.parameter.ManualParameter', - 'instrument': 'tests.instrument_mocks.MockSource', - 'instrument_name': 'source', - 'label': 'noise', - 'name': 'noise', - 'ts': None, - 'unit': '', - 'value': None, - 'vals': '' - }) - - noise.set(100) - noisesnap = self.source.snapshot()['parameters']['noise'] - self.assertEqual(noisesnap['value'], 100) - - noise_ts = datetime.strptime(noisesnap['ts'], '%Y-%m-%d %H:%M:%S') - self.assertLessEqual(noise_ts, datetime.now()) - self.assertGreater(noise_ts, datetime.now() - timedelta(seconds=1.1)) - - def tests_get_latest(self): - self.source.add_parameter('noise', parameter_class=ManualParameter) - noise = self.source.noise - - self.assertIsNone(noise.get_latest()) - - noise.set(100) - - mock_ts = datetime(2000, 3, 4) - ts_str = mock_ts.strftime('%Y-%m-%d %H:%M:%S') - noise.setattr('_latest_ts', mock_ts) - self.assertEqual(noise.snapshot()['ts'], ts_str) - - self.assertEqual(noise.get_latest(), 100) - self.assertEqual(noise.get_latest.get(), 100) - - # get_latest should not update ts - self.assertEqual(noise.snapshot()['ts'], ts_str) - - # get_latest is not settable - with self.assertRaises(AttributeError): - noise.get_latest.set(50) - - def test_base_instrument_errors(self): - b = Instrument('silent', server_name=None) - - with self.assertRaises(NotImplementedError): - b.write('hello!') - with self.assertRaises(NotImplementedError): - b.ask('how are you?') - - with self.assertRaises(TypeError): - b.add_function('skip', call_cmd='skip {}', - args=['not a validator']) - with self.assertRaises(NoCommandError): - b.add_function('jump') - with self.assertRaises(NoCommandError): - b.add_parameter('height') - - def test_manual_parameter(self): - self.source.add_parameter('bias_resistor', - parameter_class=ManualParameter, - initial_value=1000) - res = self.source.bias_resistor - self.assertEqual(res.get(), 1000) - - res.set(1e9) - self.assertEqual(res.get(), 1e9) - # default vals is all numbers - # set / get with __call__ shortcut - res(-1) - self.assertEqual(res(), -1) - - self.source.add_parameter('alignment', - parameter_class=ManualParameter, - vals=Enum('lawful', 'neutral', 'chaotic')) - alignment = self.source.alignment - - # a ManualParameter can have initial_value=None (default) even if - # that's not a valid value to set later - self.assertIsNone(alignment.get()) - with self.assertRaises(ValueError): - alignment.set(None) - - alignment.set('lawful') - self.assertEqual(alignment.get(), 'lawful') - - # None is the only invalid initial_value you can use - with self.assertRaises(TypeError): - self.source.add_parameter('alignment2', - parameter_class=ManualParameter, - initial_value='nearsighted') - - def test_deferred_ops(self): - gates = self.gates - c0, c1, c2 = gates.chan0, gates.chan1, gates.chan2 - - c0.set(0) - c1.set(1) - c2.set(2) - - self.assertEqual((c0 + c1 + c2)(), 3) - self.assertEqual((10 + (c0**2) + (c1**2) + (c2**2))(), 15) - - d = c1.get_latest / c0.get_latest - with self.assertRaises(ZeroDivisionError): - d() - - def test_attr_access(self): - instrument = self.gates - - # set one attribute with nested levels - instrument.setattr('d1', {'a': {1: 2}}) - - # get the whole dict - self.assertEqual(instrument.getattr('d1'), {'a': {1: 2}}) - self.assertEqual(instrument.getattr('d1', 55), {'a': {1: 2}}) - - # get parts - self.assertEqual(instrument.getattr('d1["a"]'), {1: 2}) - self.assertEqual(instrument.getattr("d1['a'][1]"), 2) - self.assertEqual(instrument.getattr('d1["a"][1]', 3), 2) - - # add an attribute inside, then delete it again - instrument.setattr('d1["a"][2]', 23) - self.assertEqual(instrument.getattr('d1'), {'a': {1: 2, 2: 23}}) - instrument.delattr('d1["a"][2]') - self.assertEqual(instrument.getattr('d1'), {'a': {1: 2}}) - - # test restarting the InstrumentServer - this clears these attrs - instrument._manager.restart() - self.assertIsNone(instrument.getattr('d1', None)) - - def test_component_attr_access(self): - instrument = self.gates - method = instrument.add5 - parameter = instrument.chan1 - function = instrument.reset - - # RemoteMethod objects have no attributes besides __doc__, so test - # that this gets appropriately decorated - self.assertIn('RemoteMethod add5 in RemoteInstrument', method.__doc__) - # and also contains the remote doc - self.assertIn('not the same function as the original method', - method.__doc__) - - # unit is a remote attribute of parameters - # this one is initially blank - self.assertEqual(parameter.unit, '') - parameter.unit = 'Smoots' - self.assertEqual(parameter.unit, 'Smoots') - self.assertNotIn('unit', parameter.__dict__) - self.assertEqual(instrument.getattr(parameter.name + '.unit'), - 'Smoots') - # we can delete it remotely, and this is reflected in dir() - self.assertIn('unit', dir(parameter)) - del parameter.unit - self.assertNotIn('unit', dir(parameter)) - with self.assertRaises(AttributeError): - parameter.unit - - # and set it again, it's still remote. - parameter.unit = 'Furlongs per fortnight' - self.assertIn('unit', dir(parameter)) - self.assertEqual(parameter.unit, 'Furlongs per fortnight') - self.assertNotIn('unit', parameter.__dict__) - self.assertEqual(instrument.getattr(parameter.name + '.unit'), - 'Furlongs per fortnight') - # we get the correct result if someone else sets it on the server - instrument._write_server('setattr', parameter.name + '.unit', 'T') - self.assertEqual(parameter.unit, 'T') - self.assertEqual(parameter.getattr('unit'), 'T') - - # attributes not specified as remote are local - with self.assertRaises(AttributeError): - parameter.something - parameter.something = 42 - self.assertEqual(parameter.something, 42) - self.assertEqual(parameter.__dict__['something'], 42) - with self.assertRaises(AttributeError): - instrument.getattr(parameter.name + '.something') - with self.assertRaises(AttributeError): - # getattr method is only for remote attributes - parameter.getattr('something') - self.assertIn('something', dir(parameter)) - del parameter.something - self.assertNotIn('something', dir(parameter)) - with self.assertRaises(AttributeError): - parameter.something - - # call a remote method - self.assertEqual(set(parameter.callattr('get_attrs')), - parameter._attrs) - - # functions have remote attributes too - self.assertEqual(function._args, []) - self.assertNotIn('_args', function.__dict__) - function._args = 'args!' - self.assertEqual(function._args, 'args!') - - # a component with no docstring still gets the decoration - foo = instrument.foo - self.assertEqual(foo.__doc__, - 'RemoteParameter foo in RemoteInstrument gates') - - def test_update_components(self): - gates = self.gates - - gates.delattr('chan0.label') - gates.setattr('chan0.cheese', 'gorgonzola') - # we've altered the server copy, but not the RemoteParameter - self.assertIn('label', gates.chan0._attrs) - self.assertNotIn('cheese', gates.chan0._attrs) - # keep a reference to the original chan0 RemoteParameter to make sure - # it is still the same object later - chan0_original = gates.chan0 - - gates.update() - - self.assertIs(gates.chan0, chan0_original) - # now the RemoteParameter should have the updates - self.assertNotIn('label', gates.chan0._attrs) - self.assertIn('cheese', gates.chan0._attrs) - - def test_add_delete_components(self): - gates = self.gates - - # rather than call gates.add_parameter, which has a special proxy - # on the remote so it updates the components immediately, we'll call - # the server version directly - attr_list = gates.callattr('add_parameter', 'chan0X', get_cmd='c0?', - set_cmd='c0:{:.4f}', get_parser=float) - gates.delattr('parameters["chan0"]') - - # the RemoteInstrument does not have these changes yet - self.assertIn('chan0', gates.parameters) - self.assertNotIn('chan0X', gates.parameters) - - gates.update() - - # now the RemoteInstrument has the changes - self.assertNotIn('chan0', gates.parameters) - self.assertIn('chan0X', gates.parameters) - self.assertEqual(gates.chan0X._attrs, set(attr_list)) - - def test_reprs(self): - gates = self.gates - self.assertIn(gates.name, repr(gates)) - self.assertIn('chan1', repr(gates.chan1)) - self.assertIn('reset', repr(gates.reset)) - - def test_remote_sweep_values(self): - chan1 = self.gates.chan1 - - sv1 = chan1[1:4:1] - self.assertEqual(len(sv1), 3) - self.assertIn(2, sv1) - - sv2 = chan1.sweep(start=2, stop=3, num=6) - self.assertEqual(len(sv2), 6) - self.assertIn(2.2, sv2) - - def test_add_function(self): - gates = self.gates - # add a function remotely - gates.add_function('reset2', call_cmd='rst') - gates.chan1(4) - self.assertEqual(gates.chan1(), 4) - gates.reset2() - self.assertEqual(gates.chan1(), 0) - - -class TestLocalMock(TestCase): - - @classmethod - def setUpClass(cls): - cls.model = AMockModel() - - cls.gates = MockGates(model=cls.model, server_name=None) - cls.source = MockSource(model=cls.model, server_name=None) - cls.meter = MockMeter(model=cls.model, server_name=None) - - @classmethod - def tearDownClass(cls): - cls.model.close() - for instrument in [cls.gates, cls.source, cls.meter]: - instrument.close() - - def test_local(self): - self.gates.chan1.set(3.33) - self.assertEqual(self.gates.chan1.get(), 3.33) - - self.gates.reset() - self.assertEqual(self.gates.chan1.get(), 0) - - with self.assertRaises(ValueError): - self.gates.ask('knock knock? Oh never mind.') - - def test_instances(self): - # copied from the main (server-based) version - # make sure it all works the same here - instruments = [self.gates, self.source, self.meter] - for instrument in instruments: - for other_instrument in instruments: - instances = instrument.instances() - # check that each instrument is in only its own - # instances list - if other_instrument is instrument: - self.assertIn(instrument, instances) - else: - self.assertNotIn(other_instrument, instances) - - # check that we can find each instrument from any other - # use find_component here to test that it rolls over to - # find_instrument if only a name is given - self.assertEqual( - instrument, - other_instrument.find_component(instrument.name)) - - self.assertEqual( - instrument.name, - other_instrument.find_component(instrument.name + '.name')) - - # check that we can find this instrument from the base class - self.assertEqual(instrument, - Instrument.find_instrument(instrument.name)) - - -class TestModelAttrAccess(TestCase): - - def setUp(self): - self.model = AMockModel() - - def tearDown(self): - self.model.close() - - def test_attr_access(self): - model = self.model - - model.a = 'local' - with self.assertRaises(AttributeError): - model.getattr('a') - - self.assertEqual(model.getattr('a', 'dflt'), 'dflt') - - model.setattr('a', 'remote') - self.assertEqual(model.a, 'local') - self.assertEqual(model.getattr('a'), 'remote') - - model.delattr('a') - self.assertEqual(model.getattr('a', 'dflt'), 'dflt') - - model.fmt = 'local override of a remote method' - self.assertEqual(model.callattr('fmt', 42), '42.000') - self.assertEqual(model.callattr('fmt', value=12.4), '12.400') +from .instrument_mocks import DummyInstrument class TestInstrument2(TestCase): @@ -995,7 +36,7 @@ def test_attr_access(self): instrument.close() # make sure we can still print the instrument - _ = instrument.__repr__() + instrument.__repr__() # make sure the gate is removed self.assertEqual(hasattr(instrument, 'dac1'), False) From 0bf8f21cd4c95db8e2ba998729df32287ae21767 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 16:13:35 +0100 Subject: [PATCH 09/36] feature: Rremove more mocks --- qcodes/process/__init__.py | 0 qcodes/tests/data_mocks.py | 24 ------------------------ qcodes/tests/test_data.py | 19 +++---------------- 3 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 qcodes/process/__init__.py diff --git a/qcodes/process/__init__.py b/qcodes/process/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/qcodes/tests/data_mocks.py b/qcodes/tests/data_mocks.py index 73cc7c70230..d561064afe2 100644 --- a/qcodes/tests/data_mocks.py +++ b/qcodes/tests/data_mocks.py @@ -1,34 +1,10 @@ import numpy -import multiprocessing as mp from qcodes.data.data_array import DataArray from qcodes.data.data_set import new_data from qcodes.data.io import DiskIO -class MockDataManager: - query_lock = mp.RLock() - - def __init__(self): - self.needs_restart = False - - def ask(self, *args, timeout=None): - if args == ('get_data', 'location'): - return self.location - elif args == ('get_data',): - return self.live_data - elif args[0] == 'new_data' and len(args) == 2: - if self.needs_restart: - raise AttributeError('data_manager needs a restart') - else: - self.data_set = args[1] - else: - raise Exception('unexpected query to MockDataManager') - - def restart(self): - self.needs_restart = False - - class MockFormatter: def read(self, data_set): data_set.has_read_data = True diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index ccd7af58181..d000346f5b1 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -1,21 +1,18 @@ from unittest import TestCase -from unittest.mock import patch import numpy as np import os import pickle import logging from qcodes.data.data_array import DataArray -from qcodes.data.manager import get_data_manager, NoData from qcodes.data.io import DiskIO from qcodes.data.data_set import load_data, new_data, DataSet -from qcodes.process.helpers import kill_processes from qcodes.utils.helpers import LogCapture -from qcodes import active_children -from .data_mocks import (MockDataManager, MockFormatter, MatchIO, - MockLive, MockArray, DataSet2D, DataSet1D, +from .data_mocks import (MockFormatter, MatchIO, + DataSet2D, DataSet1D, DataSetCombined, RecordingMockFormatter) + from .common import strip_qc @@ -284,9 +281,6 @@ def test_fraction_complete(self): class TestLoadData(TestCase): - def setUp(self): - kill_processes() - def test_no_saved_data(self): with self.assertRaises(IOError): load_data('_no/such/file_') @@ -296,9 +290,6 @@ def test_load_false(self): load_data(False) def test_get_read(self): - dm = MockDataManager() - dm.location = 'somewhere else' - data = load_data(formatter=MockFormatter(), location='here!') self.assertEqual(data.has_read_data, True) self.assertEqual(data.has_read_metadata, True) @@ -346,7 +337,6 @@ class TestNewData(TestCase): @classmethod def setUpClass(cls): - kill_processes() cls.original_lp = DataSet.location_provider @classmethod @@ -383,9 +373,6 @@ def my_location2(io, record): class TestDataSet(TestCase): - def tearDown(self): - kill_processes() - def test_constructor_errors(self): # no location - only allowed with load_data with self.assertRaises(ValueError): From c599b8ab25c37c13fa35ce855874ab6d245e8dbc Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Mon, 6 Mar 2017 16:48:16 +0100 Subject: [PATCH 10/36] finally eradicate mp --- qcodes/__init__.py | 7 - qcodes/data/manager.py | 163 -------- qcodes/instrument/base.py | 21 +- qcodes/instrument/mock.py | 280 +------------ qcodes/instrument/remote.py | 529 ------------------------- qcodes/instrument/server.py | 190 --------- qcodes/instrument/visa.py | 21 - qcodes/process/helpers.py | 63 --- qcodes/process/qcodes_process.py | 72 ---- qcodes/process/server.py | 399 ------------------- qcodes/process/stream_queue.py | 152 ------- qcodes/station.py | 9 +- qcodes/tests/instrument_mocks.py | 195 --------- qcodes/tests/test_driver_testcase.py | 58 --- qcodes/tests/test_instrument_server.py | 157 -------- qcodes/tests/test_multiprocessing.py | 471 ---------------------- qcodes/tests/test_nested_attrs.py | 100 ----- qcodes/tests/test_visa.py | 43 -- 18 files changed, 5 insertions(+), 2925 deletions(-) delete mode 100644 qcodes/data/manager.py delete mode 100644 qcodes/instrument/remote.py delete mode 100644 qcodes/instrument/server.py delete mode 100644 qcodes/process/helpers.py delete mode 100644 qcodes/process/qcodes_process.py delete mode 100644 qcodes/process/server.py delete mode 100644 qcodes/process/stream_queue.py delete mode 100644 qcodes/tests/test_driver_testcase.py delete mode 100644 qcodes/tests/test_instrument_server.py delete mode 100644 qcodes/tests/test_multiprocessing.py delete mode 100644 qcodes/tests/test_nested_attrs.py diff --git a/qcodes/__init__.py b/qcodes/__init__.py index b1b2644ccd9..f12b0489fc1 100644 --- a/qcodes/__init__.py +++ b/qcodes/__init__.py @@ -2,10 +2,6 @@ # flake8: noqa (we don't need the "<...> imported but unused" error) -# just for convenience in debugging, so we don't have to -# separately import multiprocessing -from multiprocessing import active_children - # config from qcodes.config import Config @@ -13,7 +9,6 @@ config = Config() from qcodes.version import __version__ -from qcodes.process.helpers import set_mp_method from qcodes.utils.helpers import in_notebook # code that should only be imported into the main (notebook) thread @@ -44,7 +39,6 @@ from qcodes.measure import Measure from qcodes.actions import Task, Wait, BreakIf -from qcodes.data.manager import get_data_manager from qcodes.data.data_set import DataSet, new_data, load_data from qcodes.data.location import FormatLocation from qcodes.data.data_array import DataArray @@ -56,7 +50,6 @@ from qcodes.instrument.base import Instrument from qcodes.instrument.ip import IPInstrument from qcodes.instrument.visa import VisaInstrument -from qcodes.instrument.mock import MockInstrument, MockModel from qcodes.instrument.function import Function from qcodes.instrument.parameter import ( diff --git a/qcodes/data/manager.py b/qcodes/data/manager.py deleted file mode 100644 index 53121dac2fd..00000000000 --- a/qcodes/data/manager.py +++ /dev/null @@ -1,163 +0,0 @@ -from datetime import datetime, timedelta -from queue import Empty -from traceback import format_exc -import logging - -from qcodes.process.server import ServerManager, BaseServer - - -def get_data_manager(only_existing=False): - """ - create or retrieve the storage manager - makes sure we don't accidentally create multiple DataManager processes - """ - dm = DataManager.default - if dm and dm._server.is_alive(): - return dm - elif only_existing: - return None - return DataManager() - - -class NoData: - """ - A placeholder object for DataServer to hold - when there is no loop running. - """ - location = None - - def store(self, *args, **kwargs): - raise RuntimeError('no DataSet to add to') - - def write(self, *args, **kwargs): - pass - - -class DataManager(ServerManager): - default = None - """ - creates a separate process (DataServer) that holds running measurement - and monitor data, and manages writing these to disk or other storage - - DataServer communicates with other processes through messages - Written using multiprocessing Queue's, but should be easily - extensible to other messaging systems - """ - def __init__(self): - type(self).default = self - super().__init__(name='DataServer', server_class=DataServer) - - def restart(self, force=False): - """ - Restart the DataServer - Use force=True to abort a running measurement. - """ - if (not force) and self.ask('get_data', 'location'): - raise RuntimeError('A measurement is running. Use ' - 'restart(force=True) to override.') - super().restart() - - -class DataServer(BaseServer): - """ - Running in its own process, receives, holds, and returns current `Loop` and - monitor data, and writes it to disk (or other storage) - - When a `Loop` is *not* running, the DataServer also calls the monitor - routine. But when a `Loop` *is* running, *it* calls the monitor so that it - can avoid conflicts. Also while a `Loop` is running, there are - complementary `DataSet` objects in the loop and `DataServer` processes - - they are nearly identical objects, but are configured differently so that - the loop `DataSet` doesn't hold any data itself, it only passes that data - on to the `DataServer` - """ - default_storage_period = 1 # seconds between data storage calls - queries_per_store = 5 - default_monitor_period = 60 # seconds between monitoring storage calls - - def __init__(self, query_queue, response_queue, extras=None): - super().__init__(query_queue, response_queue, extras) - - self._storage_period = self.default_storage_period - self._monitor_period = self.default_monitor_period - - self._data = NoData() - self._measuring = False - - self.run_event_loop() - - def run_event_loop(self): - self.running = True - next_store_ts = datetime.now() - next_monitor_ts = datetime.now() - - while self.running: - read_timeout = self._storage_period / self.queries_per_store - try: - query = self._query_queue.get(timeout=read_timeout) - self.process_query(query) - except Empty: - pass - - try: - now = datetime.now() - - if self._measuring and now > next_store_ts: - td = timedelta(seconds=self._storage_period) - next_store_ts = now + td - self._data.write() - - if now > next_monitor_ts: - td = timedelta(seconds=self._monitor_period) - next_monitor_ts = now + td - # TODO: update the monitor data storage - - except: - logging.error(format_exc()) - - ###################################################################### - # query handlers # - ###################################################################### - - def handle_new_data(self, data_set): - """ - Load a new (normally empty) DataSet into the DataServer, and - prepare it to start receiving and storing data - """ - if self._measuring: - raise RuntimeError('Already executing a measurement') - - self._data = data_set - self._data.init_on_server() - self._measuring = True - - def handle_finalize_data(self): - """ - Mark this DataSet as complete and write its final changes to storage - """ - self._data.finalize() - self._measuring = False - - def handle_store_data(self, *args): - """ - Put some data into the DataSet - """ - self._data.store(*args) - - def handle_get_measuring(self): - """ - Is a measurement loop presently running? - """ - return self._measuring - - def handle_get_data(self, attr=None): - """ - Return the active DataSet or some attribute of it - """ - return getattr(self._data, attr) if attr else self._data - - def handle_get_changes(self, synced_indices): - """ - Return all new data after the last sync - """ - return self._data.get_changes(synced_indices) diff --git a/qcodes/instrument/base.py b/qcodes/instrument/base.py index c131f7b9663..ff1ef079413 100644 --- a/qcodes/instrument/base.py +++ b/qcodes/instrument/base.py @@ -1,21 +1,17 @@ """Instrument base class.""" import logging import time -import warnings import weakref import numpy as np from qcodes.utils.metadata import Metadatable from qcodes.utils.helpers import DelegateAttributes, strip_attrs, full_class -from qcodes.utils.nested_attrs import NestedAttrAccess from qcodes.utils.validators import Anything from .parameter import StandardParameter from .function import Function -from .metaclass import InstrumentMetaclass -class Instrument(Metadatable, DelegateAttributes, NestedAttrAccess, - metaclass=InstrumentMetaclass): +class Instrument(Metadatable, DelegateAttributes): """ Base class for all QCodes instruments. @@ -137,21 +133,6 @@ def get_idn(self): return dict(zip(('vendor', 'model', 'serial', 'firmware'), idparts)) - @classmethod - def default_server_name(cls, **kwargs): - """ - Generate a default name for the server to host this instrument. - - Args: - **kwargs: the constructor kwargs, used if necessary to choose a - name. - - Returns: - str: The default server name for the specific instrument instance - we are constructing. - """ - return 'Instruments' - def connect_message(self, idn_param='IDN', begin_time=None): """ Print a standard message on initial connection to an instrument. diff --git a/qcodes/instrument/mock.py b/qcodes/instrument/mock.py index ec100134e6f..c53cf3e93a0 100644 --- a/qcodes/instrument/mock.py +++ b/qcodes/instrument/mock.py @@ -1,288 +1,10 @@ """Mock instruments for testing purposes.""" -import time -from datetime import datetime -from .base import Instrument from .parameter import MultiParameter from qcodes import Loop from qcodes.data.data_array import DataArray -from qcodes.process.server import ServerManager, BaseServer -from qcodes.utils.nested_attrs import _NoDefault -class MockInstrument(Instrument): - - """ - Create a software instrument, mostly for testing purposes. - - Also works for simulations, but usually this will be simpler, easier to - use, and faster if made as a single ``Instrument`` subclass. - - ``MockInstrument``\s have extra overhead as they serialize all commands - (to mimic a network communication channel) and use at least two processes - (instrument server and model server) both of which must be involved in any - given query. - - parameters to pass to model should be declared with: - - - get_cmd = param_name + '?' - - set_cmd = param_name + ':{:.3f}' (specify the format & precision) - - alternatively independent set/get functions may still be provided. - - Args: - name (str): The name of this instrument. - - delay (number): Time (in seconds) to wait after any operation - to simulate communication delay. Default 0. - - model (MockModel): A model to connect to. Subclasses MUST accept - ``model`` as a constructor kwarg ONLY, even though it is required. - See notes in ``Instrument`` docstring. - The model should have one or two methods related directly to this - instrument by ``name``: - ``_set(param, value)``: set a parameter on the model - ``_get(param)``: returns the value of a parameter - - keep_history (bool): Whether to record (in self.history) every command - sent to this instrument. Default True. - - server_name (Union[str, None]): leave default ('') to make a - MockInsts-####### server with the number matching the model server - id, or set None to not use a server. - - Attributes: - shared_kwargs (List[str]): Class attribute, constructor kwargs to - provide via server init. For MockInstrument this should always be - ['model'] at least. - - keep_history (bool): Whether to record all commands and responses. Set - on init, but may be changed at any time. - - history (List[tuple]): All commands and responses while keep_history is - enabled, as tuples: - (timestamp, 'ask' or 'write', param_name[, value]) - """ - - shared_kwargs = ['model'] - - def __init__(self, name, delay=0, model=None, keep_history=True, **kwargs): - super().__init__(name, **kwargs) - - if not isinstance(delay, (int, float)) or delay < 0: - raise TypeError('delay must be a non-negative number') - self._delay = delay - - # try to access write and ask so we know they exist - model.write - model.ask - self._model = model - - # keep a record of every command sent to this instrument - # for debugging purposes? - self.keep_history = bool(keep_history) - self.history = [] - - @classmethod - def default_server_name(cls, **kwargs): - """ - Get the default server name for this instrument. - - Args: - **kwargs: All the kwargs supplied in the constructor. - - Returns: - str: Default MockInstrument server name is MockInsts-#######, where - ####### is the first 7 characters of the MockModel's uuid. - """ - model = kwargs.get('model', None) - if model: - return model.name.replace('Model', 'MockInsts') - return 'MockInstruments' - - def get_idn(self): - """Shim for IDN parameter.""" - return { - 'vendor': None, - 'model': type(self).__name__, - 'serial': self.name, - 'firmware': None - } - - def write_raw(self, cmd): - """ - Low-level interface to ``model.write``. - - Prepends self.name + ':' to the command, so the ``MockModel`` - will direct this query to its ``_set`` method - - Args: - cmd (str): The command to send to the instrument. - """ - if self._delay: - time.sleep(self._delay) - - try: - parameter, value = cmd.split(':', 1) - except ValueError: - parameter, value = cmd, None # for functions with no value - - if self.keep_history: - self.history.append((datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'write', parameter, value)) - - self._model.write('cmd', self.name + ':' + cmd) - - def ask_raw(self, cmd): - """ - Low-level interface to ``model.ask``. - - Prepends self.name + ':' to the command, so the ``MockModel`` - will direct this query to its ``_get`` method - - Args: - cmd (str): The command to send to the instrument. - - Returns: - str: The instrument's response. - - Raises: - ValueError: If ``cmd`` is malformed in that it contains text - after the '?' - """ - if self._delay: - time.sleep(self._delay) - - parameter, blank = cmd.split('?') - if blank: - raise ValueError('text found after end of query') - - if self.keep_history: - self.history.append((datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'ask', parameter)) - - return self._model.ask('cmd', self.name + ':' + cmd) - - -# MockModel is purely in service of mock instruments which *are* tested -# so coverage testing this (by running it locally) would be a waste. -class MockModel(ServerManager, BaseServer): # pragma: no cover - - """ - Base class for models to connect to various MockInstruments. - - Creates a separate process that holds the model state, so that - any process can interact with the model and get the same state. - - Args: - name (str): The server name to create for the model. - Default 'Model-{:.7s}' uses the first 7 characters of - the server's uuid. - - for every instrument that connects to this model, create two methods: - - ``_set(param, value)``: set a parameter on the model - - ``_get(param)``: returns the value of a parameter - ``param`` and the set/return values should all be strings - - If ``param`` and/or ``value`` is not recognized, the method should raise - an error. - - Other uses of ServerManager use separate classes for the server and its - manager, but here I put the two together into a single class, to make it - easier to define models. The downside is you have a local object with - methods you shouldn't call: the extras (_(set|get)) should - only be called on the server copy. Normally this should only be called via - the attached instruments anyway. - - The model supports ``NestedAttrAccess`` calls ``getattr``, ``setattr``, - ``callattr``, and ``delattr`` Because the manager and server are the same - object, we override these methods with proxy methods after the server has - been started. - """ - - def __init__(self, name='Model-{:.7s}'): - super().__init__(name, server_class=None) - - # now that the server has started, we can remap attribute access - # from the private methods (_getattr) to the public ones (getattr) - # but the server copy will still have the NestedAttrAccess ones - self.getattr = self._getattr - self.setattr = self._setattr - self.callattr = self._callattr - self.delattr = self._delattr - - def _run_server(self): - self.run_event_loop() - - def handle_cmd(self, cmd): - """ - Handler for all model queries. - - Args: - cmd (str): Can take several forms: - - - ':?': - calls ``self._get()`` and forwards - the return value. - - '::': - calls ``self._set(, )`` - - ':'. - calls ``self._set(, None)`` - - Returns: - Union(str, None): The parameter value, if ``cmd`` has the form - ':?', otherwise no return. - - Raises: - ValueError: if cmd does not match one of the patterns above. - """ - query = cmd.split(':') - - instrument = query[0] - param = query[1] - - if param[-1] == '?' and len(query) == 2: - return getattr(self, instrument + '_get')(param[:-1]) - - elif len(query) <= 3: - value = query[2] if len(query) == 3 else None - getattr(self, instrument + '_set')(param, value) - - else: - raise ValueError() - - def _getattr(self, attr, default=_NoDefault): - """ - Get a (possibly nested) attribute of this model on its server. - - See NestedAttrAccess for details. - """ - return self.ask('method_call', 'getattr', attr, default) - - def _setattr(self, attr, value): - """ - Set a (possibly nested) attribute of this model on its server. - - See NestedAttrAccess for details. - """ - self.ask('method_call', 'setattr', attr, value) - - def _callattr(self, attr, *args, **kwargs): - """ - Call a (possibly nested) method of this model on its server. - - See NestedAttrAccess for details. - """ - return self.ask('method_call', 'callattr', attr, *args, **kwargs) - - def _delattr(self, attr): - """ - Delete a (possibly nested) attribute of this model on its server. - - See NestedAttrAccess for details. - """ - self.ask('method_call', 'delattr', attr) - class ArrayGetter(MultiParameter): """ Example parameter that just returns a single array @@ -313,4 +35,4 @@ def get(self): loop = Loop(self.sweep_values, self.delay).each(self.measured_param) data = loop.run_temp() array = data.arrays[self.measured_param.full_name] - return (array,) \ No newline at end of file + return (array,) diff --git a/qcodes/instrument/remote.py b/qcodes/instrument/remote.py deleted file mode 100644 index 37401e80b23..00000000000 --- a/qcodes/instrument/remote.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Proxies to interact with server-based instruments from another process.""" -import multiprocessing as mp - -from qcodes.utils.deferred_operations import DeferredOperations -from qcodes.utils.helpers import DelegateAttributes, named_repr -from .parameter import Parameter, GetLatest -from .server import get_instrument_server_manager - - -class RemoteInstrument(DelegateAttributes): - - """ - A proxy for an instrument (of any class) running on a server process. - - Creates the server if necessary, then loads this instrument onto it, - then mirrors the API to that instrument. - - Args: - *args: Passed along to the real instrument constructor. - - instrument_class (type): The class of the real instrument to make. - - server_name (str): The name of the server to create or use for this - instrument. If not provided (''), gets a name from - ``instrument_class.default_server_name(**kwargs)`` using the - same kwargs passed to the instrument constructor. - - **kwargs: Passed along to the real instrument constructor, also - to ``default_server_name`` as mentioned. - - Attributes: - name (str): an identifier for this instrument, particularly for - attaching it to a Station. - - parameters (Dict[Parameter]): All the parameters supported by this - instrument. Usually populated via ``add_parameter`` - - functions (Dict[Function]): All the functions supported by this - instrument. Usually populated via ``add_function`` - """ - - delegate_attr_dicts = ['_methods', 'parameters', 'functions'] - - def __init__(self, *args, instrument_class=None, server_name='', - **kwargs): - if server_name == '': - server_name = instrument_class.default_server_name(**kwargs) - - shared_kwargs = {} - for kwname in instrument_class.shared_kwargs: - if kwname in kwargs: - shared_kwargs[kwname] = kwargs[kwname] - del kwargs[kwname] - - self._server_name = server_name - self._shared_kwargs = shared_kwargs - self._manager = get_instrument_server_manager(self._server_name, - self._shared_kwargs) - - self._instrument_class = instrument_class - self._args = args - self._kwargs = kwargs - - self.connect() - - def connect(self): - """Create the instrument on the server and replicate its API here.""" - - # connection_attrs is created by instrument.connection_attrs(), - # called by InstrumentServer.handle_new after it creates the instrument - # on the server. - connection_attrs = self._manager.connect(self, self._instrument_class, - self._args, self._kwargs) - - self.name = connection_attrs['name'] - self._id = connection_attrs['id'] - self._methods = {} - self.parameters = {} - self.functions = {} - - # bind all the different categories of actions we need - # to interface with the remote instrument - - self._update_components(connection_attrs) - - def _update_components(self, connection_attrs): - """ - Update the three component dicts with new or updated connection attrs. - - Args: - connection_attrs (dict): as returned by - ``Instrument.connection_attrs``, should contain at least keys - ``_methods``, ``parameters``, and ``functions``, whose values - are themselves dicts of {component_name: list of attributes}. - These get translated into the corresponding dicts eg: - ``self.parameters = {parameter_name: RemoteParameter}`` - """ - component_types = (('_methods', RemoteMethod), - ('parameters', RemoteParameter), - ('functions', RemoteFunction)) - - for container_name, component_class in component_types: - container = getattr(self, container_name) - components_spec = connection_attrs[container_name] - - # first delete components that are gone and update those that - # have changed - for name in list(container.keys()): - if name in components_spec: - container[name].update(components_spec[name]) - else: - del container[name] - - # then add new components - for name, attrs in components_spec.items(): - if name not in container: - container[name] = component_class(name, self, attrs) - - def update(self): - """Check with the server for updated components.""" - connection_attrs = self._ask_server('connection_attrs', self._id) - self._update_components(connection_attrs) - - def _ask_server(self, func_name, *args, **kwargs): - """Query the server copy of this instrument, expecting a response.""" - return self._manager.ask('cmd', self._id, func_name, *args, **kwargs) - - def _write_server(self, func_name, *args, **kwargs): - """Send a command to the server, without waiting for a response.""" - self._manager.write('cmd', self._id, func_name, *args, **kwargs) - - def add_parameter(self, name, **kwargs): - """ - Proxy to add a new parameter to the server instrument. - - This is only for adding parameters remotely to the server copy. - Normally parameters are added in the instrument constructor, rather - than via this method. This method is limited in that you can generally - only use the string form of a command, not the callable form. - - Args: - name (str): How the parameter will be stored within - ``instrument.parameters`` and also how you address it using the - shortcut methods: ``instrument.set(param_name, value)`` etc. - - parameter_class (Optional[type]): You can construct the parameter - out of any class. Default ``StandardParameter``. - - **kwargs: constructor arguments for ``parameter_class``. - """ - attrs = self._ask_server('add_parameter', name, **kwargs) - self.parameters[name] = RemoteParameter(name, self, attrs) - - def add_function(self, name, **kwargs): - """ - Proxy to add a new Function to the server instrument. - - This is only for adding functions remotely to the server copy. - Normally functions are added in the instrument constructor, rather - than via this method. This method is limited in that you can generally - only use the string form of a command, not the callable form. - - Args: - name (str): how the function will be stored within - ``instrument.functions`` and also how you address it using the - shortcut methods: ``instrument.call(func_name, *args)`` etc. - - **kwargs: constructor kwargs for ``Function`` - """ - attrs = self._ask_server('add_function', name, **kwargs) - self.functions[name] = RemoteFunction(name, self, attrs) - - def instances(self): - """ - A RemoteInstrument shows as an instance of its proxied class. - - Returns: - List[Union[Instrument, RemoteInstrument]] - """ - return self._instrument_class.instances() - - def find_instrument(self, name, instrument_class=None): - """ - Find an existing instrument by name. - - Args: - name (str) - - Returns: - Union[Instrument, RemoteInstrument] - - Raises: - KeyError: if no instrument of that name was found, or if its - reference is invalid (dead). - """ - return self._instrument_class.find_instrument( - name, instrument_class=instrument_class) - - def close(self): - """Irreversibly close and tear down the server & remote instruments.""" - if hasattr(self, '_manager'): - if self._manager._server in mp.active_children(): - self._manager.delete(self._id) - del self._manager - self._instrument_class.remove_instance(self) - - def restart(self): - """ - Remove and recreate the server copy of this instrument. - - All instrument state will be returned to the initial conditions, - including deleting any parameters you've added after initialization, - or modifications to parameters etc. - """ - self._manager.delete(self._id) - self.connect() - - def __getitem__(self, key): - """Delegate instrument['name'] to parameter or function 'name'.""" - try: - return self.parameters[key] - except KeyError: - return self.functions[key] - - def __repr__(self): - """repr including the instrument name.""" - return named_repr(self) - - -class RemoteComponent: - - """ - An object that lives inside a RemoteInstrument. - - Proxies all of its calls and specific listed attributes to the - corresponding object in the server instrument. - - Args: - name (str): The name of this component. - - instrument (RemoteInstrument): the instrument this is part of. - - attrs (List[str]): instance attributes to proxy to the server - copy of this component. - - Attributes: - name (str): The name of this component. - - _instrument (RemoteInstrument): the instrument this is part of. - - _attrs (Set[str]): All the attributes we are allowed to proxy. - - _delattrs (Set[str]): Attributes we've deleted from the server, - a subset of ``_attrs``, but if you set them again, they will - still be set on the server. - - _local_attrs (Set[str]): (class attribute only) Attributes that we - shouldn't look for on the server, even if they do not exist - locally. Mostly present to prevent infinite recursion in the - accessors. - """ - _local_attrs = { - '_attrs', - 'name', - '_instrument', - '_local_attrs', - '__doc__', - '_delattrs' - } - - def __init__(self, name, instrument, attrs): - self.name = name - self._instrument = instrument - self.update(attrs) - - def update(self, attrs): - """ - Update the set of attributes proxied by this component. - - The docstring is not proxied every time it is accessed, but it is - read and updated during this method. - - Args: - attrs (Sequence[str]): the new set of attributes to proxy. - """ - self._attrs = set(attrs) - self._delattrs = set() - self._set_doc() - - def __getattr__(self, attr): - """ - Get an attribute value from the server. - - If there was a local attribute, we don't even get here. - """ - if attr not in type(self)._local_attrs and attr in self._attrs: - full_attr = self.name + '.' + attr - return self._instrument._ask_server('getattr', full_attr) - else: - raise AttributeError('RemoteComponent has no local or remote ' - 'attribute: ' + attr) - - def __setattr__(self, attr, val): - """ - Set a new attribute value. - - If the attribute is listed as remote, we'll set it on the server, - otherwise we'll set it locally. - """ - if attr not in type(self)._local_attrs and attr in self._attrs: - full_attr = self.name + '.' + attr - self._instrument._ask_server('setattr', full_attr, val) - if attr in self._delattrs: - self._delattrs.remove(attr) - else: - object.__setattr__(self, attr, val) - - def __delattr__(self, attr): - """ - Delete an attribute. - - If the attribute is listed as remote, we'll delete it on the server, - otherwise we'll delete it locally. - """ - - if attr not in type(self)._local_attrs and attr in self._attrs: - full_attr = self.name + '.' + attr - self._instrument._ask_server('delattr', full_attr) - self._delattrs.add(attr) - - else: - object.__delattr__(self, attr) - - def __dir__(self): - """dir listing including both local and server attributes.""" - remote_attrs = self._attrs - self._delattrs - return sorted(remote_attrs.union(super().__dir__())) - - def _set_doc(self): - """ - Prepend a note about remoteness to the server docstring. - - If no server docstring is found, we leave the class docstring. - - __doc__, as a magic attribute, is handled differently from - other attributes so we won't make it dynamic (updating on the - server when you change it here) - """ - doc = self._instrument._ask_server('getattr', - self.name + '.__doc__') - - docbase = '{} {} in RemoteInstrument {}'.format( - type(self).__name__, self.name, self._instrument.name) - - self.__doc__ = docbase + (('\n---\n\n' + doc) if doc else '') - - def __repr__(self): - """repr including the component name.""" - return named_repr(self) - - -class RemoteMethod(RemoteComponent): - - """Proxy for a method of the server instrument.""" - - def __call__(self, *args, **kwargs): - """Call the method on the server, passing on any args and kwargs.""" - return self._instrument._ask_server(self.name, *args, **kwargs) - - -class RemoteParameter(RemoteComponent, DeferredOperations): - - """Proxy for a Parameter of the server instrument.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.get_latest = GetLatest(self) - - def __call__(self, *args): - """ - Shortcut to get (with no args) or set (with one arg) the parameter. - - Args: - *args: If empty, get the parameter. If one arg, set the parameter - to this value - - Returns: - any: The parameter value, if called with no args, - otherwise no return. - """ - if len(args) == 0: - return self.get() - else: - self.set(*args) - - def get(self): - """ - Read the value of this parameter. - - Returns: - any: the current value of the parameter. - """ - return self._instrument._ask_server('get', self.name) - - def set(self, value): - """ - Set a new value of this parameter. - - Args: - value (any): the new value for the parameter. - """ - # TODO: sometimes we want set to block (as here) and sometimes - # we want it async... which would just be changing the '_ask_server' - # to '_write_server' below. how do we decide, and how do we let the - # user do it? - self._instrument._ask_server('set', self.name, value) - - def validate(self, value): - """ - Raise an error if the given value is not allowed for this Parameter. - - Args: - value (any): the proposed new parameter value. - - Raises: - TypeError: if ``value`` has the wrong type for this Parameter. - ValueError: if the type is correct but the value is wrong. - """ - self._instrument._ask_server('callattr', - self.name + '.validate', value) - - # manually copy over sweep and __getitem__ so they execute locally - # and are still based off the RemoteParameter - def __getitem__(self, keys): - """Create a SweepValues from this parameter with slice notation.""" - return Parameter.__getitem__(self, keys) - - def sweep(self, *args, **kwargs): - """Create a SweepValues from this parameter. See Parameter.sweep.""" - return Parameter.sweep(self, *args, **kwargs) - - def _latest(self): - return self._instrument._ask_server('callattr', self.name + '._latest') - - def snapshot(self, update=False): - """ - State of the parameter as a JSON-compatible dict. - - Args: - update (bool): If True, update the state by querying the - instrument. If False, just use the latest value in memory. - - Returns: - dict: snapshot - """ - return self._instrument._ask_server('callattr', - self.name + '.snapshot', update) - - def setattr(self, attr, value): - """ - Set an attribute of the parameter on the server. - - Args: - attr (str): the attribute name. Can be nested as in - ``NestedAttrAccess``. - value: The new value to set. - """ - self._instrument._ask_server('setattr', self.name + '.' + attr, value) - - def getattr(self, attr): - """ - Get an attribute of the parameter on the server. - - Args: - attr (str): the attribute name. Can be nested as in - ``NestedAttrAccess``. - - Returns: - any: The attribute value. - """ - return self._instrument._ask_server('getattr', self.name + '.' + attr) - - def callattr(self, attr, *args, **kwargs): - """ - Call arbitrary methods of the parameter on the server. - - Args: - attr (str): the method name. Can be nested as in - ``NestedAttrAccess``. - *args: positional args to the method - **kwargs: keyword args to the method - - Returns: - any: the return value of the called method. - """ - return self._instrument._ask_server( - 'callattr', self.name + '.' + attr, *args, **kwargs) - - -class RemoteFunction(RemoteComponent): - - """Proxy for a Function of the server instrument.""" - - def __call__(self, *args): - """ - Call the Function. - - Args: - *args: The positional args to this Function. Functions only take - positional args, not kwargs. - - Returns: - any: the return value of the function. - """ - return self._instrument._ask_server('call', self.name, *args) - - def call(self, *args): - """An alias for __call__.""" - return self.__call__(*args) - - def validate(self, *args): - """ - Raise an error if the given args are not allowed for this Function. - - Args: - *args: the proposed arguments with which to call the Function. - """ - return self._instrument._ask_server( - 'callattr', self.name + '.validate', *args) diff --git a/qcodes/instrument/server.py b/qcodes/instrument/server.py deleted file mode 100644 index 0772ff68783..00000000000 --- a/qcodes/instrument/server.py +++ /dev/null @@ -1,190 +0,0 @@ -import multiprocessing as mp - -from qcodes.process.server import ServerManager, BaseServer - - -def get_instrument_server_manager(server_name, shared_kwargs={}): - """ - Find or make a given `InstrumentServerManager`. - - An `InstrumentServer` holds one or more Instrument objects, and an - `InstrumentServerManager` allows other processes to communicate with this - `InstrumentServer`. - - Both the name and the shared attributes must match exactly. If no manager - exists with this name, it will be created with the given `shared_kwargs`. - If an manager exists with this name but different `shared_kwargs` we - raise an error. - - server_name: (default 'Instruments') which server to put the instrument on. - If a server with this name exists, the instrument will be added to it. - If not, a new server is created with this name. - - shared_kwargs: unpicklable items needed by the instruments on the - server, will get sent with the manager when it's started up - and included in the kwargs to construct each new instrument - """ - if not server_name: - server_name = 'Instruments' - - instances = InstrumentServerManager.instances - manager = instances.get(server_name, None) - - if manager and manager._server in mp.active_children(): - if shared_kwargs and manager.shared_kwargs != shared_kwargs: - # it's OK to add another instrument that has *no* shared_kwargs - # but if there are some and they're different from what's - # already associated with this server, that's an error. - raise ValueError(('An InstrumentServer with name "{}" already ' - 'exists but with different shared_attrs' - ).format(server_name)) - else: - manager = InstrumentServerManager(server_name, shared_kwargs) - - return manager - - -class InstrumentServerManager(ServerManager): - """ - Creates and manages connections to an InstrumentServer - - Args: - name: the name of the server to create - kwargs: extra items to send to the server on creation (such as - additional queues, that can only be shared on creation) - These items will be set as attributes of any instrument that - connects to the server - """ - instances = {} - - def __init__(self, name, shared_kwargs=None): - self.name = name - self.shared_kwargs = shared_kwargs - self.instances[name] = self - - self.instruments = {} - - super().__init__(name=name, server_class=InstrumentServer, - shared_attrs=shared_kwargs) - - def restart(self): - """ - Restart the InstrumentServer and reconnect the instruments that - had been connected previously - """ - super().restart() - - instruments = self.instruments.values() - self.instruments = {} - for instrument in instruments: - instrument.connect() - - def connect(self, remote_instrument, instrument_class, args, kwargs): - new_id = self.ask('new_id') - try: - info = self.ask('new', instrument_class, new_id, *args, **kwargs) - self.instruments[new_id] = remote_instrument - - except: - # if anything went wrong adding a new instrument, delete it - # in case it still exists there half-formed. - self.delete(new_id) - raise - - return info - - def delete(self, instrument_id): - self.write('delete', instrument_id) - - if self.instruments.get(instrument_id, None): - del self.instruments[instrument_id] - - if not self.instruments: - self.close() - self.instances.pop(self.name, None) - - -class InstrumentServer(BaseServer): - # just for testing - how long to allow it to wait on a queue.get - timeout = None - - def __init__(self, query_queue, response_queue, shared_kwargs): - super().__init__(query_queue, response_queue, shared_kwargs) - - self.instruments = {} - self.next_id = 0 - - # Ensure no references of instruments defined in the main process - # are copied to the server process. With the spawn multiprocessing - # method this is not an issue, as the class is reimported in the - # new process, but with fork it can be a problem ironically. - from qcodes.instrument.base import Instrument - Instrument._all_instruments = {} - - self.run_event_loop() - - def handle_new_id(self): - """ - split out id generation from adding an instrument - so that we can delete it if something goes wrong! - """ - new_id = self.next_id - self.next_id += 1 - return new_id - - def handle_new(self, instrument_class, new_id, *args, **kwargs): - """ - Add a new instrument to the server. - - After the initial load, the instrument is referred to by its ID. - - Args: - instrument_class (class): The type of instrument to construct. - - new_id (int): The ID by which this instrument will be known on the - server. - - *args: positional arguments to the instrument constructor. - - **kwargs: keyword arguments to the instrument constructor. - - Returns: - dict: info to reconstruct this instrument's API in the remote. - See ``Instrument.connection_attrs`` for details. - """ - - # merge shared_kwargs into kwargs for the constructor, - # but only if this instrument_class is expecting them. - # The *first* instrument put on a given server must have - # all the shared_kwargs sent with it, but others may skip - # (for now others must have *none* but later maybe they could - # just skip some of them) - for key, value in self._shared_attrs.items(): - if key in instrument_class.shared_kwargs: - kwargs[key] = value - ins = instrument_class(*args, server_name=None, **kwargs) - - self.instruments[new_id] = ins - - # info to reconstruct the instrument API in the RemoteInstrument - return ins.connection_attrs(new_id) - - def handle_delete(self, instrument_id): - """ - Delete an instrument from the server, and stop the server if their - are no more instruments left after this. - """ - if instrument_id in self.instruments: - self.instruments[instrument_id].close() - - del self.instruments[instrument_id] - - if not any(self.instruments): - self.handle_halt() - - def handle_cmd(self, instrument_id, func_name, *args, **kwargs): - """ - Run some method of an instrument - """ - func = getattr(self.instruments[instrument_id], func_name) - return func(*args, **kwargs) diff --git a/qcodes/instrument/visa.py b/qcodes/instrument/visa.py index d020a65ac2b..8260c650fad 100644 --- a/qcodes/instrument/visa.py +++ b/qcodes/instrument/visa.py @@ -1,6 +1,5 @@ """Visa instrument driver based on pyvisa.""" import visa -import logging from .base import Instrument import qcodes.utils.validators as vals @@ -57,26 +56,6 @@ def __init__(self, name, address=None, timeout=5, terminator='', **kwargs): self.set_terminator(terminator) self.timeout.set(timeout) - @classmethod - def default_server_name(cls, **kwargs): - """ - Get the default server name for this instrument. - - Args: - **kwargs: All the kwargs supplied in the constructor. - - Returns: - str: The default server name, either 'GPIBServer', 'SerialServer', - or 'VisaServer' depending on ``kwargs['address']``. - """ - upper_address = kwargs.get('address', '').upper() - if 'GPIB' in upper_address: - return 'GPIBServer' - elif 'ASRL' in upper_address: - return 'SerialServer' - - return 'VisaServer' - def set_address(self, address): """ Change the address for this instrument. diff --git a/qcodes/process/helpers.py b/qcodes/process/helpers.py deleted file mode 100644 index 195eb2928fb..00000000000 --- a/qcodes/process/helpers.py +++ /dev/null @@ -1,63 +0,0 @@ -"""multiprocessing helper functions.""" - -import multiprocessing as mp -import time -import warnings - -MP_ERR = 'context has already been set' - - -def set_mp_method(method, force=False): - """ - An idempotent wrapper for multiprocessing.set_start_method. - - The most important use of this is to force Windows behavior - on a Mac or Linux: set_mp_method('spawn') - args are the same: - - Args: - method (string): one of the following - - - 'fork' (default on unix/mac) - - 'spawn' (default, and only option, on windows) - - 'forkserver' - - force (bool): allow changing context? default False - in the original function, even calling the function again - with the *same* method raises an error, but here we only - raise the error if you *don't* force *and* the context changes - """ - warnings.warn("Multiprocessing is in beta, use at own risk", UserWarning) - try: - mp.set_start_method(method, force=force) - except RuntimeError as err: - if err.args != (MP_ERR, ): - raise - - mp_method = mp.get_start_method() - if mp_method != method: - raise RuntimeError( - 'unexpected multiprocessing method ' - '\'{}\' when trying to set \'{}\''.format(mp_method, method)) - - -def kill_queue(queue): - """Tear down a multiprocessing.Queue to help garbage collection.""" - try: - queue.close() - queue.join_thread() - except: - pass - - -def kill_processes(): - """Kill all running child processes.""" - # TODO: Instrument processes don't appropriately stop in all tests... - for process in mp.active_children(): - try: - process.terminate() - except: - pass - - if mp.active_children(): - time.sleep(0.2) diff --git a/qcodes/process/qcodes_process.py b/qcodes/process/qcodes_process.py deleted file mode 100644 index 7d80bea1d07..00000000000 --- a/qcodes/process/qcodes_process.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Modifications to multiprocessing.Process common to all Qcodes processes.""" - -import multiprocessing as mp -from traceback import print_exc -import signal - -from qcodes.utils.helpers import in_notebook - -from .stream_queue import get_stream_queue - - -class QcodesProcess(mp.Process): - - """ - Modified multiprocessing.Process specialized to Qcodes needs. - - - Nicer repr - - Automatic streaming of stdout and stderr to our StreamQueue singleton - for reporting back to the main process - - Ignore interrupt signals so that commands in the main process can be - canceled without affecting server and background processes. - """ - - def __init__(self, *args, name='QcodesProcess', queue_streams=True, - daemon=True, **kwargs): - """ - Construct the QcodesProcess, but like Process, do not start it. - - name: string to include in repr, and in the StreamQueue - default 'QcodesProcess' - queue_streams: should we connect stdout and stderr to the StreamQueue? - default True - daemon: should this process be treated as daemonic, so it gets - terminated with the parent. - default True, overriding the base inheritance - any other args and kwargs are passed to multiprocessing.Process - """ - # make sure the singleton StreamQueue exists - # prior to launching a new process - if queue_streams and in_notebook(): - self.stream_queue = get_stream_queue() - else: - self.stream_queue = None - super().__init__(*args, name=name, daemon=daemon, **kwargs) - - def run(self): - """Executed in the new process, and calls the target function.""" - # ignore interrupt signals, as they come from `KeyboardInterrupt` - # which we want only to apply to the main process and not the - # server and background processes (which can be halted in different - # ways) - signal.signal(signal.SIGINT, signal.SIG_IGN) - - if self.stream_queue: - self.stream_queue.connect(str(self.name)) - try: - super().run() - except: - # if we let the system print the exception by itself, sometimes - # it disconnects the stream partway through printing. - print_exc() - finally: - if (self.stream_queue and - self.stream_queue.initial_streams is not None): - self.stream_queue.disconnect() - - def __repr__(self): - """Shorter and more helpful repr of our processes.""" - cname = self.__class__.__name__ - r = super().__repr__() - r = r.replace(cname + '(', '').replace(')>', '>') - return r.replace(', started daemon', '') diff --git a/qcodes/process/server.py b/qcodes/process/server.py deleted file mode 100644 index d3a103b6f2a..00000000000 --- a/qcodes/process/server.py +++ /dev/null @@ -1,399 +0,0 @@ -"""Common Server process and ServerManager architecture.""" - -import multiprocessing as mp -from traceback import format_exc -from uuid import uuid4 -import builtins -import logging - -QUERY_WRITE = 'WRITE' -QUERY_ASK = 'ASK' -RESPONSE_OK = 'OK' -RESPONSE_ERROR = 'ERROR' - -from qcodes.utils.nested_attrs import NestedAttrAccess -from .qcodes_process import QcodesProcess -from .helpers import kill_queue - - -class ServerManager: - - """ - Creates and communicates with a separate server process. - - Starts a *QcodesProcess*, and on that process it constructs a server - object of type *server_class*, which should normally be a subclass of - `BaseServer`. Client processes query the server via: - - - ``manager.ask(func_name, *args, **kwargs)``: if they want a response or want - to wait for confirmation that the query has completed - - ``manager.write(func_name, *args, **kwargs)``: if they want to continue - immediately without blocking for the query. - - The server communicates with this manager via two multiprocessing *Queue*\s. - """ - - def __init__(self, name, server_class, shared_attrs=None, - query_timeout=None): - """ - Construct the ServerManager and start its server. - - Args: - name: the name of the server. Can include .format specs to insert - all or part of the uuid - server_class: the class to create within the new process. - the constructor will be passed arguments: - query_queue, response_queue, shared_attrs - and should start an infinite loop watching query_queue and posting - responses to response_queue. - shared_attrs: any objects that need to be passed to the server on - startup, generally objects like Queues that are picklable only for - inheritance by a new process. - query_timeout: (default None) the default max time to wait for - responses - """ - self._query_queue = mp.Queue() - self._response_queue = mp.Queue() - self._server_class = server_class - self._shared_attrs = shared_attrs - - # query_lock is only used with queries that get responses - # to make sure the process that asked the question is the one - # that gets the response. - # Any query that does NOT expect a response can just dump it in - # and move on. - self.query_lock = mp.RLock() - - # uuid is used to pass references to this object around - # for example, to get it after someone else has sent it to a server - self.uuid = uuid4().hex - - self.name = name.format(self.uuid) - - self.query_timeout = query_timeout - self._start_server() - - def _start_server(self): - self._server = QcodesProcess(target=self._run_server, name=self.name) - self._server.start() - - def _run_server(self): - self._server_class(self._query_queue, self._response_queue, - self._shared_attrs) - - def _check_alive(self): - try: - if not self._server.is_alive(): - logging.warning('restarted {}'.format(self._server)) - self.restart() - except: - # can't test is_alive from outside the main process - pass - - def write(self, func_name, *args, **kwargs): - """ - Send a query to the server that does not expect a response. - - `write(func_name, *args, **kwargs)` proxies to server method: - `server.handle_(*args, **kwargs)` - """ - self._check_alive() - self._query_queue.put((QUERY_WRITE, func_name, args, kwargs)) - - def ask(self, func_name, *args, timeout=None, **kwargs): - """ - Send a query to the server and wait for a response. - - `resp = ask(func_name, *args, **kwargs)` proxies to server method: - `resp = server.handle_(*args, **kwargs)` - - optional timeout (default None) - not recommended, as if we quit - before reading the response, the query queue can get out of sync - """ - self._check_alive() - - timeout = timeout or self.query_timeout - self._expect_error = False - - query = (QUERY_ASK, func_name, args, kwargs) - - with self.query_lock: - # in case a previous query errored and left something on the - # response queue, clear it - while not self._response_queue.empty(): - value = self._get_response() - logging.warning( - 'unexpected data in response queue before ask:\n' + - repr(value)) - - self._query_queue.put(query) - - value = self._get_response(timeout=timeout, query=query) - - while not self._response_queue.empty(): - logging .warning( - 'unexpected multiple responses in queue during ask, ' - 'using the last one. earlier item(s):\n' + - repr(value)) - value = self._get_response(query=query) - - return value - - def _get_response(self, timeout=None, query=None): - res = self._response_queue.get(timeout=timeout) - try: - code, value = res - except (TypeError, ValueError): - code, value = '', res - - if code == RESPONSE_OK: - return value - - self._handle_error(code, value, query) - - def _handle_error(self, code, error_str, query=None): - error_head = '*** error on {} ***'.format(self.name) - - if query: - error_head += '\nwhile executing query: {}'.format(repr(query)) - - if code != RESPONSE_ERROR: - error_head += '\nunrecognized response code: {}'.format(code) - - # try to match the error type, if it's a built-in type - error_type_line = error_str.rstrip().rsplit('\n', 1)[-1] - error_type_str = error_type_line.split(':')[0].strip() - - err_type = getattr(builtins, error_type_str, None) - if err_type is None or not issubclass(err_type, Exception): - err_type = RuntimeError - - raise err_type(error_head + '\n\n' + error_str) - - def halt(self, timeout=2): - """ - Halt the server and end its process. - - Does not tear down, after this the server can still be started again. - """ - try: - if self._server.is_alive(): - self.write('halt') - self._server.join(timeout) - - if self._server.is_alive(): - self._server.terminate() - logging.warning('ServerManager did not respond to halt ' - 'signal, terminated') - self._server.join(timeout) - except AssertionError: - # happens when we get here from other than the main process - # where we shouldn't be able to kill the server anyway - pass - - def restart(self): - """Restart the server.""" - self.halt() - self._start_server() - - def close(self): - """Irreversibly stop the server and manager.""" - self.halt() - for q in ['query', 'response', 'error']: - qname = '_{}_queue'.format(q) - if hasattr(self, qname): - kill_queue(getattr(self, qname)) - del self.__dict__[qname] - if hasattr(self, 'query_lock'): - del self.query_lock - - -class BaseServer(NestedAttrAccess): - - """ - Base class for servers to run in separate processes. - - The server is started inside a `QcodesProcess` by a `ServerManager`, - and unifies the query handling protocol so that we are robust against - deadlocks, out of sync queues, or hidden errors. - - This base class doesn't start the event loop, a subclass should - either call `self.run_event_loop()` at the end of its `__init__` or - provide its own event loop. If making your own event loop, be sure to - call `self.process_query(query)` on any item that arrives in - `self._query_queue`. - - Subclasses should define handlers `handle_`, such that calls: - `response = server_manager.ask(func_name, *args, **kwargs)` - `server_manager.write(func_name, *args, **kwargs)` - map onto method calls: - `response = self.handle_(*args, **kwargs)` - - The actual query passed through the queue and unpacked by `process_query` - has the form `(code, func_name[, args][, kwargs])` where `code` is: - - - `QUERY_ASK` (from `server_manager.ask`): will always send a response, - even if the function returns nothing (None) or throws an error. - - - `QUERY_WRITE` (from `server_manager.write`): will NEVER send a response, - return values are ignored and errors go to the logging framework. - - Three handlers are predefined: - - - `handle_halt` (but override it if your event loop does not use - self.running=False to stop) - - - `handle_get_handlers` (lists all available handler methods) - - - `handle_method_call` (call an arbitrary method on the server) - """ - - # just for testing - how long to allow it to wait on a queue.get - # in real situations this should always be None - timeout = None - - def __init__(self, query_queue, response_queue, shared_attrs=None): - """ - Create the BaseServer. - - Subclasses should match this call signature exactly, even if they - do not need shared_attrs, because it is used by `ServerManager` - to instantiate the server. - The base class does not start the event loop, subclasses should do - this at the end of their own `__init__`. - - query_queue: a multiprocessing.Queue that we listen to - - response_queue: a multiprocessing.Queue where we put responses - - shared_attrs: (default None) any objects (such as other Queues) - that we need to supply on initialization of the server because - they cannot be picked normally to pass through the Queue later. - """ - self._query_queue = query_queue - self._response_queue = response_queue - self._shared_attrs = shared_attrs - - def run_event_loop(self): - """ - The default event loop. When this method returns, the server stops. - - Override this method if you need to do more than just process queries - repeatedly, but make sure your event loop: - - - calls `self.process_query` to ensure robust error handling - - provides a way to halt the server (and override `handle_halt` if - it's not by setting `self.running = False`) - """ - self.running = True - while self.running: - query = self._query_queue.get(timeout=self.timeout) - self.process_query(query) - - def process_query(self, query): - """ - Act on one query received through the query queue. - - query: should have the form `(code, func_name[, args][, kwargs])` - """ - try: - code = None - code, func_name = query[:2] - - func = getattr(self, 'handle_' + func_name) - - args = None - kwargs = None - for part in query[2:]: - if isinstance(part, tuple) and args is None: - args = part - elif isinstance(part, dict) and kwargs is None: - kwargs = part - else: - raise ValueError(part) - - if code == QUERY_ASK: - self._process_ask(func, args or (), kwargs or {}) - elif code == QUERY_WRITE: - self._process_write(func, args or (), kwargs or {}) - else: - raise ValueError(code) - except: - self.report_error(query, code) - - def report_error(self, query, code): - """ - Common error handler for all queries. - - QUERY_ASK puts errors into the response queue for the asker to see. - QUERY_WRITE shouldn't write a response, so it logs errors instead. - Unknown modes do *both*, because we don't know where the user will be - looking and an error that severe it's OK to muck up the queue. - That's the only way you'll get a response without asking for one. - """ - error_str = ( - 'Expected query to be a tuple (code, func_name[, args][, kwargs]) ' - 'where code is QUERY_ASK or QUERY_WRITE, func_name points to a ' - 'method `handle_`, and optionally args is a tuple and ' - 'kwargs is a dict\nquery: ' + repr(query) + '\n' + format_exc()) - - if code != QUERY_ASK: - logging.error(error_str) - if code != QUERY_WRITE: - try: - self._response_queue.put((RESPONSE_ERROR, error_str)) - except: - logging.error('Could not put error on response queue\n' + - error_str) - - def _process_ask(self, func, args, kwargs): - try: - response = func(*args, **kwargs) - self._response_queue.put((RESPONSE_OK, response)) - except: - self._response_queue.put( - (RESPONSE_ERROR, repr((func, args, kwargs)) + '\n' + - format_exc())) - - def _process_write(self, func, args, kwargs): - try: - func(*args, **kwargs) - except: - logging.error(repr((func, args, kwargs)) + '\n' + format_exc()) - - def handle_halt(self): - """ - Quit this server. - - Just sets self.running=False, which the default event loop looks for - between queries. If you provide your own event loop and it does NOT - look for self.running, you should override this handler with a - different way to halt. - """ - self.running = False - - def handle_get_handlers(self): - """List all available query handlers.""" - handlers = [] - for name in dir(self): - if name.startswith('handle_') and callable(getattr(self, name)): - handlers.append(name[len('handle_'):]) - return handlers - - def handle_method_call(self, method_name, *args, **kwargs): - """ - Pass through arbitrary method calls to the server. - - Args: - method_name (str): the method name to call. - Primarily intended for NestedAttrAccess, ie: - ``getattr``, ``setattr``, ``callattr``, ``delattr``. - - *args (Any): passed to the method - - **kwargs (Any): passed to the method - - Returns: - Any: the return value of the method - """ - return getattr(self, method_name)(*args, **kwargs) diff --git a/qcodes/process/stream_queue.py b/qcodes/process/stream_queue.py deleted file mode 100644 index 3dcbc5bff62..00000000000 --- a/qcodes/process/stream_queue.py +++ /dev/null @@ -1,152 +0,0 @@ -"""StreamQueue: collect subprocess stdout/stderr to a single queue.""" - -import multiprocessing as mp -import sys -import time - -from datetime import datetime - -from .helpers import kill_queue - - -def get_stream_queue(): - """ - Convenience function to get a singleton StreamQueue. - - note that this must be called from the main process before starting any - subprocesses that will use it, otherwise the subprocess will create its - own StreamQueue that no other processes know about - """ - if StreamQueue.instance is None: - StreamQueue.instance = StreamQueue() - return StreamQueue.instance - - -class StreamQueue: - - """ - Manages redirection of child process output for the main process to view. - - Do not instantiate this directly: use get_stream_queue so we only make one. - One StreamQueue should be created in the consumer process, and passed - to each child process. In the child, we call StreamQueue.connect with a - process name that will be unique and meaningful to the user. The consumer - then periodically calls StreamQueue.get() to read these messages. - - inspired by http://stackoverflow.com/questions/23947281/ - """ - - instance = None - - def __init__(self, *args, **kwargs): - """Create a StreamQueue, passing all args & kwargs to Queue.""" - self.queue = mp.Queue(*args, **kwargs) - self.last_read_ts = mp.Value('d', time.time()) - self._last_stream = None - self._on_new_line = True - self.lock = mp.RLock() - self.initial_streams = None - - def connect(self, process_name): - """ - Connect a child process to the StreamQueue. - - After this, stdout and stderr go to a queue rather than being - printed to a console. - - process_name: a short string that will clearly identify this process - to the user. - """ - if self.initial_streams is not None: - raise RuntimeError('StreamQueue is already connected') - - self.initial_streams = (sys.stdout, sys.stderr) - - sys.stdout = _SQWriter(self, process_name) - sys.stderr = _SQWriter(self, process_name + ' ERR') - - def disconnect(self): - """Disconnect a child from the queues and revert stdout & stderr.""" - if self.initial_streams is None: - raise RuntimeError('StreamQueue is not connected') - sys.stdout, sys.stderr = self.initial_streams - self.initial_streams = None - - def get(self): - """Read new messages from the queue and format them for printing.""" - out = '' - while not self.queue.empty(): - timestr, stream_name, msg = self.queue.get() - line_head = '[{} {}] '.format(timestr, stream_name) - - if self._on_new_line: - out += line_head - elif stream_name != self._last_stream: - out += '\n' + line_head - - out += msg[:-1].replace('\n', '\n' + line_head) + msg[-1] - - self._on_new_line = (msg[-1] == '\n') - self._last_stream = stream_name - - self.last_read_ts.value = time.time() - return out - - def __del__(self): - """Tear down the StreamQueue either on the main or a child process.""" - try: - self.disconnect() - except: - pass - - if hasattr(type(self), 'instance'): - # so nobody else tries to use this dismantled stream queue later - type(self).instance = None - - if hasattr(self, 'queue'): - kill_queue(self.queue) - del self.queue - if hasattr(self, 'lock'): - del self.lock - - -class _SQWriter: - MIN_READ_TIME = 3 - - def __init__(self, stream_queue, stream_name): - self.queue = stream_queue.queue - self.last_read_ts = stream_queue.last_read_ts - self.stream_name = stream_name - - def write(self, msg): - try: - if msg: - msgtuple = (datetime.now().strftime('%H:%M:%S.%f')[:-3], - self.stream_name, msg) - self.queue.put(msgtuple) - - queue_age = time.time() - self.last_read_ts.value - if queue_age > self.MIN_READ_TIME and msg != '\n': - # long time since the queue was read? maybe nobody is - # watching it at all - send messages to the terminal too - # but they'll still be in the queue if someone DOES look. - termstr = '[{} {}] {}'.format(*msgtuple) - # we always want a new line this way (so I don't use - # end='' in the print) but we don't want an extra if the - # caller already included a newline. - if termstr[-1] == '\n': - termstr = termstr[:-1] - try: - print(termstr, file=sys.__stdout__) - except ValueError: # pragma: no cover - # ValueError: underlying buffer has been detached - # this may just occur in testing on Windows, not sure. - pass - except: - # don't want to get an infinite loop if there's something wrong - # with the queue - put the regular streams back before handling - sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__ - raise - - def flush(self): - pass diff --git a/qcodes/station.py b/qcodes/station.py index 08bd8ba44fb..7b6cd9a020b 100644 --- a/qcodes/station.py +++ b/qcodes/station.py @@ -3,8 +3,6 @@ from qcodes.utils.metadata import Metadatable from qcodes.utils.helpers import make_unique, DelegateAttributes -from qcodes.instrument.remote import RemoteInstrument -from qcodes.instrument.remote import RemoteParameter from qcodes.instrument.base import Instrument from qcodes.instrument.parameter import Parameter from qcodes.instrument.parameter import ManualParameter @@ -86,13 +84,12 @@ def snapshot_base(self, update=False): } for name, itm in self.components.items(): - if isinstance(itm, (RemoteInstrument, - Instrument)): + if isinstance(itm, (Instrument)): snap['instruments'][name] = itm.snapshot(update=update) elif isinstance(itm, (Parameter, ManualParameter, - StandardParameter, - RemoteParameter)): + StandardParameter + )): snap['parameters'][name] = itm.snapshot(update=update) else: snap['components'][name] = itm.snapshot(update=update) diff --git a/qcodes/tests/instrument_mocks.py b/qcodes/tests/instrument_mocks.py index 3163d2e95b6..59c264d8b08 100644 --- a/qcodes/tests/instrument_mocks.py +++ b/qcodes/tests/instrument_mocks.py @@ -1,205 +1,10 @@ -import time import numpy as np from qcodes.instrument.base import Instrument -from qcodes.instrument.mock import MockInstrument, MockModel from qcodes.utils.validators import Numbers from qcodes.instrument.parameter import MultiParameter, ManualParameter -class AMockModel(MockModel): - - def __init__(self): - self._memory = {} - self._reset() - super().__init__() - - def _reset(self): - self._gates = [0.0, 0.0, 0.0] - self._excitation = 0.1 - - @staticmethod - def fmt(value): - return '{:.3f}'.format(value) - - def gates_set(self, parameter, value): - if parameter[0] == 'c': - self._gates[int(parameter[1:])] = float(value) - elif parameter == 'rst' and value is None: - # resets gates AND excitation, so we can use gates.reset() to - # reset the entire model - self._reset() - elif parameter[:3] == 'mem': - slot = int(parameter[3:]) - self._memory[slot] = value - else: - raise ValueError - - def gates_get(self, parameter): - if parameter[0] == 'c': - return self.fmt(self._gates[int(parameter[1:])]) - elif parameter[:3] == 'mem': - slot = int(parameter[3:]) - return self._memory[slot] - else: - raise ValueError - - def source_set(self, parameter, value): - if parameter == 'ampl': - try: - self._excitation = float(value) - except ValueError: - # "Off" as in the MultiType sweep step test - self._excitation = None - else: - raise ValueError(parameter, value) - - def source_get(self, parameter): - if parameter == 'ampl': - return self.fmt(self._excitation) - # put mem here too, just so we can be 100% sure it's going through - # the model - elif parameter[:3] == 'mem': - slot = int(parameter[3:]) - return self._memory[slot] - else: - raise ValueError - - def meter_get(self, parameter): - if parameter == 'ampl': - gates = self._gates - # here's my super complex model output! - return self.fmt(self._excitation * - (gates[0] + gates[1]**2 + gates[2]**3)) - elif parameter[:5] == 'echo ': - return self.fmt(float(parameter[5:])) - - # alias because we need new names when we instantiate an instrument - # locally at the same time as remotely - def gateslocal_set(self, parameter, value): - return self.gates_set(parameter, value) - - def gateslocal_get(self, parameter): - return self.gates_get(parameter) - - def sourcelocal_set(self, parameter, value): - return self.source_set(parameter, value) - - def sourcelocal_get(self, parameter): - return self.source_get(parameter) - - def meterlocal_get(self, parameter): - return self.meter_get(parameter) - - -class ParamNoDoc: - - def __init__(self, name, *args, **kwargs): - self.name = name - - def get_attrs(self): - return [] - - -class MockInstTester(MockInstrument): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.attach_adder() - - def attach_adder(self): - """ - this function attaches a closure to the object, so can only be - executed after creating the server because a closure is not - picklable - """ - a = 5 - - def f(b): - """ - not the same function as the original method - """ - return a + b - self.add5 = f - - def add5(self, b): - """ - The class copy of this should not get run, because it should - be overwritten on the server by the closure version. - """ - raise RuntimeError('dont run this one!') - - -class MockGates(MockInstTester): - - def __init__(self, name='gates', model=None, **kwargs): - super().__init__(name, model=model, delay=0.001, **kwargs) - - for i in range(3): - cmdbase = 'c{}'.format(i) - self.add_parameter('chan{}'.format(i), get_cmd=cmdbase + '?', - set_cmd=cmdbase + ':{:.4f}', - get_parser=float, - vals=Numbers(-10, 10)) - self.add_parameter('chan{}step'.format(i), - get_cmd=cmdbase + '?', - set_cmd=cmdbase + ':{:.4f}', - get_parser=float, - vals=Numbers(-10, 10), - step=0.1, delay=0.005) - - self.add_parameter('chan0slow', get_cmd='c0?', - set_cmd=self.slow_neg_set, get_parser=float, - vals=Numbers(-10, 10), step=0.2, - delay=0.02) - self.add_parameter('chan0slow2', get_cmd='c0?', - set_cmd=self.slow_neg_set, get_parser=float, - vals=Numbers(-10, 10), step=0.2, - delay=0.01, max_delay=0.02) - self.add_parameter('chan0slow3', get_cmd='c0?', - set_cmd=self.slow_neg_set, get_parser=float, - vals=Numbers(-10, 10), step=0.2, - delay=0.01, max_delay=0.08) - self.add_parameter('chan0slow4', get_cmd='c0?', - set_cmd=self.slow_neg_set, get_parser=float, - vals=Numbers(-10, 10), - delay=0.01, max_delay=0.02) - self.add_parameter('chan0slow5', get_cmd='c0?', - set_cmd=self.slow_neg_set, get_parser=float, - vals=Numbers(-10, 10), - delay=0.01, max_delay=0.08) - - self.add_function('reset', call_cmd='rst') - - self.add_parameter('foo', parameter_class=ParamNoDoc) - - def slow_neg_set(self, val): - if val < 0: - time.sleep(0.05) - self.chan0.set(val) - - -class MockSource(MockInstTester): - - def __init__(self, name='source', model=None, **kwargs): - super().__init__(name, model=model, delay=0.001, **kwargs) - - self.add_parameter('amplitude', get_cmd='ampl?', - set_cmd='ampl:{:.4f}', get_parser=float, - vals=Numbers(0, 1), - step=0.2, delay=0.005) - - -class MockMeter(MockInstTester): - - def __init__(self, name='meter', model=None, **kwargs): - super().__init__(name, model=model, delay=0.001, **kwargs) - - self.add_parameter('amplitude', get_cmd='ampl?', get_parser=float) - self.add_function('echo', call_cmd='echo {:.2f}?', - args=[Numbers(0, 1000)], return_parser=float) - - class MockParabola(Instrument): ''' Holds dummy parameters which are get and set able as well as provides diff --git a/qcodes/tests/test_driver_testcase.py b/qcodes/tests/test_driver_testcase.py deleted file mode 100644 index b91b04411c5..00000000000 --- a/qcodes/tests/test_driver_testcase.py +++ /dev/null @@ -1,58 +0,0 @@ -import unittest - -from qcodes.instrument_drivers.test import DriverTestCase -from qcodes.instrument.mock import MockInstrument, MockModel - - -class EmptyModel(MockModel): - pass - - -class MockMock(MockInstrument): - pass - - -@unittest.skip('just need this definition') -class HasNoDriver(DriverTestCase): - noskip = True - - -class MockMock2(MockInstrument): - pass - - -@unittest.skip('just need this definition') -class HasNoInstances(DriverTestCase): - noskip = True - driver = MockMock2 - - -class TestDriverTestCase(DriverTestCase): - driver = MockMock - noskip = True - - @classmethod - def setUpClass(cls): - cls.an_empty_model = EmptyModel() - cls.an_instrument = MockMock('a', model=cls.an_empty_model, server_name='') - super().setUpClass() - - @classmethod - def tearDownClass(cls): - cls.an_empty_model.close() - cls.an_instrument._manager.close() - - def test_instance_found(self): - self.assertEqual(self.instrument, self.an_instrument) - - def test_no_driver(self): - with self.assertRaises(TypeError): - HasNoDriver.setUpClass() - - def test_no_instances(self): - baseMock = MockInstrument('not the same class', - model=self.an_empty_model) - self.assertIn(baseMock, MockInstrument.instances()) - - with self.assertRaises(ValueError): - HasNoInstances.setUpClass() diff --git a/qcodes/tests/test_instrument_server.py b/qcodes/tests/test_instrument_server.py deleted file mode 100644 index 1284e03d0b7..00000000000 --- a/qcodes/tests/test_instrument_server.py +++ /dev/null @@ -1,157 +0,0 @@ -from unittest import TestCase -import time -import multiprocessing as mp - -from qcodes.instrument.server import InstrumentServer -from qcodes.instrument.base import Instrument -from qcodes.process.server import (QUERY_WRITE, QUERY_ASK, RESPONSE_OK, - RESPONSE_ERROR) -from qcodes.utils.helpers import LogCapture - - -def schedule(queries, query_queue): - """ - Args: - queries: is a sequence of (delay, args) - query_queue: is a queue to push these queries to, with each one waiting - its delay after sending the previous one - """ - for delay, args in queries: - time.sleep(delay) - query_queue.put(args) - - -def run_schedule(queries, query_queue): - p = mp.Process(target=schedule, args=(queries, query_queue)) - p.start() - return p - - -def get_results(response_queue): - time.sleep(0.05) # wait for any lingering messages to the queues - responses = [] - while not response_queue.empty(): - responses.append(response_queue.get()) - - return responses - - -class Holder: - shared_kwargs = ['where'] - name = 'J Edgar' - parameters = {} - functions = {} - - def __init__(self, server_name=None, **kwargs): - self.kwargs = kwargs - self.d = {} - - def get(self, key): - return self.d[key] - - def set(self, key, val): - self.d[key] = val - - def get_extras(self): - return self.kwargs - - def _get_method_attrs(self): - return {} - - def close(self): - pass - - def connection_attrs(self, new_id): - return Instrument.connection_attrs(self, new_id) - - -class TimedInstrumentServer(InstrumentServer): - timeout = 2 - - -class TestInstrumentServer(TestCase): - maxDiff = None - - @classmethod - def setUpClass(cls): - cls.query_queue = mp.Queue() - cls.response_queue = mp.Queue() - cls.error_queue = mp.Queue() - - @classmethod - def tearDownClass(cls): - del cls.query_queue - del cls.response_queue - del cls.error_queue - - def test_normal(self): - # we really only need to test local here - as a server it's already - # used in other tests, but only implicitly (and not covered as it's - # in a subprocess) - queries = ( - # add an "instrument" to the server - (0.5, (QUERY_ASK, 'new_id',)), - (0.01, (QUERY_ASK, 'new', (Holder, 0))), - - # some sets and gets that work - (0.01, (QUERY_WRITE, 'cmd', - (0, 'set', 'happiness', 'a warm gun'), {})), - (0.01, (QUERY_WRITE, 'cmd', - (0, 'set'), {'val': 42, 'key': 'the answer'})), - (0.01, (QUERY_ASK, 'cmd', (0, 'get'), {'key': 'happiness'})), - (0.01, (QUERY_ASK, 'cmd', (0, 'get', 'the answer',), {})), - - # then some that make errors - # KeyError - (0.01, (QUERY_ASK, 'cmd', (0, 'get', 'Carmen Sandiego',), {})), - # TypeError (too many args) shows up in logs - (0.01, (QUERY_WRITE, 'cmd', (0, 'set', 1, 2, 3), {})), - # TypeError (unexpected kwarg) shows up in logs - (0.01, (QUERY_WRITE, 'cmd', (0, 'set', 'do'), {'c': 'middle'})), - - # and another good one, just so we know it still works - (0.01, (QUERY_ASK, 'cmd', (0, 'get_extras'), {})), - - # delete the instrument and stop the server - # (no need to explicitly halt) - (0.01, (QUERY_ASK, 'delete', (0,))) - ) - extras = {'where': 'infinity and beyond'} - - run_schedule(queries, self.query_queue) - - try: - with LogCapture() as logs: - TimedInstrumentServer(self.query_queue, self.response_queue, - extras) - except TypeError: - from traceback import format_exc - print(format_exc()) - - self.assertEqual(logs.value.count('TypeError'), 2) - for item in ('1, 2, 3', 'middle'): - self.assertIn(item, logs.value) - - responses = get_results(self.response_queue) - - expected_responses = [ - (RESPONSE_OK, 0), - (RESPONSE_OK, { - 'functions': {}, - 'id': 0, - 'name': 'J Edgar', - '_methods': {}, - 'parameters': {} - }), - (RESPONSE_OK, 'a warm gun'), - (RESPONSE_OK, 42), - (RESPONSE_ERROR, ('KeyError', 'Carmen Sandiego')), - (RESPONSE_OK, extras) - ] - for response, expected in zip(responses, expected_responses): - if expected[0] == RESPONSE_OK: - self.assertEqual(response, expected) - else: - self.assertEqual(response[0], expected[0]) - for item in expected[1]: - self.assertIn(item, response[1]) diff --git a/qcodes/tests/test_multiprocessing.py b/qcodes/tests/test_multiprocessing.py deleted file mode 100644 index 5d9d0ee8233..00000000000 --- a/qcodes/tests/test_multiprocessing.py +++ /dev/null @@ -1,471 +0,0 @@ -from unittest import TestCase, skipIf -import time -import re -import sys -import multiprocessing as mp -from queue import Empty -from unittest.mock import patch - -import qcodes -from qcodes.process.helpers import set_mp_method, kill_queue -from qcodes.process.qcodes_process import QcodesProcess -from qcodes.process.stream_queue import get_stream_queue, _SQWriter -from qcodes.process.server import ServerManager, RESPONSE_OK, RESPONSE_ERROR -import qcodes.process.helpers as qcmp -from qcodes.utils.helpers import in_notebook, LogCapture -from qcodes.utils.timing import calibrate - -BREAK_SIGNAL = '~~BREAK~~' - - -class sqtest_echo: - def __init__(self, name, delay=0.01, has_q=True): - self.q_out = mp.Queue() - self.q_err = mp.Queue() - self.p = QcodesProcess(target=sqtest_echo_f, - args=(name, delay, self.q_out, self.q_err, - has_q), - name=name) - self.p.start() - self.delay = delay - self.resp_delay = delay * 2 + 0.03 - - def send_out(self, msg): - self.q_out.put(msg) - time.sleep(self.resp_delay) - - def send_err(self, msg): - self.q_err.put(msg) - time.sleep(self.resp_delay) - - def halt(self): - if not (hasattr(self, 'p') and self.p.is_alive()): - return - self.q_out.put(BREAK_SIGNAL) - self.p.join() - time.sleep(self.resp_delay) - for q in ['q_out', 'q_err']: - if hasattr(self, q): - queue = getattr(self, q) - kill_queue(queue) - kill_queue(queue) # repeat just to make sure it doesn't error - - def __del__(self): - self.halt() - - -def sqtest_echo_f(name, delay, q_out, q_err, has_q): - while True: - time.sleep(delay) - - if not q_out.empty(): - out = q_out.get() - - if out == BREAK_SIGNAL: - # now test that disconnect works, and reverts to - # regular stdout and stderr - if has_q: - try: - get_stream_queue().disconnect() - except RuntimeError: - pass - print('stdout ', end='', flush=True) - print('stderr ', file=sys.stderr, end='', flush=True) - break - - print(out, end='', flush=True) - - if not q_err.empty(): - print(q_err.get(), file=sys.stderr, end='', flush=True) - - -def sqtest_exception(): - raise RuntimeError('Boo!') - - -class TestMpMethod(TestCase): - def test_set_mp_method(self): - start_method = mp.get_start_method() - self.assertIn(start_method, ('fork', 'spawn', 'forkserver')) - - # multiprocessing's set_start_method is NOT idempotent - with self.assertRaises(RuntimeError): - mp.set_start_method(start_method) - - # but ours is - set_mp_method(start_method) - - # it will still error on gibberish, but different errors depending - # on whether you force or not - with self.assertRaises(RuntimeError): - set_mp_method('spoon') - with self.assertRaises(ValueError): - set_mp_method('spoon', force=True) - - # change the error we look for to test strange error handling - mp_err_normal = qcmp.MP_ERR - qcmp.MP_ERR = 'who cares?' - with self.assertRaises(RuntimeError): - set_mp_method('start_method') - qcmp.MP_ERR = mp_err_normal - - -class TestQcodesProcess(TestCase): - def setUp(self): - mp_stats = calibrate(quiet=True) - self.MP_START_DELAY = mp_stats['mp_start_delay'] - self.MP_FINISH_DELAY = mp_stats['mp_finish_delay'] - self.SLEEP_DELAY = mp_stats['sleep_delay'] - self.BLOCKING_TIME = mp_stats['blocking_time'] - self.sq = get_stream_queue() - - @skipIf(getattr(qcodes, '_IN_NOTEBOOK', False), - 'called from notebook') - def test_not_in_notebook(self): - # below we'll patch this to True, but make sure that it's False - # in the normal test runner. - self.assertEqual(in_notebook(), False) - - # and make sure that processes run this way do not use the queue - with self.sq.lock: - p = sqtest_echo('hidden', has_q=False) - time.sleep(self.MP_START_DELAY) - p.send_out('should go to stdout;') - p.send_err('should go to stderr;') - p.halt() - - self.assertEqual(self.sq.get(), '') - - @patch('qcodes.process.qcodes_process.in_notebook') - def test_qcodes_process_exception(self, in_nb_patch): - in_nb_patch.return_value = True - - with self.sq.lock: - name = 'Hamlet' - p = QcodesProcess(target=sqtest_exception, name=name) - - initial_outs = (sys.stdout, sys.stderr) - - # normally you call p.start(), but for this test we want - # the function to actually run in the main process - # it will run the actual target, but will print the exception - # (to the queue) rather than raising it. - p.run() - - # output streams are back to how they started - self.assertEqual((sys.stdout, sys.stderr), initial_outs) - time.sleep(0.01) - exc_text = self.sq.get() - # but we have the exception in the queue - self.maxDiff = None - self.assertGreaterEqual(exc_text.count(name + ' ERR'), 5) - self.assertEqual(exc_text.count('Traceback'), 1, exc_text) - self.assertEqual(exc_text.count('RuntimeError'), 2) - self.assertEqual(exc_text.count('Boo!'), 2) - - @patch('qcodes.process.qcodes_process.in_notebook') - def test_qcodes_process(self, in_nb_patch): - in_nb_patch.return_value = True - - queue_format = re.compile( - '^\[\d\d:\d\d:\d\d\.\d\d\d p\d( ERR)?\] [^\[\]]*$') - - with self.sq.lock: - p1 = sqtest_echo('p1') - p2 = sqtest_echo('p2') - time.sleep(self.MP_START_DELAY + p1.delay + p2.delay) - - self.assertEqual(self.sq.get(), '') - - procNames = ['<{}>'.format(name) - for name in ('p1', 'p2')] - - reprs = [repr(p) for p in mp.active_children()] - for name in procNames: - self.assertIn(name, reprs) - - # test each individual stream to send several messages on same - # and different lines - - for sender, label, term in ([[p1.send_out, 'p1] ', ''], - [p1.send_err, 'p1 ERR] ', '\n'], - [p2.send_out, 'p2] ', '\n'], - [p2.send_err, 'p2 ERR] ', '']]): - sender('row row ') - sender('row your boat\n') - sender('gently down ') - time.sleep(0.01) - data = [line for line in self.sq.get().split('\n') if line] - expected = [ - label + 'row row row your boat', - label + 'gently down ' - ] - # TODO - intermittent error here - self.assertEqual(len(data), len(expected), data) - for line, expected_line in zip(data, expected): - self.assertIsNotNone(queue_format.match(line), data) - self.assertEqual(line[14:], expected_line, data) - - sender(' the stream' + term) - # no label/header as we're continuing the previous line - self.assertEqual(self.sq.get(), ' the stream' + term) - - p1.send_out('marco') - p2.send_out('polo\n') # we don't see these single terminators - p1.send_out('marco\n') # when we change streams - p2.send_out('polo') - time.sleep(0.01) - - data = self.sq.get().split('\n') - for line in data: - if line: - self.assertIsNotNone(queue_format.match(line)) - - data_msgs = [line[14:] for line in data] - expected = [ - '', - 'p1] marco', - 'p2] polo', - 'p1] marco', - 'p2] polo' - ] - self.assertEqual(data_msgs, expected) - - # Some OS's start more processes just for fun... so don't test - # that p1 and p2 are the only ones. - # self.assertEqual(len(reprs), 2, reprs) - - p1.halt() - p2.halt() - # both p1 and p2 should have finished now, and ended. - reprs = [repr(p) for p in mp.active_children()] - for name in procNames: - self.assertNotIn(name, reprs) - - -class TestStreamQueue(TestCase): - def test_connection(self): - sq = get_stream_queue() - sq.connect('') - # TODO: do we really want double-connect to raise? or maybe - # only raise if the process name changes? - with self.assertRaises(RuntimeError): - sq.connect('') - sq.disconnect() - with self.assertRaises(RuntimeError): - sq.disconnect() - - def test_del(self): - sq = get_stream_queue() - self.assertTrue(hasattr(sq, 'queue')) - self.assertTrue(hasattr(sq, 'lock')) - self.assertIsNotNone(sq.instance) - - sq.__del__() - - self.assertFalse(hasattr(sq, 'queue')) - self.assertFalse(hasattr(sq, 'lock')) - self.assertIsNone(sq.instance) - - sq.__del__() # just to make sure it doesn't error - - # this is basically tested in TestQcodesProcess, but the test happens - # in a subprocess so coverage doesn't know about it. Anyway, there are - # a few edge cases left that we have to test locally. - def test_sq_writer(self): - sq = get_stream_queue() - with sq.lock: - sq_clearer = _SQWriter(sq, 'Someone else') - sq_clearer.write('Boo!\n') - - # apparently we need a little delay to make sure the queue - # properly appears populated. - time.sleep(0.01) - - sq.get() - sq_name = 'A Queue' - sqw = _SQWriter(sq, sq_name) - - # flush should exist, but does nothing - sqw.flush() - - lines = [ - 'Knock knock.\nWho\'s there?\n', - 'Interrupting cow.\n', - 'Interr-', - 'MOO!\n', - '' - ] - - for line in lines: - sqw.write(line) - - time.sleep(0.01) - - # _SQWriter doesn't do any transformations to the messages, beyond - # adding a time string and the stream name - for line in lines: - if not line: - self.assertEqual(sq.queue.empty(), True) - continue - - self.assertEqual(sq.queue.empty(), False) - timestr, stream_name, msg = sq.queue.get() - self.assertEqual(msg, line) - self.assertEqual(stream_name, sq_name) - - # now test that even if the queue is unwatched the messages still - # go there. If we're feeling adventurous maybe we can test if - # something was actually printed. - sqw.MIN_READ_TIME = -1 - new_message = 'should get printed\n' - sqw.write(new_message) - - time.sleep(0.01) - - self.assertEqual(sq.queue.empty(), False) - self.assertEqual(sq.queue.get()[2], new_message) - - # test that an error in writing resets stdout and stderr - # nose uses its own stdout and stderr... so keep them (as we will - # force stdout and stderr to several other things) so we can put - # them back at the end - nose_stdout = sys.stdout - nose_stderr = sys.stderr - sys.stdout = _SQWriter(sq, 'mock_stdout') - sys.stderr = _SQWriter(sq, 'mock_stderr') - self.assertNotIn(sys.stdout, (sys.__stdout__, nose_stdout)) - self.assertNotIn(sys.stderr, (sys.__stderr__, nose_stderr)) - - sqw.MIN_READ_TIME = 'not a number' - with self.assertRaises(TypeError): - sqw.write('trigger an error') - - self.assertEqual(sys.stdout, sys.__stdout__) - self.assertEqual(sys.stderr, sys.__stderr__) - - sys.stdout = nose_stdout - sys.stderr = nose_stderr - time.sleep(0.01) - sq.get() - - -class ServerManagerTest(ServerManager): - def _start_server(self): - # don't really start the server - we'll test its pieces separately, - # in the main process - pass - - -class EmptyServer: - def __init__(self, query_queue, response_queue, extras): - query_queue.put('why?') - response_queue.put(extras) - - -class CustomError(Exception): - pass - - -def delayed_put(queue, val, delay): - time.sleep(delay) - queue.put(val) - - -class TestServerManager(TestCase): - def check_error(self, manager, error_str, error_class): - manager._response_queue.put(RESPONSE_ERROR, error_str) - with self.assertRaises(error_class): - manager.ask('which way does the wind blow?') - - def test_mechanics(self): - extras = 'super secret don\'t tell anyone' - - sm = ServerManagerTest(name='test', server_class=EmptyServer, - shared_attrs=extras) - sm._run_server() - - self.assertEqual(sm._query_queue.get(timeout=1), 'why?') - self.assertEqual(sm._response_queue.get(timeout=1), extras) - - # builtin errors we propagate to the server - builtin_error_str = ('traceback\n lines\n and then\n' - ' OSError: your hard disk went floppy.') - sm._response_queue.put((RESPONSE_ERROR, builtin_error_str)) - with self.assertRaises(OSError): - sm.ask('which way does the wind blow?') - - # non-built-in errors we fall back on RuntimeError - custom_error_str = ('traceback\nlines\nand then\n' - 'CustomError: the Balrog is loose!') - extra_resp1 = 'should get tossed by the error checker' - extra_resp2 = 'so should this.' - sm._response_queue.put((RESPONSE_OK, extra_resp1)) - sm._response_queue.put((RESPONSE_OK, extra_resp2)) - sm._response_queue.put((RESPONSE_ERROR, custom_error_str)) - - # TODO: we have an intermittent failure below, but only when running - # the full test suite (including pyqt and matplotlib?), not if we - # run just this module, or at least not nearly as frequently. - time.sleep(0.2) - - with LogCapture() as logs: - with self.assertRaises(RuntimeError): - sm.ask('something benign') - self.assertTrue(sm._response_queue.empty()) - self.assertIn(extra_resp1, logs.value) - self.assertIn(extra_resp2, logs.value) - - # extra responses to a query, only the last should be taken - extra_resp1 = 'boo!' - extra_resp2 = 'a barrel of monkeys!' - sm._response_queue.put((RESPONSE_OK, extra_resp1)) - sm._response_queue.put((RESPONSE_OK, extra_resp2)) - time.sleep(0.05) - p = mp.Process(target=delayed_put, - args=(sm._response_queue, (RESPONSE_OK, 42), 0.05)) - p.start() - - with LogCapture() as logs: - self.assertEqual(sm.ask('what is the answer'), 42) - self.assertIn(extra_resp1, logs.value) - self.assertIn(extra_resp2, logs.value) - - # no response to a query - with self.assertRaises(Empty): - sm.ask('A sphincter says what?', timeout=0.05) - - # test halting an unresponsive server - sm._server = mp.Process(target=time.sleep, args=(1000,)) - sm._server.start() - - self.assertIn(sm._server, mp.active_children()) - - with LogCapture() as logs: - sm.halt(0.01) - self.assertIn('ServerManager did not respond ' - 'to halt signal, terminated', logs.value) - - self.assertNotIn(sm._server, mp.active_children()) - - def test_pathological_edge_cases(self): - # kill_queue should never fail - kill_queue(None) - - # and halt should ignore AssertionErrors, which arise in - # subprocesses when trying to kill a different subprocess - sm = ServerManagerTest(name='test', server_class=None) - - class HorribleProcess: - def is_alive(self): - raise AssertionError - - def write(self): - raise AssertionError - - def join(self): - raise AssertionError - - sm._server = HorribleProcess() - - sm.halt() diff --git a/qcodes/tests/test_nested_attrs.py b/qcodes/tests/test_nested_attrs.py deleted file mode 100644 index 0cf0dc19744..00000000000 --- a/qcodes/tests/test_nested_attrs.py +++ /dev/null @@ -1,100 +0,0 @@ -from unittest import TestCase -from qcodes.utils.nested_attrs import NestedAttrAccess - - -class TestNestedAttrAccess(TestCase): - def test_simple(self): - obj = NestedAttrAccess() - - # before setting attr1 - self.assertEqual(obj.getattr('attr1', 99), 99) - with self.assertRaises(AttributeError): - obj.getattr('attr1') - - with self.assertRaises(TypeError): - obj.setattr('attr1') - - self.assertFalse(hasattr(obj, 'attr1')) - - # set it to a value - obj.setattr('attr1', 98) - self.assertTrue(hasattr(obj, 'attr1')) - - self.assertEqual(obj.getattr('attr1', 99), 98) - self.assertEqual(obj.getattr('attr1'), 98) - - # then delete it - obj.delattr('attr1') - - with self.assertRaises(AttributeError): - obj.delattr('attr1') - - with self.assertRaises(AttributeError): - obj.getattr('attr1') - - # make and call a method - def f(a, b=0): - return a + b - - obj.setattr('m1', f) - self.assertEqual(obj.callattr('m1', 4, 1), 5) - self.assertEqual(obj.callattr('m1', 21, b=42), 63) - - def test_nested(self): - obj = NestedAttrAccess() - - self.assertFalse(hasattr(obj, 'd1')) - - with self.assertRaises(TypeError): - obj.setattr('d1') - - # set one attribute that creates nesting - obj.setattr('d1', {'a': {1: 2, 'l': [5, 6]}}) - - # can't nest inside a non-container - with self.assertRaises(TypeError): - obj.setattr('d1["a"][1]["secret"]', 42) - - # get the whole dict - self.assertEqual(obj.getattr('d1'), {'a': {1: 2, 'l': [5, 6]}}) - self.assertEqual(obj.getattr('d1', 55), {'a': {1: 2, 'l': [5, 6]}}) - - # get parts - self.assertEqual(obj.getattr('d1["a"]'), {1: 2, 'l': [5, 6]}) - self.assertEqual(obj.getattr('d1["a"][1]'), 2) - self.assertEqual(obj.getattr('d1["a"][1]', 3), 2) - with self.assertRaises(KeyError): - obj.getattr('d1["b"]') - - # add an attribute inside, then delete it again - obj.setattr('d1["a"][2]', 4) - self.assertEqual(obj.getattr('d1'), {'a': {1: 2, 2: 4, 'l': [5, 6]}}) - obj.delattr('d1["a"][2]') - self.assertEqual(obj.getattr('d1'), {'a': {1: 2, 'l': [5, 6]}}) - self.assertEqual(obj.d1, {'a': {1: 2, 'l': [5, 6]}}) - - # list access - obj.setattr('d1["a"]["l"][0]', 7) - obj.callattr('d1["a"]["l"].extend', [5, 3]) - obj.delattr('d1["a"]["l"][1]') - # while we're at it test single quotes - self.assertEqual(obj.getattr("d1['a']['l'][1]"), 5) - self.assertEqual(obj.d1['a']['l'], [7, 5, 3]) - - def test_bad_attr(self): - obj = NestedAttrAccess() - obj.d = {} - # this one works - obj.setattr('d["x"]', 1) - - bad_attrs = [ - '', '.', '[', 'x.', '[]', # simply malformed - '.x' # don't put a dot at the start - '["hi"]', # can't set an item at the top level - 'd[x]', 'd["x]', 'd["x\']' # quoting errors - ] - - for attr in bad_attrs: - with self.subTest(attr=attr): - with self.assertRaises(ValueError): - obj.setattr(attr, 1) diff --git a/qcodes/tests/test_visa.py b/qcodes/tests/test_visa.py index 07a7e0638b1..76407afdc84 100644 --- a/qcodes/tests/test_visa.py +++ b/qcodes/tests/test_visa.py @@ -60,12 +60,6 @@ def ask(self, cmd): class TestVisaInstrument(TestCase): - def test_default_server_name(self): - dsn = VisaInstrument.default_server_name - self.assertEqual(dsn(), 'VisaServer') - self.assertEqual(dsn(address='Gpib::10'), 'GPIBServer') - self.assertEqual(dsn(address='aSRL4'), 'SerialServer') - # error args for set(-10) args1 = [ 'be more positive!', @@ -116,43 +110,6 @@ def test_ask_write_local(self): mv.close() - def test_ask_write_server(self): - # same thing as above but Joe is on a server now... - mv = MockVisa('Joe', server_name='') - - # test normal ask and write behavior - mv.state.set(2) - self.assertEqual(mv.state.get(), 2) - mv.state.set(3.4567) - self.assertEqual(mv.state.get(), 3.457) # driver rounds to 3 digits - - # test ask and write errors - with self.assertRaises(ValueError) as e: - mv.state.set(-10) - for arg in self.args1: - self.assertIn(repr(arg), e.exception.args[0]) - self.assertEqual(mv.state.get(), -10) # set still happened - - # only built-in errors get propagated to the main process as the - # same type. Perhaps we could include some more common ones like - # this (visa.VisaIOError) in the future... - with self.assertRaises(RuntimeError) as e: - mv.state.set(0) - for arg in self.args2: - self.assertIn(repr(arg), e.exception.args[0]) - # the error type isn't VisaIOError, but it should be in the message - self.assertIn('VisaIOError', e.exception.args[0]) - self.assertIn('VI_ERROR_TMO', e.exception.args[0]) - self.assertEqual(mv.state.get(), 0) - - mv.state.set(15) - with self.assertRaises(ValueError) as e: - mv.state.get() - for arg in self.args3: - self.assertIn(repr(arg), e.exception.args[0]) - - mv.close() - @patch('qcodes.instrument.visa.visa.ResourceManager') def test_visa_backend(self, rm_mock): address_opened = [None] From 1ea05fc3d3ccbdfb54a60082ffa81b9d357304fe Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 11:21:03 +0100 Subject: [PATCH 11/36] remove Widgets --- qcodes/__init__.py | 5 - qcodes/widgets/__init__.py | 0 qcodes/widgets/display.py | 47 -------- qcodes/widgets/widgets.css | 77 ------------ qcodes/widgets/widgets.js | 237 ------------------------------------- qcodes/widgets/widgets.py | 215 --------------------------------- 6 files changed, 581 deletions(-) delete mode 100644 qcodes/widgets/__init__.py delete mode 100644 qcodes/widgets/display.py delete mode 100644 qcodes/widgets/widgets.css delete mode 100644 qcodes/widgets/widgets.js delete mode 100644 qcodes/widgets/widgets.py diff --git a/qcodes/__init__.py b/qcodes/__init__.py index f12b0489fc1..71bc19c9400 100644 --- a/qcodes/__init__.py +++ b/qcodes/__init__.py @@ -29,11 +29,6 @@ 'try "from qcodes.plots.pyqtgraph import QtPlot" ' 'to see the full error') -# only import in name space if the gui is set to noebook -# and there is multiprocessing -if config['gui']['notebook'] and config['core']['legacy_mp']: - from qcodes.widgets.widgets import show_subprocess_widget - from qcodes.station import Station from qcodes.loops import Loop from qcodes.measure import Measure diff --git a/qcodes/widgets/__init__.py b/qcodes/widgets/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/qcodes/widgets/display.py b/qcodes/widgets/display.py deleted file mode 100644 index 6a705dbb2cd..00000000000 --- a/qcodes/widgets/display.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Helper for adding content stored in a file to a jupyter notebook.""" -import os -from pkg_resources import resource_string -from IPython.display import display, Javascript, HTML - - -# Originally I implemented this using regular open() and read(), so it -# could use relative paths from the importing file. -# -# But for distributable packages, pkg_resources.resource_string is the -# best way to load data files, because it works even if the package is -# in an egg or zip file. See: -# http://pythonhosted.org/setuptools/setuptools.html#accessing-data-files-at-runtime - -def display_auto(qcodes_path, file_type=None): - """ - Display some javascript, css, or html content in a jupyter notebook. - - Content comes from a package-relative file path. Will use the file - extension to determine file type unless overridden by file_type - - Args: - qcodes_path (str): the path to the target file within the qcodes - package, like 'widgets/widgets.js' - - file_type (Optional[str]): Override the file extension to determine - what type of file this is. Case insensitive, supported values - are 'js', 'css', and 'html' - """ - contents = resource_string('qcodes', qcodes_path).decode('utf-8') - - if file_type is None: - ext = os.path.splitext(qcodes_path)[1].lower() - elif 'js' in file_type.lower(): - ext = '.js' - elif 'css' in file_type.lower(): - ext = '.css' - else: - ext = '.html' - - if ext == '.js': - display(Javascript(contents)) - elif ext == '.css': - display(HTML('')) - else: - # default to html. Anything else? - display(HTML(contents)) diff --git a/qcodes/widgets/widgets.css b/qcodes/widgets/widgets.css deleted file mode 100644 index ab4204ae5e4..00000000000 --- a/qcodes/widgets/widgets.css +++ /dev/null @@ -1,77 +0,0 @@ -.qcodes-output-view:not(.ui-draggable) { - bottom: 0; - right: 5px; -} -.qcodes-output-view { - position: fixed; - z-index: 999; - background-color: #fff; - box-shadow: 0 0 12px 1px rgba(87, 87, 87, 0.2); -} - -.qcodes-output-header { - float: right; -} - -.qcodes-highlight { - animation: pulse 1s linear; - background-color: #fa4; -} - -@keyframes pulse { - 0% { - background-color: #f00; - } - 100% { - background-color: #fa4; - } -} - -.qcodes-process-list { - float: left; - max-width: 780px; - margin: 3px 5px 3px 10px; - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; -} - -.qcodes-output-view[qcodes-state=minimized] .qcodes-process-list { - max-width: 300px; -} - -.qcodes-output-view span { - padding: 2px 6px 3px 12px; -} - -.qcodes-output-view .btn { - margin: 0 3px 0 0; -} - -.qcodes-output-view[qcodes-state=docked] .qcodes-docked, -.qcodes-output-view[qcodes-state=floated] .qcodes-floated, -.qcodes-output-view[qcodes-state=minimized] .qcodes-minimized, -.qcodes-output-view[qcodes-state=minimized] .qcodes-content { - display: none; -} - -.qcodes-output-view .disabled { - opacity: 0.4; -} - -.qcodes-abort-loop { - background-color: #844; - color: #fff; -} - -.qcodes-output-view pre { - clear: both; - margin: 0; - border: 0; - border-top: 1px solid #ccc; - background-color: #ffe; - min-height: 50px; - max-height: 400px; - min-width: 400px; - max-width: 1000px; -} \ No newline at end of file diff --git a/qcodes/widgets/widgets.js b/qcodes/widgets/widgets.js deleted file mode 100644 index c93f0882d38..00000000000 --- a/qcodes/widgets/widgets.js +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Qcodes Jupyter/IPython widgets - */ -require([ - 'nbextensions/widgets/widgets/js/widget', - 'nbextensions/widgets/widgets/js/manager' -], function (widget, manager) { - - var UpdateView = widget.DOMWidgetView.extend({ - render: function() { - window.MYWIDGET = this; - this._interval = 0; - this.update(); - }, - update: function() { - this.display(this.model.get('_message')); - this.setInterval(); - }, - display: function(message) { - /* - * display method: override this for custom display logic - */ - this.el.innerHTML = message; - }, - remove: function() { - clearInterval(this._updater); - }, - setInterval: function(newInterval) { - var me = this; - if(newInterval===undefined) newInterval = me.model.get('interval'); - if(newInterval===me._interval) return; - - me._interval = newInterval; - - if(me._updater) clearInterval(me._updater); - - if(me._interval) { - me._updater = setInterval(function() { - me.send({myupdate: true}); - if(!me.model.comm_live) { - console.log('missing comm, canceling widget updates', me); - clearInterval(me._updater); - } - }, me._interval * 1000); - } - } - }); - manager.WidgetManager.register_widget_view('UpdateView', UpdateView); - - var HiddenUpdateView = UpdateView.extend({ - display: function(message) { - this.$el.hide(); - } - }); - manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView); - - var SubprocessView = UpdateView.extend({ - render: function() { - var me = this; - me._interval = 0; - me._minimize = ''; - me._restore = ''; - - // max lines of output to show - me.maxOutputLength = 500; - - // in case there is already an outputView present, - // like from before restarting the kernel - $('.qcodes-output-view').not(me.$el).remove(); - - me.$el - .addClass('qcodes-output-view') - .attr('qcodes-state', 'docked') - .html( - '
' + - '
' + - '' + - '' + - '' + - '' + - '' + - '' + - '
' + - '
'
-                );
-
-            me.clearButton = me.$el.find('.qcodes-clear-output');
-            me.minButton = me.$el.find('.qcodes-minimize');
-            me.outputArea = me.$el.find('pre');
-            me.subprocessList = me.$el.find('.qcodes-process-list');
-            me.abortButton = me.$el.find('.qcodes-abort-loop');
-            me.processLinesButton = me.$el.find('.qcodes-processlines')
-
-            me.outputLines = [];
-
-            me.clearButton.click(function() {
-                me.outputArea.html('');
-                me.clearButton.addClass('disabled');
-            });
-
-            me.abortButton.click(function() {
-                me.send({abort: true});
-            });
-
-            me.processLinesButton.click(function() {
-                // toggle multiline process list display
-                me.subprocessesMultiline = !me.subprocessesMultiline;
-                me.showSubprocesses();
-            });
-
-            me.$el.find('.js-state').click(function() {
-                var state = this.className.substr(this.className.indexOf('qcodes'))
-                        .split('-')[1].split(' ')[0];
-                me.model.set('_state', state);
-            });
-
-            $(window)
-                .off('resize.qcodes')
-                .on('resize.qcodes', function() {me.clipBounds();});
-
-            me.update();
-        },
-
-        updateState: function() {
-            var me = this,
-                oldState = me.$el.attr('qcodes-state'),
-                state = me.model.get('_state');
-
-            if(state === oldState) return;
-
-            setTimeout(function() {
-                // not sure why I can't pop it out of the widgetarea in render, but it seems that
-                // some other bit of code resets the parent after render if I do it there.
-                // To be safe, just do it on every state click.
-                me.$el.appendTo('body');
-
-                if(oldState === 'floated') {
-                    console.log('here');
-                    me.$el.draggable('destroy').css({left:'', top: ''});
-                }
-
-                me.$el.attr('qcodes-state', state);
-
-                if(state === 'floated') {
-                    me.$el
-                        .draggable({stop: function() { me.clipBounds(); }})
-                        .css({
-                            left: window.innerWidth - me.$el.width() - 15,
-                            top: window.innerHeight - me.$el.height() - 10
-                        });
-                }
-
-                // any previous highlighting is now moot
-                me.$el.removeClass('qcodes-highlight');
-            }, 0);
-
-        },
-
-        clipBounds: function() {
-            var me = this;
-            if(me.$el.attr('qcodes-state') === 'floated') {
-                var bounds = me.$el[0].getBoundingClientRect(),
-                    minVis = 40,
-                    maxLeft = window.innerWidth - minVis,
-                    minLeft = minVis - bounds.width,
-                    maxTop = window.innerHeight - minVis;
-
-                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);
-                else if(bounds.left < minLeft) me.$el.css('left', minLeft);
-
-                if(bounds.top > maxTop) me.$el.css('top', maxTop);
-                else if(bounds.top < 0) me.$el.css('top', 0);
-            }
-        },
-
-        display: function(message) {
-            var me = this;
-            if(message) {
-                var initialScroll = me.outputArea.scrollTop();
-                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));
-                var scrollBottom = me.outputArea.scrollTop();
-
-                if(me.$el.attr('qcodes-state') === 'minimized') {
-                    // if we add text and the box is minimized, highlight the
-                    // title bar to alert the user that there are new messages.
-                    // remove then add the class, so we get the animation again
-                    // if it's already highlighted
-                    me.$el.removeClass('qcodes-highlight');
-                    setTimeout(function(){
-                        me.$el.addClass('qcodes-highlight');
-                    }, 0);
-                }
-
-                var newLines = message.split('\n'),
-                    out = me.outputLines,
-                    outLen = out.length;
-                if(outLen) out[outLen - 1] += newLines[0];
-                else out.push(newLines[0]);
-
-                for(var i = 1; i < newLines.length; i++) {
-                    out.push(newLines[i]);
-                }
-
-                if(out.length > me.maxOutputLength) {
-                    out.splice(0, out.length - me.maxOutputLength + 1,
-                        '<<< Output clipped >>>');
-                }
-
-                me.outputArea.text(out.join('\n'));
-                me.clearButton.removeClass('disabled');
-
-                // if we were scrolled to the bottom initially, make sure
-                // we stay that way.
-                me.outputArea.scrollTop(initialScroll === scrollBottom ?
-                    me.outputArea.prop('scrollHeight') : initialScroll);
-            }
-
-            me.showSubprocesses();
-            me.updateState();
-        },
-
-        showSubprocesses: function() {
-            var me = this,
-                replacer = me.subprocessesMultiline ? '
' : ', ', - processes = (me.model.get('_processes') || '') - .replace(/\n/g, '>' + replacer + '<'); - - if(processes) processes = '<' + processes + '>'; - else processes = 'No subprocesses'; - - me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1); - - me.subprocessList.html(processes); - } - }); - manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView); -}); diff --git a/qcodes/widgets/widgets.py b/qcodes/widgets/widgets.py deleted file mode 100644 index 56ea36dd45c..00000000000 --- a/qcodes/widgets/widgets.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Qcodes-specific widgets for jupyter notebook.""" -from IPython.display import display -from ipywidgets import widgets -from multiprocessing import active_children -from traitlets import Unicode, Float, Enum - -from qcodes.process.stream_queue import get_stream_queue -from .display import display_auto -from qcodes.loops import MP_NAME, halt_bg - -display_auto('widgets/widgets.js') -display_auto('widgets/widgets.css') - - -class UpdateWidget(widgets.DOMWidget): - - """ - Execute a callable periodically, and display its return in the output area. - - The Javascript portion of this is in widgets.js with the same name. - - Args: - fn (callable): To be called (with no parameters) periodically. - - interval (number): The call period, in seconds. Can be changed later - by setting the ``interval`` attribute. ``interval=0`` or the - ``halt()`` method disables updates. - - first_call (bool): Whether to call the update function immediately - or only after the first interval. Default True. - """ - - _view_name = Unicode('UpdateView', sync=True) # see widgets.js - _message = Unicode(sync=True) - interval = Float(sync=True) - - def __init__(self, fn, interval, first_call=True): - super().__init__() - - self._fn = fn - self.interval = interval - self.previous_interval = interval - - # callbacks send the widget (self) as the first arg - # so bind to __func__ and we can leave the duplicate out - # of the method signature - self.on_msg(self.do_update.__func__) - - if first_call: - self.do_update({}, []) - - def do_update(self, content=None, buffers=None): - """ - Execute the callback and send its return value to the notebook. - - Args: - content: required by DOMWidget, unused - buffers: required by DOMWidget, unused - """ - self._message = str(self._fn()) - - def halt(self): - """ - Stop future updates. - - Keeps a record of the interval so we can ``restart()`` later. - You can also restart by explicitly setting ``self.interval`` to a - positive value. - """ - if self.interval: - self.previous_interval = self.interval - self.interval = 0 - - def restart(self, **kwargs): - """ - Reinstate updates with the most recent interval. - - TODO: why did I include kwargs? - """ - if not hasattr(self, 'previous_interval'): - self.previous_interval = 1 - - if self.interval != self.previous_interval: - self.interval = self.previous_interval - - -class HiddenUpdateWidget(UpdateWidget): - - """ - A variant on UpdateWidget that hides its section of the output area. - - The Javascript portion of this is in widgets.js with the same name. - - Just lets the front end periodically execute code that takes care of its - own display. By default, first_call is False here, unlike UpdateWidget, - because it is assumed this widget is created to update something that - has been displayed by other means. - - Args: - fn (callable): To be called (with no parameters) periodically. - - interval (number): The call period, in seconds. Can be changed later - by setting the ``interval`` attribute. ``interval=0`` or the - ``halt()`` method disables updates. - - first_call (bool): Whether to call the update function immediately - or only after the first interval. Default False. - """ - - _view_name = Unicode('HiddenUpdateView', sync=True) # see widgets.js - - def __init__(self, *args, first_call=False, **kwargs): - super().__init__(*args, first_call=first_call, **kwargs) - - -def get_subprocess_widget(**kwargs): - """ - Convenience function to get a singleton SubprocessWidget. - - Restarts widget updates if it has been halted. - - Args: - **kwargs: passed to SubprocessWidget constructor - - Returns: - SubprocessWidget - """ - if SubprocessWidget.instance is None: - w = SubprocessWidget(**kwargs) - else: - w = SubprocessWidget.instance - - w.restart() - - return w - - -def show_subprocess_widget(**kwargs): - """ - Display the subprocess widget, creating it if needed. - - Args: - **kwargs: passed to SubprocessWidget constructor - """ - display(get_subprocess_widget(**kwargs)) - - -class SubprocessWidget(UpdateWidget): - - """ - Display subprocess output in a box in the jupyter notebook window. - - Output is collected from each process's stdout and stderr by the - ``StreamQueue`` and read periodically from the main process, triggered - by Javascript. - - The Javascript portion of this is in widgets.js with the same name. - - Args: - interval (number): The call period, in seconds. Can be changed later - by setting the ``interval`` attribute. ``interval=0`` or the - ``halt()`` method disables updates. Default 0.5. - state (str): starting window state of the widget. Options are - 'docked' (default), 'minimized', 'floated' - """ - - _view_name = Unicode('SubprocessView', sync=True) # see widgets.js - _processes = Unicode(sync=True) - _state = Enum(('minimized', 'docked', 'floated'), sync=True) - - instance = None - - # max seconds to wait for a measurement to abort - abort_timeout = 30 - - def __init__(self, interval=0.5, state='docked'): - if self.instance is not None: - raise RuntimeError( - 'Only one instance of SubprocessWidget should exist at ' - 'a time. Use the function get_subprocess_output to find or ' - 'create it.') - - self.stream_queue = get_stream_queue() - self._state = state - super().__init__(fn=None, interval=interval) - - self.__class__.instance = self - - def do_update(self, content=None, buffers=None): - """ - Update the information to be displayed in the widget. - - Send any new messages to the notebook, and update the list of - active processes. - - Args: - content: required by DOMWidget, unused - buffers: required by DOMWidget, unused - """ - self._message = self.stream_queue.get() - - loops = [] - others = [] - - for p in active_children(): - if getattr(p, 'name', '') == MP_NAME: - # take off the <> on the ends, just to shorten the names - loops.append(str(p)[1:-1]) - else: - others.append(str(p)[1:-1]) - - self._processes = '\n'.join(loops + others) - - if content.get('abort'): - halt_bg(timeout=self.abort_timeout, traceback=False) From 3aada668eac77197122039864f71e367215b4686 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 11:38:52 +0100 Subject: [PATCH 12/36] chore: Remove benchmarking and reloading Benchmarking should go if anything to its own repo reloading is dangerous, and it can trick many new users. --- benchmarking/mptest.py | 156 ------------------------ benchmarking/mptest_results_mac.txt | 117 ------------------ benchmarking/mptest_results_windows.txt | 88 ------------- benchmarking/thread_test.py | 55 --------- qcodes/utils/reload_code.py | 98 --------------- 5 files changed, 514 deletions(-) delete mode 100644 benchmarking/mptest.py delete mode 100644 benchmarking/mptest_results_mac.txt delete mode 100644 benchmarking/mptest_results_windows.txt delete mode 100644 benchmarking/thread_test.py delete mode 100644 qcodes/utils/reload_code.py diff --git a/benchmarking/mptest.py b/benchmarking/mptest.py deleted file mode 100644 index 4072ef92169..00000000000 --- a/benchmarking/mptest.py +++ /dev/null @@ -1,156 +0,0 @@ -# stress test multiprocessing -# run with: python mptest.py -# proc_count: the number of processes to spin up and load qcodes -# period: milliseconds to wait between calling each process -# repetitions: how many times to call each one -# start_method: multiprocessing method to use (fork, forkserver, spawn) - -import multiprocessing as mp -import sys -import os -import psutil -import time - -import qcodes as qc - - -timer = time.perf_counter - - -def print_perf(): - print(time.perf_counter()) - - -def mp_test(name, qin, qout, qglobal): - ''' - simple test that keeps a process running until asked to stop, - and looks for an attribute within qcodes just to ensure it has loaded it - ''' - delays = [] - first = True - while True: - item, qtime = qin.get() - if first: # ignore the first one... process is still starting - first = False - else: - delays.append(timer() - qtime) - if item == 'break': - qglobal.put({ - 'name': name, - 'avg': sum(delays) / len(delays), - 'max': max(delays) - }) - break - qout.put(repr(getattr(qc, item))) - - -def get_memory(pid): - mi = psutil.Process(pid).memory_info() - return mi.rss, mi.vms - - -def get_all_memory(processes, title): - main_memory = get_memory(os.getpid()) - proc_memory = [0, 0] - for proc in processes: - for i, v in enumerate(get_memory(proc.pid)): - proc_memory[i] += v - # print(v) - - return { - 'main_physical': main_memory[0]/1e6, - 'main_virtual': main_memory[1]/1e6, - 'proc_physical': proc_memory[0]/1e6, - 'proc_virtual': proc_memory[1]/1e6, - 'title': title - } - - -def format_memory(mem): - return ('{title}\n' - ' main: {main_physical:.0f} MB phys, ' - '{main_virtual:.0f} MB virt\n' - ' procs: {proc_physical:.0f} MB phys, ' - '{proc_virtual:.0f} MB virt\n' - '').format(**mem) - - -if __name__ == '__main__': - proc_count = int(sys.argv[-4]) - period = float(sys.argv[-3]) - reps = int(sys.argv[-2]) - method = sys.argv[-1] - mp.set_start_method(method) - - qglobal = mp.Queue() - - mem = [get_all_memory([], 'on startup')] - - queues = [] - processes = [] - resp_delays = [] - - t_before_start = timer() - - for proc_num in range(proc_count): - qin = mp.Queue() - qout = mp.Queue() - queues.append((qin, qout)) - p = mp.Process(target=mp_test, - args=('p{}'.format(proc_num), qin, qout, qglobal)) - processes.append(p) - p.start() - - start_delay = (timer() - t_before_start) * 1000 - - mem.append(get_all_memory(processes, 'procs started')) - - for i in range(reps): - for qin, qout in queues: - t1 = timer() - qin.put(('Loop', timer())) - loop_repr = qout.get() - if i: - # ignore the first one, process is still starting - resp_delays.append((timer() - t1) * 1000) - if(loop_repr != repr(qc.Loop)): - raise RuntimeError('{} != {}'.format(loop_repr, repr(qc.Loop))) - print('.', end='', flush=True) - time.sleep(period / 1000) - print('') - - mem.append(get_all_memory(processes, 'procs done working')) - - for qin, qout in queues: - qin.put(('break', timer())) - - t_before_join = timer() - for proc in processes: - proc.join() - join_delay = (timer() - t_before_join) * 1000 - - delays = [qglobal.get() for proc in processes] - avg_delay = sum([d['avg'] for d in delays]) * 1000 / len(delays) - max_delay = max([d['max'] for d in delays]) * 1000 - - avg_resp_delay = sum(resp_delays) / len(resp_delays) - max_resp_delay = max(resp_delays) - - print(('Ran {} procs using "{}" method\n' - 'sent messages every {} milliseconds, {} times\n' - ).format(proc_count, method, period, reps)) - - print('Milliseconds to start all processes: {:.3f}'.format(start_delay)) - print('Final join delay: {:.3f}\n'.format(join_delay)) - - print('Milliseconds to receive to queue request') - print(' avg: {:.6f}'.format(avg_delay)) - print(' max: {:.6f}\n'.format(max_delay)) - - print('Milliseconds to respond to queue request') - print(' avg: {:.6f}'.format(avg_resp_delay)) - print(' max: {:.6f}\n'.format(max_resp_delay)) - - # report on the memory results - for m in mem: - print(format_memory(m)) diff --git a/benchmarking/mptest_results_mac.txt b/benchmarking/mptest_results_mac.txt deleted file mode 100644 index ec8e2a7765d..00000000000 --- a/benchmarking/mptest_results_mac.txt +++ /dev/null @@ -1,117 +0,0 @@ -Mac OS X -------------------- - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 30.601 -Final join delay: 130.124 - -Milliseconds to receive to queue request - avg: 0.321102 - max: 1.061535 - -Milliseconds to respond to queue request - avg: 0.607558 - max: 1.465819 - -on startup - main: 29 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 8 MB phys, 12327 MB virt - -procs done working - main: 29 MB phys, 2524 MB virt - procs: 142 MB phys, 12509 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 512.364 -Final join delay: 456.155 - -Milliseconds to receive to queue request - avg: 0.302726 - max: 2.754109 - -Milliseconds to respond to queue request - avg: 0.555251 - max: 1.123662 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2498 MB virt - procs: 150 MB phys, 49561 MB virt - -procs done working - main: 29 MB phys, 2603 MB virt - procs: 571 MB phys, 50029 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 5242.341 -Final join delay: 3806.779 - -Milliseconds to receive to queue request - avg: 0.527770 - max: 59.421732 - -Milliseconds to respond to queue request - avg: 0.416881 - max: 4.560305 - -on startup - main: 28 MB phys, 2497 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 30 MB phys, 2499 MB virt - procs: 1234 MB phys, 248003 MB virt - -procs done working - main: 11 MB phys, 3026 MB virt - procs: 2144 MB phys, 250162 MB virt - - -Alexs-Air-2:Qcodes alex$ p3 mptest.py 100 50 50 fork -.................................................. -Ran 100 procs using "fork" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 447.375 -Final join delay: 113.426 - -Milliseconds to receive to queue request - avg: 0.351694 - max: 11.889809 - -Milliseconds to respond to queue request - avg: 0.494252 - max: 2.525026 - -on startup - main: 28 MB phys, 2498 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 29 MB phys, 2499 MB virt - procs: 444 MB phys, 249616 MB virt - -procs done working - main: 32 MB phys, 3025 MB virt - procs: 525 MB phys, 250142 MB virt diff --git a/benchmarking/mptest_results_windows.txt b/benchmarking/mptest_results_windows.txt deleted file mode 100644 index e267edeb280..00000000000 --- a/benchmarking/mptest_results_windows.txt +++ /dev/null @@ -1,88 +0,0 @@ -Windows -------------------- - -c:\Qcodes>python mptest.py 5 50 50 spawn -.................................................. -Ran 5 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 15.406 -Final join delay: 92.359 - -Milliseconds to receive to queue request - avg: -510.603209 - max: -509.684951 - -Milliseconds to respond to queue request - avg: 0.360488 - max: 0.659211 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 14 MB phys, 9 MB virt - -procs done working - main: 36 MB phys, 32 MB virt - procs: 179 MB phys, 155 MB virt - - -c:\Qcodes>python mptest.py 20 50 50 spawn -.................................................. -Ran 20 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 126.134 -Final join delay: 228.260 - -Milliseconds to receive to queue request - avg: -1591.011659 - max: -1563.003352 - -Milliseconds to respond to queue request - avg: 0.365974 - max: 0.794690 - -on startup - main: 36 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 36 MB phys, 31 MB virt - procs: 98 MB phys, 60 MB virt - -procs done working - main: 37 MB phys, 32 MB virt - procs: 716 MB phys, 620 MB virt - - -c:\Qcodes>python mptest.py 100 50 50 spawn -.................................................. -Ran 100 procs using "spawn" method -sent messages every 50.0 milliseconds, 50 times - -Milliseconds to start all processes: 761.556 -Final join delay: 927.847 - -Milliseconds to receive to queue request - avg: -8560.453780 - max: -8470.848083 - -Milliseconds to respond to queue request - avg: 0.423033 - max: 1.299948 - -on startup - main: 35 MB phys, 31 MB virt - procs: 0 MB phys, 0 MB virt - -procs started - main: 37 MB phys, 32 MB virt - procs: 872 MB phys, 572 MB virt - -procs done working - main: 40 MB phys, 36 MB virt - procs: 3575 MB phys, 3094 MB virt diff --git a/benchmarking/thread_test.py b/benchmarking/thread_test.py deleted file mode 100644 index 40dac28bc69..00000000000 --- a/benchmarking/thread_test.py +++ /dev/null @@ -1,55 +0,0 @@ -import threading -import time - - -def sleeper(t, n, out, make_error): - time.sleep(t) - out[n] = n - if make_error: - raise RuntimeError('hello from # {}!'.format(n)) - - -def runmany(n, t, error_nums): - out = [None] * n - - t0 = time.perf_counter() - - threads = [ - CatchingThread(target=sleeper, args=(t, i, out, i in error_nums)) - for i in range(n)] - - # start threads backward - [t.start() for t in reversed(threads)] - [t.join() for t in threads] - - t1 = time.perf_counter() - - out_ok = [] - for i in range(n): - if out[i] != i: - out_ok += ['ERROR! out[{}] = {}'.format(i, out[i])] - - if not out_ok: - out_ok += ['all output correct'] - - print('{} parallel threads sleeping\n'.format(n) + - 'given time: {}\n'.format(t) + - 'resulting time: {}\n'.format(t1 - t0) + - '\n'.join(out_ok)) - - -class CatchingThread(threading.Thread): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.exception = None - - def run(self): - try: - super().run() - except Exception as e: - self.exception = e - - def join(self): - super().join() - if self.exception: - raise self.exception diff --git a/qcodes/utils/reload_code.py b/qcodes/utils/reload_code.py deleted file mode 100644 index 08cedf5d1dd..00000000000 --- a/qcodes/utils/reload_code.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import sys -import imp -from traceback import format_exc - -# Routine to reload modules during execution, for development only. -# This is finicky and SHOULD NOT be used in regular experiments, -# as they can cause non-intuitive errors later on. It is not included in -# the base qcodes import, nor tested; Use at your own risk. - - -# see http://stackoverflow.com/questions/22195382/ -# how-to-check-if-a-module-library-package-is-part-of-the-python-standard-library -syspaths = [os.path.abspath(p) for p in sys.path] -stdlib = tuple(p for p in syspaths - if p.startswith((sys.prefix, sys.base_prefix)) - and 'site-packages' not in p) -# a few things in site-packages we will consider part of the standard lib -# it causes problems if we reload some of these, others are just stable -# dependencies - this is mainly for reloading our own code. -# could even whitelist site-packages items to allow, rather than to ignore? -otherlib = ('jupyter', 'ipy', 'IPy', 'matplotlib', 'numpy', 'scipy', 'pyvisa', - 'traitlets', 'zmq', 'tornado', 'dateutil', 'six', 'pexpect') -otherpattern = tuple('site-packages/' + n for n in otherlib) - - -def reload_code(pattern=None, lib=False, site=False): - ''' - reload all modules matching a given pattern - or all (non-built-in) modules if pattern is omitted - if lib is False (default), ignore the standard library and major packages - if site is False (default), ignore everything in site-packages, only reload - files in nonstandard paths - ''' - reloaded_files = [] - - for i in range(2): - # sometimes we need to reload twice to propagate all links, - # even though we reload the deepest modules first. Not sure if - # twice is always sufficient, but we'll try it. - for module in sys.modules.values(): - if (pattern is None or pattern in module.__name__): - reload_recurse(module, reloaded_files, lib, site) - - return reloaded_files - - -def is_good_module(module, lib=False, site=False): - ''' - is an object (module) a module we can reload? - if lib is False (default), ignore the standard library and major packages - ''' - # take out non-modules and underscore modules - name = getattr(module, '__name__', '_') - if name[0] == '_' or not isinstance(module, type(sys)): - return False - - # take out modules we can't find and built-ins - if name in sys.builtin_module_names or not hasattr(module, '__file__'): - return False - - path = os.path.abspath(module.__file__) - - if 'site-packages' in path and not site: - return False - - if not lib: - if path.startswith(stdlib) and 'site-packages' not in path: - return False - - for pattern in otherpattern: - if pattern in path: - return False - - return True - - -def reload_recurse(module, reloaded_files, lib, site): - ''' - recursively search module for its own dependencies to reload, - ignoring those already in reloaded_files - if lib is False (default), ignore the standard library and major packages - ''' - if (not is_good_module(module, lib, site) or - module.__file__ in reloaded_files): - return - - reloaded_files.append(module.__file__) - - try: - for name in dir(module): - module2 = getattr(module, name) - reload_recurse(module2, reloaded_files, lib, site) - imp.reload(module) - - except: - print('error reloading "{}"'.format(getattr(module, '__name__', '?'))) - print(format_exc()) From ee5375f59ed7f92ab3458357dabe38fa00dd9e4d Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 12:00:50 +0100 Subject: [PATCH 13/36] chore: Add todo --- qcodes/tests/test_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qcodes/tests/test_helpers.py b/qcodes/tests/test_helpers.py index 483792db060..f55827d0c9c 100644 --- a/qcodes/tests/test_helpers.py +++ b/qcodes/tests/test_helpers.py @@ -167,6 +167,10 @@ def test_bad_calls(self): permissive_range(*args) def test_good_calls(self): + # TODO(giulioungaretti) + # not sure what we are testing here. + # in pyhton 1.0 and 1 are actually the same + # https://docs.python.org/3.5/library/functions.html#hash good_args = { (1, 7, 2): [1, 3, 5], (1, 7, 4): [1, 5], From 44138c59167553864265b5e28804413dd324c47f Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 12:36:47 +0100 Subject: [PATCH 14/36] chore: Use py.test move faster without reinventing the wheel at all times --- CONTRIBUTING.rst | 113 ++++------------------------------- README.rst | 4 +- docs/community/index.rst | 1 - docs/community/testing.rst | 80 ------------------------- qcodes/__init__.py | 1 - qcodes/test.py | 119 ------------------------------------- 6 files changed, 15 insertions(+), 303 deletions(-) delete mode 100644 docs/community/testing.rst delete mode 100644 qcodes/test.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 63aef9e7e31..abaa46f3066 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -59,13 +59,18 @@ Setup Running Tests ~~~~~~~~~~~~~ -The core test runner is in ``qcodes/test.py``: +We don't want to reinvent the wheel, and thus use py.test. +It's easy to install: :: - python qcodes/test.py - # optional extra verbosity and fail fast - python qcodes/test.py -v -f + pip install coverage pytest-cov pytest + +Then to test and view the coverage: + +:: + py.test --cov=qcodes --cov-report xml --cov-config=.coveragerc + You can also run single tests with: @@ -78,92 +83,6 @@ You can also run single tests with: # or python -m unittest qcodes.tests.test_metadata.TestMetadatable.test_snapshot -If you run the core test runner, you should see output that looks -something like this: - -:: - - .........***** found one MockMock, testing ***** - ............................................Timing resolution: - startup time: 0.000e+00 - min/med/avg/max dev: 9.260e-07, 9.670e-07, 1.158e-06, 2.109e-03 - async sleep delays: - startup time: 2.069e-04 - min/med/avg/max dev: 3.372e-04, 6.376e-04, 6.337e-04, 1.007e-03 - multiprocessing startup delay and regular sleep delays: - startup time: 1.636e-02 - min/med/avg/max dev: 3.063e-05, 2.300e-04, 2.232e-04, 1.743e-03 - should go to stdout;should go to stderr;.stdout stderr stdout stderr ..[10:44:09.063 A Queue] should get printed - ................................... - ---------------------------------------------------------------------- - Ran 91 tests in 4.192s - - OK - Name Stmts Miss Cover Missing - ---------------------------------------------------------- - data/data_array.py 104 0 100% - data/data_set.py 179 140 22% 38-55, 79-94, 99-104, 123-135, 186-212, 215-221, 224-244, 251-254, 257-264, 272, 280-285, 300-333, 347-353, 360-384, 395-399, 405-407, 414-420, 426-427, 430, 433-438 - data/format.py 225 190 16% 44-55, 61-62, 70, 78-97, 100, 114-148, 157-188, 232, 238, 246, 258-349, 352, 355-358, 361-368, 375-424, 427-441, 444, 447-451 - data/io.py 76 50 34% 71-84, 90-91, 94, 97, 103, 109-110, 119-148, 154-161, 166, 169, 172, 175-179, 182, 185-186 - data/manager.py 124 89 28% 15-20, 31, 34, 48-62, 65-67, 70, 76-77, 80-84, 90-102, 108-110, 117-121, 142-151, 154-182, 185, 188, 207-208, 215-221, 227-229, 237, 243, 249 - instrument/base.py 74 0 100% - instrument/function.py 45 1 98% 77 - instrument/ip.py 20 12 40% 10-16, 19-20, 24-25, 29-38 - instrument/mock.py 63 0 100% - instrument/parameter.py 200 2 99% 467, 470 - instrument/sweep_values.py 107 33 69% 196-207, 220-227, 238-252, 255-277 - instrument/visa.py 36 24 33% 10-25, 28-32, 35-36, 40-41, 47-48, 57-58, 62-64, 68 - loops.py 285 239 16% 65-74, 81-91, 120-122, 133-141, 153-165, 172-173, 188-207, 216-240, 243-313, 316-321, 324-350, 354-362, 371-375, 378-381, 414-454, 457-474, 477-484, 487-491, 510-534, 537-543, 559-561, 564, 577, 580, 590-608, 611-618, 627-628, 631 - station.py 35 24 31% 17-32, 35, 45-50, 60, 67-82, 88 - utils/helpers.py 95 0 100% - utils/metadata.py 13 0 100% - utils/multiprocessing.py 95 2 98% 125, 134 - utils/sync_async.py 114 8 93% 166, 171-173, 176, 180, 184, 189-191 - utils/timing.py 72 0 100% - utils/validators.py 110 0 100% - ---------------------------------------------------------- - TOTAL 2072 814 61% - -The key is ``OK`` in the middle (that means all the tests passed), and -the presence of the coverage report after it. If any tests fail, we do -not show a coverage report, and the end of the output will contain -tracebacks and messages about what failed, for example: - -:: - - ====================================================================== - FAIL: test_sweep_steps_edge_case (tests.test_instrument.TestParameters) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "/Users/alex/qdev/Qcodes/qcodes/tests/test_instrument.py", line 360, in test_sweep_steps_edge_case - self.check_set_amplitude2('Off', log_count=1, history_count=2) - File "/Users/alex/qdev/Qcodes/qcodes/tests/test_instrument.py", line 345, in check_set_amplitude2 - self.assertTrue(line.startswith('negative delay'), line) - AssertionError: False is not true : cannot sweep amplitude2 from 0.1 to Off - jumping. - - ---------------------------------------------------------------------- - Ran 91 tests in 4.177s - - FAILED (failures=1) - -The coverage report is only useful if you have been adding new code, to -see whether your tests visit all of your code. Look at the file(s) you -have been working on, and ensure that the "missing" section does not -contain the line numbers of any of the blocks you have touched. -Currently the core still has a good deal of untested code - eventually -we will have all of this tested, but for now you can ignore all the rest -of the missing coverage. - -You can also run these tests from inside python. The output is similar -except that a) you don't get coverage reporting, and b) one test has to -be skipped because it does not apply within a notebook, so the output -will end ``OK (skipped=1)``: - -.. code:: python - - import qcodes - qcodes.test_core() # optional verbosity = 1 (default) or 2 - If the tests pass, you should be ready to start developing! To tests actual instruments, first instantiate them in an interactive @@ -315,13 +234,7 @@ simplify to a one command that says: if there's enough cover, and all good or fail and where it fails. - The standard test commands are listed above under - :ref:`runnningtests`. More notes on different test runners can - be found in :ref:`testing`. - -- Core tests live in - `qcodes/tests `__ - and instrument tests live in the same directories as the instrument - drivers. + :ref:`runnningtests` . - We should have a *few* high-level "integration" tests, but simple unit tests (that just depend on code in one module) are more valuable @@ -330,7 +243,6 @@ good or fail and where it fails. - When features change it is likely that more tests will need to change - Unit tests can cover many scenarios much faster than integration tests. - - If you're having difficulty making unit tests, first consider whether your code could be restructured to make it less dependent on other modules. Often, however, extra techniques are needed to break down a @@ -339,9 +251,8 @@ good or fail and where it fails. - Patching, one of the most useful parts of the `unittest.mock `__ library. This lets you specify exactly how other functions/objects - should behave when they're called by the code you are testing. For a - simple example, see - `test\_multiprocessing.py `__ + should behave when they're called by the code you are testing. + - Supporting files / data: Lets say you have a test of data acquisition and analysis. You can break that up into an acquisition test and an analysis by saving the intermediate state, namely the data file, in diff --git a/README.rst b/README.rst index 423dfab85e8..c17387b7d92 100644 --- a/README.rst +++ b/README.rst @@ -70,9 +70,11 @@ $QCODES_INSTALL_DIR is the folder where you want to have the source code. cd $QCODES_INSTALL_DIR pyenv install 3.5.2 pyenv virtualenv 3.5.2 qcodes-dev + pyenv activate qcodes-dev pip install -r requirements.txt + pip install coverage pytest-cov pytest --upgrade pip install -e . - python qcodes/test.py -f + py.test --cov=qcodes --cov-config=.coveragerc If the tests pass you are ready to hack! This is the reference setup one needs to have to contribute, otherwise diff --git a/docs/community/index.rst b/docs/community/index.rst index 0d42653900d..c0cd05f6962 100644 --- a/docs/community/index.rst +++ b/docs/community/index.rst @@ -11,4 +11,3 @@ This is the guide for you, the developer that wants to maintain/expand QCoDes. install contributing objects - testing diff --git a/docs/community/testing.rst b/docs/community/testing.rst deleted file mode 100644 index dc0ef7d2210..00000000000 --- a/docs/community/testing.rst +++ /dev/null @@ -1,80 +0,0 @@ -.. _testing: - - -Notes on test runners compatible with Qcodes -============================================ - -There is now a test script `test.py `__ in the qcodes -directory that uses the standard ``unittest`` machinery to run all the -core tests (does not include instrument drivers). It has been tested on -Mac (terminal), and Windows (cmd, git bash, and PowerShell). It includes -coverage testing, but will only print a coverage report if tests pass. - -The biggest difficulty with testing Qcodes is windows multiprocessing. -The spawn method restricts execution in ways that are annoying for -regular users (no class/function definitions in the notebook, no -closures) but seem to be completely incompatible with some test runners -(and/or coverage tracking) - -I considered the following test runners: - **nose**: works well, but it -has a `note on its homepage `__ -that it is no longer being actively maintained (in favor of nose2 -development), so we should not use it long-term. - -- **unittest**: the standard, built-in python tester. The only thing we - really need to add to this is coverage testing, so now the question - is what's the easiest way to do this? On Windows just using unittest - wrapped in coverage fails. - -- **nose2**: has a broken coverage plugin - it reports all the - unindented lines, ie everything # that executes on import, as - uncovered - but can be used by wrapping it inside coverage instead, - just like unittest. - -- **py.test**: seems to add lots of features, but it's not clear they - are useful for us? Has a good coverage plugin but seems to require - tons of command-line options. Requires both ``pytest`` and - ``pytest-cov`` packages - -on Mac terminal: - -:: - - # the following work with coverage: - nosetests - python setup.py nosetests - py.test --cov-config .coveragerc --cov qcodes --cov-report term-missing - coverage run -m nose2 && coverage report -m - # both of these run unittest: - coverage run setup.py test && coverage report -m - coverage run -m unittest && coverage report -m - - # nose2's coverage plugin is broken - it reports all the unindented lines (everything - # that executes on import) as uncovered - nose2 -C --coverage-report term-missing - -Windows cmd shell and git bash behave identically, PowerShell has -different chain syntax (commands with &&): - -:: - - # the following work with coverage: - nosetests - py.test --cov-config .coveragerc --cov qcodes --cov-report term-missing - coverage run -m nose2 && coverage report -m # cmd or bash - (coverage run -m nose2) -and (coverage report -m) # PowerShell - - # the following work without coverage: - python -m unittest discover - python -m unittest # discover is unnecessary now, perhaps because I put test_suite in setup.py? - - # the following do not work: - - # fails on relative import in unittest inside separate process (why is it importing that anyway?) - coverage run -m unittest discover && coverage report -m # cmd or bash - (coverage run -m unittest discover) -and (coverage report -m) # PowerShell - # these fail asking for freeze_support() but nothing I do with that seems to help - python setup.py test - coverage run setup.py test && coverage report -m # cmd or bash - (coverage run setup.py test) -and (coverage report -m) # PowerShell - python setup.py nosetests diff --git a/qcodes/__init__.py b/qcodes/__init__.py index 71bc19c9400..b7d7676af23 100644 --- a/qcodes/__init__.py +++ b/qcodes/__init__.py @@ -60,4 +60,3 @@ from qcodes.utils import validators from qcodes.instrument_drivers.test import test_instruments, test_instrument -from qcodes.test import test_core, test_part diff --git a/qcodes/test.py b/qcodes/test.py deleted file mode 100644 index b04c412b06a..00000000000 --- a/qcodes/test.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Unified qcodes test runners.""" - -import sys - - -def test_core(verbosity=1, failfast=False): - """ - Run the qcodes core tests. - - Args: - verbosity (int, optional): 0, 1, or 2, higher displays more info - Default 1. - failfast (bool, optional): If true, stops running on first failure - Default False. - - Coverage testing is only available from the command line - """ - import qcodes - if qcodes.in_notebook(): - qcodes._IN_NOTEBOOK = True - - _test_core(verbosity=verbosity, failfast=failfast) - - -def _test_core(test_pattern='test*.py', **kwargs): - import unittest - - import qcodes.tests as qctest - import qcodes - - suite = unittest.defaultTestLoader.discover( - qctest.__path__[0], top_level_dir=qcodes.__path__[0], - pattern=test_pattern) - if suite.countTestCases() == 0: - print('found no tests') - sys.exit(1) - print('testing %d cases' % suite.countTestCases()) - - result = unittest.TextTestRunner(**kwargs).run(suite) - return result.wasSuccessful() - - -def test_part(name): - """ - Run part of the qcodes core test suite. - - Args: - name (str): a name within the qcodes.tests directory. May be: - - a module ('test_loop') - - a TestCase ('test_loop.TestLoop') - - a test method ('test_loop.TestLoop.test_nesting') - """ - import unittest - fullname = 'qcodes.tests.' + name - suite = unittest.defaultTestLoader.loadTestsFromName(fullname) - return unittest.TextTestRunner().run(suite).wasSuccessful() - -if __name__ == '__main__': - import argparse - import os - import multiprocessing as mp - - try: - import coverage - coverage_missing = False - except ImportError: - coverage_missing = True - - # make sure coverage looks for .coveragerc in the right place - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - parser = argparse.ArgumentParser( - description=('Core test suite for Qcodes, ' - 'covering everything except instrument drivers')) - - parser.add_argument('-v', '--verbose', action='store_true', - help='increase verbosity') - - parser.add_argument('-q', '--quiet', action='store_true', - help='reduce verbosity (opposite of --verbose)') - - parser.add_argument('-s', '--skip-coverage', action='store_true', - help='skip coverage reporting') - - parser.add_argument('-t', '--test_pattern', type=str, default='test*.py', - help=('regexp for test name to match, ' - 'default "test*.py"')) - - parser.add_argument('-f', '--failfast', action='store_true', - help='halt on first error/failure') - - parser.add_argument('-m', '--mp-spawn', action='store_true', - help=('force "spawn" method of starting child ' - 'processes to emulate Win behavior on Unix')) - - args = parser.parse_args() - - if args.mp_spawn: - mp.set_start_method('spawn') - - args.skip_coverage |= coverage_missing - - if not args.skip_coverage: - cov = coverage.Coverage(source=['qcodes']) - cov.start() - - success = _test_core(verbosity=(1 + args.verbose - args.quiet), - failfast=args.failfast, - test_pattern=args.test_pattern) - - if not args.skip_coverage: - cov.stop() - cov.save() - cov.report() - - # restore unix-y behavior - # exit status 1 on fail - if not success: - sys.exit(1) From 480933e64822b2c1ada70884f29e027c73fe4737 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Tue, 7 Mar 2017 16:33:24 +0100 Subject: [PATCH 15/36] fix: Remove nested attrs as unused --- qcodes/utils/nested_attrs.py | 175 ----------------------------------- 1 file changed, 175 deletions(-) delete mode 100644 qcodes/utils/nested_attrs.py diff --git a/qcodes/utils/nested_attrs.py b/qcodes/utils/nested_attrs.py deleted file mode 100644 index 66c9063fa3f..00000000000 --- a/qcodes/utils/nested_attrs.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Nested attribute / item access for use by remote proxies.""" - -import re - - -class _NoDefault: - - """Empty class to provide a missing default to getattr.""" - - -class NestedAttrAccess: - - """ - A Mixin class to provide nested access to attributes and their items. - - Primarily for use by remote proxies, so we don't need to separately - proxy all the components, and all of their components, and worry about - which are picklable, etc. - """ - - def getattr(self, attr, default=_NoDefault): - """ - Get a (possibly nested) attribute of this object. - - If there is no ``.`` or ``[]`` in ``attr``, this exactly matches - the ``getattr`` function, but you can also access smaller pieces. - - Args: - attr (str): An attribute or accessor string, like: - ``'attr.subattr[item]'``. ``item`` can be an integer or a - string. If it's a string it must be quoted. - default (any): If the attribute does not exist (at any level of - nesting), we return this. If no default is provided, throws - an ``AttributeError``. - - Returns: - The value of this attribute. - - Raises: - ValueError: If ``attr`` could not be understood. - AttributeError: If the attribute is missing and no default is - provided. - KeyError: If the item cannot be found and no default is provided. - """ - parts = self._split_attr(attr) - - try: - return self._follow_parts(parts) - - except (AttributeError, KeyError): - if default is _NoDefault: - raise - else: - return default - - def setattr(self, attr, value): - """ - Set a (possibly nested) attribute of this object. - - If there is no ``.`` or ``[]`` in ``attr``, this exactly matches - the ``setattr`` function, but you can also access smaller pieces. - - Args: - attr (str): An attribute or accessor string, like: - ``'attr.subattr[item]'``. ``item`` can be an integer or a - string; If it's a string it must be quoted as usual. - - value (any): The object to store in this attribute. - - Raises: - ValueError: If ``attr`` could not be understood - - TypeError: If an intermediate nesting level is not a container - and the next level is an item. - - AttributeError: If an attribute with this name cannot be set. - """ - parts = self._split_attr(attr) - obj = self._follow_parts(parts[:-1]) - leaf = parts[-1] - - if str(leaf).startswith('.'): - setattr(obj, leaf[1:], value) - else: - obj[leaf] = value - - def delattr(self, attr): - """ - Delete a (possibly nested) attribute of this object. - - If there is no ``.`` or ``[]`` in ``attr``, this exactly matches - the ``delattr`` function, but you can also access smaller pieces. - - Args: - attr (str): An attribute or accessor string, like: - ``'attr.subattr[item]'``. ``item`` can be an integer or a - string; If it's a string it must be quoted as usual. - - Raises: - ValueError: If ``attr`` could not be understood - """ - parts = self._split_attr(attr) - obj = self._follow_parts(parts[:-1]) - leaf = parts[-1] - - if str(leaf).startswith('.'): - delattr(obj, leaf[1:]) - else: - del obj[leaf] - - def callattr(self, attr, *args, **kwargs): - """ - Call a (possibly nested) method of this object. - - Args: - attr (str): An attribute or accessor string, like: - ``'attr.subattr[item]'``. ``item`` can be an integer or a - string; If it's a string it must be quoted as usual. - - *args: Passed on to the method. - - **kwargs: Passed on to the method. - - Returns: - any: Whatever the method returns. - - Raises: - ValueError: If ``attr`` could not be understood - """ - func = self.getattr(attr) - return func(*args, **kwargs) - - _PARTS_RE = re.compile(r'([\.\[])') - _ITEM_RE = re.compile(r'\[(?P[^\[\]]+)\]') - _QUOTED_RE = re.compile(r'(?P[\'"])(?P[^\'"]*)(?P=q)') - - def _split_attr(self, attr): - """ - Return attr as a list of parts. - - Items in the list are: - str '.attr' for attribute access, - str 'item' for string dict keys, - integers for integer dict/sequence keys. - Other key formats are not supported - """ - # the first item is implicitly an attribute - parts = ('.' + self._PARTS_RE.sub(r'~\1', attr)).split('~') - for i, part in enumerate(parts): - item_match = self._ITEM_RE.fullmatch(part) - if item_match: - item = item_match.group('item') - quoted_match = self._QUOTED_RE.fullmatch(item) - if quoted_match: - parts[i] = quoted_match.group('str') - else: - try: - parts[i] = int(item) - except ValueError: - raise ValueError('unrecognized item: ' + item) - elif part[0] != '.' or len(part) < 2: - raise ValueError('unrecognized attribute part: ' + part) - - return parts - - def _follow_parts(self, parts): - obj = self - - for key in parts: - if str(key).startswith('.'): - obj = getattr(obj, key[1:]) - else: - obj = obj[key] - - return obj From 783e0f97c5f13ab7c13f6f706bfef5c5c88fdfe3 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Wed, 8 Mar 2017 17:45:58 +0100 Subject: [PATCH 16/36] Fix remove try/Interrupt, missing variable imax --- qcodes/loops.py | 136 ++++++++++++++++++-------------------- qcodes/tests/test_loop.py | 6 +- 2 files changed, 67 insertions(+), 75 deletions(-) diff --git a/qcodes/loops.py b/qcodes/loops.py index f00a35b6aa6..5f1a3c6ee78 100644 --- a/qcodes/loops.py +++ b/qcodes/loops.py @@ -777,74 +777,70 @@ def _run_loop(self, first_delay=0, action_indices=(), self.last_task_failed = False - try: - for i, value in enumerate(self.sweep_values): - if self.progress_interval is not None: - tprint('loop %s: %d/%d (%.1f [s])' % ( - self.sweep_values.name, i, imax, time.time() - t0), - dt=self.progress_interval, tag='outerloop') - - set_val = self.sweep_values.set(value) - - new_indices = loop_indices + (i,) - new_values = current_values + (value,) - data_to_store = {} - - if hasattr(self.sweep_values, "parameters"): - set_name = self.data_set.action_id_map[action_indices] - if hasattr(self.sweep_values, 'aggregate'): - value = self.sweep_values.aggregate(*set_val) - self.data_set.store(new_indices, {set_name: value}) - for j, val in enumerate(set_val): - set_index = action_indices + (j+1, ) - set_name = (self.data_set.action_id_map[set_index]) - data_to_store[set_name] = val - else: - set_name = self.data_set.action_id_map[action_indices] - data_to_store[set_name] = value - - self.data_set.store(new_indices, data_to_store) - - if not self._nest_first: - # only wait the delay time if an inner loop will not inherit it - self._wait(delay) - - try: - for f in callables: - f(first_delay=delay, - loop_indices=new_indices, - current_values=new_values) - - # after the first action, no delay is inherited - delay = 0 - except _QcodesBreak: - break - - # after the first setpoint, delay reverts to the loop delay - delay = self.delay - - # now check for a background task and execute it if it's - # been long enough since the last time - # don't let exceptions in the background task interrupt - # the loop - # if the background task fails twice consecutively, stop - # executing it - if self.bg_task is not None: - t = time.time() - if t - last_task >= self.bg_min_delay: - try: - self.bg_task() - except Exception: - if self.last_task_failed: - self.bg_task = None - self.last_task_failed = True - log.exception("Failed to execute bg task") - - last_task = t - - except Interrupt: - log.debug("Stopping loop cleanly") - return + for i, value in enumerate(self.sweep_values): + if self.progress_interval is not None: + tprint('loop %s: %d/%d (%.1f [s])' % ( + self.sweep_values.name, i, imax, time.time() - t0), + dt=self.progress_interval, tag='outerloop') + + set_val = self.sweep_values.set(value) + + new_indices = loop_indices + (i,) + new_values = current_values + (value,) + data_to_store = {} + + if hasattr(self.sweep_values, "parameters"): + set_name = self.data_set.action_id_map[action_indices] + if hasattr(self.sweep_values, 'aggregate'): + value = self.sweep_values.aggregate(*set_val) + self.data_set.store(new_indices, {set_name: value}) + for j, val in enumerate(set_val): + set_index = action_indices + (j+1, ) + set_name = (self.data_set.action_id_map[set_index]) + data_to_store[set_name] = val + else: + set_name = self.data_set.action_id_map[action_indices] + data_to_store[set_name] = value + + self.data_set.store(new_indices, data_to_store) + + if not self._nest_first: + # only wait the delay time if an inner loop will not inherit it + self._wait(delay) + + try: + for f in callables: + f(first_delay=delay, + loop_indices=new_indices, + current_values=new_values) + + # after the first action, no delay is inherited + delay = 0 + except _QcodesBreak: + break + + # after the first setpoint, delay reverts to the loop delay + delay = self.delay + + # now check for a background task and execute it if it's + # been long enough since the last time + # don't let exceptions in the background task interrupt + # the loop + # if the background task fails twice consecutively, stop + # executing it + if self.bg_task is not None: + t = time.time() + if t - last_task >= self.bg_min_delay: + try: + self.bg_task() + except Exception: + if self.last_task_failed: + self.bg_task = None + self.last_task_failed = True + log.exception("Failed to execute bg task") + + last_task = t + # run the background task one last time to catch the last setpoint(s) if self.bg_task is not None: self.bg_task() @@ -862,7 +858,3 @@ def _wait(self, delay): finish_clock = time.perf_counter() + delay t = wait_secs(finish_clock) time.sleep(t) - - -class Interrupt(Exception): - pass diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index ae76594936a..6409e67e1cf 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -4,8 +4,8 @@ import numpy as np from unittest.mock import patch -from qcodes.loops import Loop, Interrupt -from qcodes.actions import Task, Wait, BreakIf +from qcodes.loops import Loop +from qcodes.actions import Task, Wait, BreakIf, _QcodesBreak from qcodes.station import Station from qcodes.data.data_array import DataArray from qcodes.instrument.parameter import ManualParameter @@ -461,7 +461,7 @@ def __init__(self, *args, count=1, msg=None, **kwargs): def get(self): self._count -= 1 if self._count <= 0: - raise Interrupt + raise _QcodesBreak return super().get() def reset(self): From addd056bf7a5d4c88a9c721129ea2e5e4edacd38 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Fri, 10 Mar 2017 15:12:30 +0100 Subject: [PATCH 17/36] fix: Remove trailing number from test name --- qcodes/tests/test_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcodes/tests/test_instrument.py b/qcodes/tests/test_instrument.py index d4a94c92832..d02ccdcc254 100644 --- a/qcodes/tests/test_instrument.py +++ b/qcodes/tests/test_instrument.py @@ -6,7 +6,7 @@ from .instrument_mocks import DummyInstrument -class TestInstrument2(TestCase): +class TestInstrument(TestCase): def setUp(self): self.instrument = DummyInstrument( From c08ca659d7c9b4eba9bea6cbb399fb60a9eb5212 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Fri, 10 Mar 2017 15:14:46 +0100 Subject: [PATCH 18/36] fix: Use unit --- qcodes/tests/instrument_mocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/instrument_mocks.py b/qcodes/tests/instrument_mocks.py index 59c264d8b08..4de3c460dad 100644 --- a/qcodes/tests/instrument_mocks.py +++ b/qcodes/tests/instrument_mocks.py @@ -102,7 +102,8 @@ def __init__(self, name='dummy', gates=['dac1', 'dac2', 'dac3'], **kwargs): self.add_parameter(g, parameter_class=ManualParameter, initial_value=0, - label='Gate {} (arb. units)'.format(g), + label='Gate {}'.format(g), + unit="V", vals=Numbers(-800, 400)) From c7b4ac7b5be5f1b8d65d61a8647ca5b59a0cd306 Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Fri, 10 Mar 2017 15:15:30 +0100 Subject: [PATCH 19/36] chore: Update documentation --- CONTRIBUTING.rst | 3 --- docs/api/private.rst | 3 --- docs/api/public.rst | 9 --------- 3 files changed, 15 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index abaa46f3066..0657ab003c0 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -233,9 +233,6 @@ and then unit testing should be run on pull-request, using CI. Maybe simplify to a one command that says: if there's enough cover, and all good or fail and where it fails. -- The standard test commands are listed above under - :ref:`runnningtests` . - - We should have a *few* high-level "integration" tests, but simple unit tests (that just depend on code in one module) are more valuable for several reasons: diff --git a/docs/api/private.rst b/docs/api/private.rst index 8fffb7cb2fa..f5e38caebc9 100644 --- a/docs/api/private.rst +++ b/docs/api/private.rst @@ -12,8 +12,5 @@ Classes and Functions utils.command utils.deferred_operations - utils.nested_attrs utils.helpers - utils.timing utils.metadata - process.qcodes_process diff --git a/docs/api/public.rst b/docs/api/public.rst index 12acc2e9464..04521a6f0d5 100644 --- a/docs/api/public.rst +++ b/docs/api/public.rst @@ -42,8 +42,6 @@ Loops .. autosummary:: :toctree: generated/ - get_bg - halt_bg Loop Measure @@ -70,9 +68,6 @@ Data .. autosummary:: :toctree: generated/ - get_data_manager - qcodes.data.manager.DataManager - DataMode DataSet new_data load_data @@ -102,8 +97,6 @@ Instrument Instrument IPInstrument VisaInstrument - MockInstrument - MockModel Plot @@ -123,7 +116,5 @@ Utils & misc :toctree: generated/ qcodes.utils.validators - qcodes.process.helpers.set_mp_method qcodes.utils.helpers.in_notebook - qcodes.widgets.widgets.show_subprocess_widget From 5f8fb7ea5b95fb2ff026950e51393d7ef8fe357e Mon Sep 17 00:00:00 2001 From: Giulio Ungaretti Date: Fri, 10 Mar 2017 15:27:11 +0100 Subject: [PATCH 20/36] fix: Remove toymodel, and simlify examples --- docs/examples/Measure without a Loop.ipynb | 475 +----- docs/examples/Tutorial.ipynb | 1567 ++------------------ docs/examples/toymodel.py | 149 -- 3 files changed, 161 insertions(+), 2030 deletions(-) delete mode 100644 docs/examples/toymodel.py diff --git a/docs/examples/Measure without a Loop.ipynb b/docs/examples/Measure without a Loop.ipynb index 828db49fce4..f59aeebab11 100644 --- a/docs/examples/Measure without a Loop.ipynb +++ b/docs/examples/Measure without a Loop.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Measure without a Loop\n", "\n", @@ -13,358 +16,38 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/*\n", - " * Qcodes Jupyter/IPython widgets\n", - " */\n", - "require([\n", - " 'nbextensions/widgets/widgets/js/widget',\n", - " 'nbextensions/widgets/widgets/js/manager'\n", - "], function (widget, manager) {\n", - "\n", - " var UpdateView = widget.DOMWidgetView.extend({\n", - " render: function() {\n", - " window.MYWIDGET = this;\n", - " this._interval = 0;\n", - " this.update();\n", - " },\n", - " update: function() {\n", - " this.display(this.model.get('_message'));\n", - " this.setInterval();\n", - " },\n", - " display: function(message) {\n", - " /*\n", - " * display method: override this for custom display logic\n", - " */\n", - " this.el.innerHTML = message;\n", - " },\n", - " remove: function() {\n", - " clearInterval(this._updater);\n", - " },\n", - " setInterval: function(newInterval) {\n", - " var me = this;\n", - " if(newInterval===undefined) newInterval = me.model.get('interval');\n", - " if(newInterval===me._interval) return;\n", - "\n", - " me._interval = newInterval;\n", - "\n", - " if(me._updater) clearInterval(me._updater);\n", - "\n", - " if(me._interval) {\n", - " me._updater = setInterval(function() {\n", - " me.send({myupdate: true});\n", - " if(!me.model.comm_live) {\n", - " console.log('missing comm, canceling widget updates', me);\n", - " clearInterval(me._updater);\n", - " }\n", - " }, me._interval * 1000);\n", - " }\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('UpdateView', UpdateView);\n", - "\n", - " var HiddenUpdateView = UpdateView.extend({\n", - " display: function(message) {\n", - " this.$el.hide();\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView);\n", - "\n", - " var SubprocessView = UpdateView.extend({\n", - " render: function() {\n", - " var me = this;\n", - " me._interval = 0;\n", - " me._minimize = '';\n", - " me._restore = '';\n", - "\n", - " // max lines of output to show\n", - " me.maxOutputLength = 500;\n", - "\n", - " // in case there is already an outputView present,\n", - " // like from before restarting the kernel\n", - " $('.qcodes-output-view').not(me.$el).remove();\n", - "\n", - " me.$el\n", - " .addClass('qcodes-output-view')\n", - " .attr('qcodes-state', 'docked')\n", - " .html(\n", - " '
' +\n", - " '
' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '
' +\n", - " '
'\n",
-       "                );\n",
-       "\n",
-       "            me.clearButton = me.$el.find('.qcodes-clear-output');\n",
-       "            me.minButton = me.$el.find('.qcodes-minimize');\n",
-       "            me.outputArea = me.$el.find('pre');\n",
-       "            me.subprocessList = me.$el.find('.qcodes-process-list');\n",
-       "            me.abortButton = me.$el.find('.qcodes-abort-loop');\n",
-       "            me.processLinesButton = me.$el.find('.qcodes-processlines')\n",
-       "\n",
-       "            me.outputLines = [];\n",
-       "\n",
-       "            me.clearButton.click(function() {\n",
-       "                me.outputArea.html('');\n",
-       "                me.clearButton.addClass('disabled');\n",
-       "            });\n",
-       "\n",
-       "            me.abortButton.click(function() {\n",
-       "                me.send({abort: true});\n",
-       "            });\n",
-       "\n",
-       "            me.processLinesButton.click(function() {\n",
-       "                // toggle multiline process list display\n",
-       "                me.subprocessesMultiline = !me.subprocessesMultiline;\n",
-       "                me.showSubprocesses();\n",
-       "            });\n",
-       "\n",
-       "            me.$el.find('.js-state').click(function() {\n",
-       "                var state = this.className.substr(this.className.indexOf('qcodes'))\n",
-       "                        .split('-')[1].split(' ')[0];\n",
-       "                me.model.set('_state', state);\n",
-       "            });\n",
-       "\n",
-       "            $(window)\n",
-       "                .off('resize.qcodes')\n",
-       "                .on('resize.qcodes', function() {me.clipBounds();});\n",
-       "\n",
-       "            me.update();\n",
-       "        },\n",
-       "\n",
-       "        updateState: function() {\n",
-       "            var me = this,\n",
-       "                oldState = me.$el.attr('qcodes-state'),\n",
-       "                state = me.model.get('_state');\n",
-       "\n",
-       "            if(state === oldState) return;\n",
-       "\n",
-       "            setTimeout(function() {\n",
-       "                // not sure why I can't pop it out of the widgetarea in render, but it seems that\n",
-       "                // some other bit of code resets the parent after render if I do it there.\n",
-       "                // To be safe, just do it on every state click.\n",
-       "                me.$el.appendTo('body');\n",
-       "\n",
-       "                if(oldState === 'floated') {\n",
-       "                    console.log('here');\n",
-       "                    me.$el.draggable('destroy').css({left:'', top: ''});\n",
-       "                }\n",
-       "\n",
-       "                me.$el.attr('qcodes-state', state);\n",
-       "\n",
-       "                if(state === 'floated') {\n",
-       "                    me.$el\n",
-       "                        .draggable({stop: function() { me.clipBounds(); }})\n",
-       "                        .css({\n",
-       "                            left: window.innerWidth - me.$el.width() - 15,\n",
-       "                            top: window.innerHeight - me.$el.height() - 10\n",
-       "                        });\n",
-       "                }\n",
-       "\n",
-       "                // any previous highlighting is now moot\n",
-       "                me.$el.removeClass('qcodes-highlight');\n",
-       "            }, 0);\n",
-       "\n",
-       "        },\n",
-       "\n",
-       "        clipBounds: function() {\n",
-       "            var me = this;\n",
-       "            if(me.$el.attr('qcodes-state') === 'floated') {\n",
-       "                var bounds = me.$el[0].getBoundingClientRect(),\n",
-       "                    minVis = 40,\n",
-       "                    maxLeft = window.innerWidth - minVis,\n",
-       "                    minLeft = minVis - bounds.width,\n",
-       "                    maxTop = window.innerHeight - minVis;\n",
-       "\n",
-       "                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);\n",
-       "                else if(bounds.left < minLeft) me.$el.css('left', minLeft);\n",
-       "\n",
-       "                if(bounds.top > maxTop) me.$el.css('top', maxTop);\n",
-       "                else if(bounds.top < 0) me.$el.css('top', 0);\n",
-       "            }\n",
-       "        },\n",
-       "\n",
-       "        display: function(message) {\n",
-       "            var me = this;\n",
-       "            if(message) {\n",
-       "                var initialScroll = me.outputArea.scrollTop();\n",
-       "                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));\n",
-       "                var scrollBottom = me.outputArea.scrollTop();\n",
-       "\n",
-       "                if(me.$el.attr('qcodes-state') === 'minimized') {\n",
-       "                    // if we add text and the box is minimized, highlight the\n",
-       "                    // title bar to alert the user that there are new messages.\n",
-       "                    // remove then add the class, so we get the animation again\n",
-       "                    // if it's already highlighted\n",
-       "                    me.$el.removeClass('qcodes-highlight');\n",
-       "                    setTimeout(function(){\n",
-       "                        me.$el.addClass('qcodes-highlight');\n",
-       "                    }, 0);\n",
-       "                }\n",
-       "\n",
-       "                var newLines = message.split('\\n'),\n",
-       "                    out = me.outputLines,\n",
-       "                    outLen = out.length;\n",
-       "                if(outLen) out[outLen - 1] += newLines[0];\n",
-       "                else out.push(newLines[0]);\n",
-       "\n",
-       "                for(var i = 1; i < newLines.length; i++) {\n",
-       "                    out.push(newLines[i]);\n",
-       "                }\n",
-       "\n",
-       "                if(out.length > me.maxOutputLength) {\n",
-       "                    out.splice(0, out.length - me.maxOutputLength + 1,\n",
-       "                        '<<< Output clipped >>>');\n",
-       "                }\n",
-       "\n",
-       "                me.outputArea.text(out.join('\\n'));\n",
-       "                me.clearButton.removeClass('disabled');\n",
-       "\n",
-       "                // if we were scrolled to the bottom initially, make sure\n",
-       "                // we stay that way.\n",
-       "                me.outputArea.scrollTop(initialScroll === scrollBottom ?\n",
-       "                    me.outputArea.prop('scrollHeight') : initialScroll);\n",
-       "            }\n",
-       "\n",
-       "            me.showSubprocesses();\n",
-       "            me.updateState();\n",
-       "        },\n",
-       "\n",
-       "        showSubprocesses: function() {\n",
-       "            var me = this,\n",
-       "                replacer = me.subprocessesMultiline ? '
' : ', ',\n", - " processes = (me.model.get('_processes') || '')\n", - " .replace(/\\n/g, '>' + replacer + '<');\n", - "\n", - " if(processes) processes = '<' + processes + '>';\n", - " else processes = 'No subprocesses';\n", - "\n", - " me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1);\n", - "\n", - " me.subprocessList.html(processes);\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView);\n", - "});\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "# setup\n", "%matplotlib nbagg\n", - "import numpy as np\n", - "import qcodes as qc" + "import qcodes as qc\n", + "# import dummy driver for the tutorial\n", + "from qcodes.tests.instrument_mocks import DummyInstrument\n", + "from qcodes.instrument.mock import ArrayGetter\n", + "\n", + "dac1 = DummyInstrument(name=\"dac\")\n", + "dac2 = DummyInstrument(name=\"dac2\")\n", + "# the default dummy instrument returns always a constant value, in the following line we make it random \n", + "# just for the looks 💅\n", + "import random\n", + "dac2.dac2.get = lambda: random.randint(0,100)\n", + "\n", + "# The station is a container for all instruments that makes it easy \n", + "# to log meta-data\n", + "station = qc.Station(dac1, dac2)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Instantiates all the instruments needed for the demo\n", "\n", @@ -372,34 +55,11 @@ ] }, { - "cell_type": "code", - "execution_count": 2, + "cell_type": "markdown", "metadata": { - "collapsed": true + "deletable": true, + "editable": true }, - "outputs": [], - "source": [ - "from toymodel import AModel, MockGates, MockSource, MockMeter, AverageAndRaw\n", - "from qcodes.instrument.mock import ArrayGetter\n", - "\n", - "# now create this \"experiment\", note that all these are instruments \n", - "model = AModel()\n", - "gates = MockGates('gates', model=model)\n", - "source = MockSource('source', model=model)\n", - "meter = MockMeter('meter', model=model)\n", - "\n", - "# The station is a container for all instruments that makes it easy \n", - "# to log meta-data\n", - "station = qc.Station(gates, source, meter)\n", - "\n", - "# it's nice to have the key parameters be part of the global namespace\n", - "# that way they're objects that we can easily set, get, and slice\n", - "c0, c1, c2, vsd = gates.chan0, gates.chan1, gates.chan2, source.amplitude" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, "source": [ "### Only array output\n", "The arguments to Measure are all the same actions you use in a Loop.\n", @@ -408,74 +68,21 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DataSet:\n", - " mode = DataMode.LOCAL\n", - " location = '2016-10-20/12-14-31'\n", - " | | | \n", - " Measured | chan2_1 | chan2 | (100,)\n", - " Measured | meter_amplitude_1_0 | amplitude | (100,)\n", - " Measured | chan2_3 | chan2 | (100,)\n", - " Measured | meter_amplitude_3_0 | amplitude | (100,)\n", - "acquired at 2016-10-20 12:14:31\n" - ] - } - ], + "outputs": [], "source": [ "data = qc.Measure(\n", - " qc.Task(c0.set, 0),\n", - " ArrayGetter(meter.amplitude, c2[-10:10:0.2], 0.001),\n", - " qc.Task(c0.set, 2),\n", - " ArrayGetter(meter.amplitude, c2[-10:10:0.2], 0.001)\n", + " qc.Task(dac1.dac1.set, 0),\n", + " ArrayGetter(dac1.dac2, dac1.dac3.sweep(-10,10,0.2), 0.001),\n", + " qc.Task(dac1.dac1.set, 2),\n", + " ArrayGetter(dac1.dac2, dac1.dac3.sweep(-10,10,0.2), 0.001),\n", ").run()" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Output with scalars\n", - "Scalars still get treated as data, but it's not exactly natural:\n", - "- They still get put into arrays, just 1D 1-element arrays.\n", - "- They need a setpoint array, at least for now, so we make a fake one \"single_set.\" There's something nice about this, actually, in that it makes it absolutely clear that these items are not really arrays." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DataSet:\n", - " mode = DataMode.LOCAL\n", - " location = '2016-10-20/12-14-35'\n", - " | | | \n", - " Setpoint | single_set | single | (1,)\n", - " Measured | gates_chan0 | chan0 | (1,)\n", - " Measured | gates_chan1 | chan1 | (1,)\n", - " Measured | chan2 | chan2 | (100,)\n", - " Measured | meter_amplitude | amplitude | (100,)\n", - " Measured | meter_avg_amplitude | avg_amplitude | (1,)\n", - "acquired at 2016-10-20 12:14:35\n" - ] - } - ], - "source": [ - "data = qc.Measure(c0, c1, AverageAndRaw(meter.amplitude, c2[-10:10:0.2], 0.001)).run()" - ] } ], "metadata": { @@ -494,7 +101,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.0" + "version": "3.5.2" } }, "nbformat": 4, diff --git a/docs/examples/Tutorial.ipynb b/docs/examples/Tutorial.ipynb index 97a4a71dc4a..11464a3a71a 100644 --- a/docs/examples/Tutorial.ipynb +++ b/docs/examples/Tutorial.ipynb @@ -2,7 +2,10 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# QCoDeS tutorial \n", "Basic overview of QCoDeS" @@ -10,7 +13,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Typical QCodes workflow \n", "1. Start up an interactive python session (e.g. using jupyter) \n", @@ -21,415 +27,84 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "collapsed": false, + "deletable": true, + "editable": true, "scrolled": false }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/*\n", - " * Qcodes Jupyter/IPython widgets\n", - " */\n", - "require([\n", - " 'nbextensions/widgets/widgets/js/widget',\n", - " 'nbextensions/widgets/widgets/js/manager'\n", - "], function (widget, manager) {\n", - "\n", - " var UpdateView = widget.DOMWidgetView.extend({\n", - " render: function() {\n", - " window.MYWIDGET = this;\n", - " this._interval = 0;\n", - " this.update();\n", - " },\n", - " update: function() {\n", - " this.display(this.model.get('_message'));\n", - " this.setInterval();\n", - " },\n", - " display: function(message) {\n", - " /*\n", - " * display method: override this for custom display logic\n", - " */\n", - " this.el.innerHTML = message;\n", - " },\n", - " remove: function() {\n", - " clearInterval(this._updater);\n", - " },\n", - " setInterval: function(newInterval) {\n", - " var me = this;\n", - " if(newInterval===undefined) newInterval = me.model.get('interval');\n", - " if(newInterval===me._interval) return;\n", - "\n", - " me._interval = newInterval;\n", - "\n", - " if(me._updater) clearInterval(me._updater);\n", - "\n", - " if(me._interval) {\n", - " me._updater = setInterval(function() {\n", - " me.send({myupdate: true});\n", - " if(!me.model.comm_live) {\n", - " console.log('missing comm, canceling widget updates', me);\n", - " clearInterval(me._updater);\n", - " }\n", - " }, me._interval * 1000);\n", - " }\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('UpdateView', UpdateView);\n", - "\n", - " var HiddenUpdateView = UpdateView.extend({\n", - " display: function(message) {\n", - " this.$el.hide();\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('HiddenUpdateView', HiddenUpdateView);\n", - "\n", - " var SubprocessView = UpdateView.extend({\n", - " render: function() {\n", - " var me = this;\n", - " me._interval = 0;\n", - " me._minimize = '';\n", - " me._restore = '';\n", - "\n", - " // max lines of output to show\n", - " me.maxOutputLength = 500;\n", - "\n", - " // in case there is already an outputView present,\n", - " // like from before restarting the kernel\n", - " $('.qcodes-output-view').not(me.$el).remove();\n", - "\n", - " me.$el\n", - " .addClass('qcodes-output-view')\n", - " .attr('qcodes-state', 'docked')\n", - " .html(\n", - " '
' +\n", - " '
' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '' +\n", - " '
' +\n", - " '
'\n",
-       "                );\n",
-       "\n",
-       "            me.clearButton = me.$el.find('.qcodes-clear-output');\n",
-       "            me.minButton = me.$el.find('.qcodes-minimize');\n",
-       "            me.outputArea = me.$el.find('pre');\n",
-       "            me.subprocessList = me.$el.find('.qcodes-process-list');\n",
-       "            me.abortButton = me.$el.find('.qcodes-abort-loop');\n",
-       "            me.processLinesButton = me.$el.find('.qcodes-processlines')\n",
-       "\n",
-       "            me.outputLines = [];\n",
-       "\n",
-       "            me.clearButton.click(function() {\n",
-       "                me.outputArea.html('');\n",
-       "                me.clearButton.addClass('disabled');\n",
-       "            });\n",
-       "\n",
-       "            me.abortButton.click(function() {\n",
-       "                me.send({abort: true});\n",
-       "            });\n",
-       "\n",
-       "            me.processLinesButton.click(function() {\n",
-       "                // toggle multiline process list display\n",
-       "                me.subprocessesMultiline = !me.subprocessesMultiline;\n",
-       "                me.showSubprocesses();\n",
-       "            });\n",
-       "\n",
-       "            me.$el.find('.js-state').click(function() {\n",
-       "                var state = this.className.substr(this.className.indexOf('qcodes'))\n",
-       "                        .split('-')[1].split(' ')[0];\n",
-       "                me.model.set('_state', state);\n",
-       "            });\n",
-       "\n",
-       "            $(window)\n",
-       "                .off('resize.qcodes')\n",
-       "                .on('resize.qcodes', function() {me.clipBounds();});\n",
-       "\n",
-       "            me.update();\n",
-       "        },\n",
-       "\n",
-       "        updateState: function() {\n",
-       "            var me = this,\n",
-       "                oldState = me.$el.attr('qcodes-state'),\n",
-       "                state = me.model.get('_state');\n",
-       "\n",
-       "            if(state === oldState) return;\n",
-       "\n",
-       "            setTimeout(function() {\n",
-       "                // not sure why I can't pop it out of the widgetarea in render, but it seems that\n",
-       "                // some other bit of code resets the parent after render if I do it there.\n",
-       "                // To be safe, just do it on every state click.\n",
-       "                me.$el.appendTo('body');\n",
-       "\n",
-       "                if(oldState === 'floated') {\n",
-       "                    console.log('here');\n",
-       "                    me.$el.draggable('destroy').css({left:'', top: ''});\n",
-       "                }\n",
-       "\n",
-       "                me.$el.attr('qcodes-state', state);\n",
-       "\n",
-       "                if(state === 'floated') {\n",
-       "                    me.$el\n",
-       "                        .draggable({stop: function() { me.clipBounds(); }})\n",
-       "                        .css({\n",
-       "                            left: window.innerWidth - me.$el.width() - 15,\n",
-       "                            top: window.innerHeight - me.$el.height() - 10\n",
-       "                        });\n",
-       "                }\n",
-       "\n",
-       "                // any previous highlighting is now moot\n",
-       "                me.$el.removeClass('qcodes-highlight');\n",
-       "            }, 0);\n",
-       "\n",
-       "        },\n",
-       "\n",
-       "        clipBounds: function() {\n",
-       "            var me = this;\n",
-       "            if(me.$el.attr('qcodes-state') === 'floated') {\n",
-       "                var bounds = me.$el[0].getBoundingClientRect(),\n",
-       "                    minVis = 40,\n",
-       "                    maxLeft = window.innerWidth - minVis,\n",
-       "                    minLeft = minVis - bounds.width,\n",
-       "                    maxTop = window.innerHeight - minVis;\n",
-       "\n",
-       "                if(bounds.left > maxLeft) me.$el.css('left', maxLeft);\n",
-       "                else if(bounds.left < minLeft) me.$el.css('left', minLeft);\n",
-       "\n",
-       "                if(bounds.top > maxTop) me.$el.css('top', maxTop);\n",
-       "                else if(bounds.top < 0) me.$el.css('top', 0);\n",
-       "            }\n",
-       "        },\n",
-       "\n",
-       "        display: function(message) {\n",
-       "            var me = this;\n",
-       "            if(message) {\n",
-       "                var initialScroll = me.outputArea.scrollTop();\n",
-       "                me.outputArea.scrollTop(me.outputArea.prop('scrollHeight'));\n",
-       "                var scrollBottom = me.outputArea.scrollTop();\n",
-       "\n",
-       "                if(me.$el.attr('qcodes-state') === 'minimized') {\n",
-       "                    // if we add text and the box is minimized, highlight the\n",
-       "                    // title bar to alert the user that there are new messages.\n",
-       "                    // remove then add the class, so we get the animation again\n",
-       "                    // if it's already highlighted\n",
-       "                    me.$el.removeClass('qcodes-highlight');\n",
-       "                    setTimeout(function(){\n",
-       "                        me.$el.addClass('qcodes-highlight');\n",
-       "                    }, 0);\n",
-       "                }\n",
-       "\n",
-       "                var newLines = message.split('\\n'),\n",
-       "                    out = me.outputLines,\n",
-       "                    outLen = out.length;\n",
-       "                if(outLen) out[outLen - 1] += newLines[0];\n",
-       "                else out.push(newLines[0]);\n",
-       "\n",
-       "                for(var i = 1; i < newLines.length; i++) {\n",
-       "                    out.push(newLines[i]);\n",
-       "                }\n",
-       "\n",
-       "                if(out.length > me.maxOutputLength) {\n",
-       "                    out.splice(0, out.length - me.maxOutputLength + 1,\n",
-       "                        '<<< Output clipped >>>');\n",
-       "                }\n",
-       "\n",
-       "                me.outputArea.text(out.join('\\n'));\n",
-       "                me.clearButton.removeClass('disabled');\n",
-       "\n",
-       "                // if we were scrolled to the bottom initially, make sure\n",
-       "                // we stay that way.\n",
-       "                me.outputArea.scrollTop(initialScroll === scrollBottom ?\n",
-       "                    me.outputArea.prop('scrollHeight') : initialScroll);\n",
-       "            }\n",
-       "\n",
-       "            me.showSubprocesses();\n",
-       "            me.updateState();\n",
-       "        },\n",
-       "\n",
-       "        showSubprocesses: function() {\n",
-       "            var me = this,\n",
-       "                replacer = me.subprocessesMultiline ? '
' : ', ',\n", - " processes = (me.model.get('_processes') || '')\n", - " .replace(/\\n/g, '>' + replacer + '<');\n", - "\n", - " if(processes) processes = '<' + processes + '>';\n", - " else processes = 'No subprocesses';\n", - "\n", - " me.abortButton.toggleClass('disabled', processes.indexOf('Measurement')===-1);\n", - "\n", - " me.subprocessList.html(processes);\n", - " }\n", - " });\n", - " manager.WidgetManager.register_widget_view('SubprocessView', SubprocessView);\n", - "});\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "%matplotlib nbagg\n", - "import matplotlib.pyplot as plt\n", - "from pprint import pprint\n", - "import time\n", - "import numpy as np\n", "import qcodes as qc" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Instantiates all the instruments needed for the demo" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "# spawn doesn't like function or class definitions in the interpreter\n", - "# session - had to move them to a file.\n", - "from toymodel import AModel, MockGates, MockSource, MockMeter, AverageGetter, AverageAndRaw\n", + "# import dummy driver for the tutorial\n", + "from qcodes.tests.instrument_mocks import DummyInstrument\n", + "from qcodes.instrument.mock import ArrayGetter\n", "\n", - "# now create this \"experiment\", note that all these are instruments \n", - "model = AModel()\n", - "gates = MockGates('gates', model=model)\n", - "source = MockSource('source', model=model)\n", - "meter = MockMeter('meter', model=model)\n", + "dac1 = DummyInstrument(name=\"dac\")\n", + "dac2 = DummyInstrument(name=\"dac2\")\n", + "# the default dummy instrument returns always a constant value, in the following line we make it random \n", + "# just for the looks 💅\n", + "import random\n", + "dac2.dac2.get = lambda: random.randint(0,100)\n", "\n", "# The station is a container for all instruments that makes it easy \n", "# to log meta-data\n", - "station = qc.Station(gates, source, meter)\n", - "\n", - "# it's nice to have the key parameters be part of the global namespace\n", - "# that way they're objects that we can easily set, get, and slice\n", - "c0, c1, c2, vsd = gates.chan0, gates.chan1, gates.chan2, source.amplitude" + "station = qc.Station(dac1, dac2)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "### The location provider can be set globally " ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "# dm = qc.data.manager.get_data_manager()\n", "loc_provider = qc.data.location.FormatLocation(fmt='data/{date}/#{counter}_{name}_{time}')\n", "qc.data.data_set.DataSet.location_provider=loc_provider" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# Running an experiment \n", "\n", @@ -437,148 +112,40 @@ "\n", "Before you run a measurement loop you do two things:\n", "1. You describe what parameter(s) to vary and how. This is the creation of a `Loop` object: `loop = Loop(sweep_values, ...)`\n", - "2. You describe what to do at each step in the loop. This is `loop.each(*actions)` which converts the `Loop` object into an `ActiveLoop` object. Actions can be:\n", + "2. You describe what to do at each step in the loop. This is `loop.each(*actions)` \n", " - measurements (any object with a `.get` method will be interpreted as a measurement)\n", " - `Task`: some callable (which can have arguments with it) to be executed each time through the loop. Does not generate data.\n", " - `Wait`: a specialized `Task` just to wait a certain time.\n", - " - `BreakIf`: some condition that, if it returns truthy, breaks (this level of) the loop\n", - " - Another `ActiveLoop` to nest inside the outer one.\n", - "\n", - "For more details, see issue #232 docs: Write bigger picture" + " - `BreakIf`: some condition that, if it returns truthy, breaks (this level of) the loop" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { - "collapsed": false + "collapsed": true, + "deletable": true, + "editable": true }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:negative delay -0.001849 sec\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DataSet:\n", - " mode = DataMode.LOCAL\n", - " location = 'data/2016-11-08/#024_testsweep_13-29-04'\n", - " | | | \n", - " Setpoint | gates_chan0_set | chan0 | (201,)\n", - " Measured | meter_amplitude | amplitude | (201,)\n", - "started at 2016-11-08 13:29:23\n" - ] - } - ], - "source": [ - "# Notice that one can use an explicit location and `overwrite=True` here so that\n", - "# running this notebook over and over won't result in extra files.\n", - "# If you leave these out, you get a new timestamped DataSet each time.\n", - "\n", - "\n", - "loop = qc.Loop(c0.sweep(0,20,0.1), delay=0.001).each(meter.amplitude)\n", - "data = loop.get_data_set(name='testsweep')\n", - "plot = qc.QtPlot()\n", - "plot.add(data.meter_amplitude)\n", - "_ = loop.with_bg_task(plot.update, plot.save).run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Output of the loop\n", - "Notice the **\"DataSet\"**. \n", - "A loop returns a dataset. \n", - "The representation of the dataset shows what arrays it contains and where it is saved. \n", - "The dataset initially starts out empty (filled with NAN's) and get's filled while the Loop get's executed. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, + "outputs": [], "source": [ - "Once the measurement is done, take a look at the file in finder/explorer (the dataset.location should give you the relative path). \n", - "Note also the snapshot that captures the settings of all instruments at the start of the Loop. \n", - "This metadata is also accesible from the dataset and captures a snapshot of each instrument listed in the station. " + "loop = qc.Loop(dac1.dac1.sweep(0,20,0.1), delay=0.001).each(dac2.dac2)\n", + "data = loop.get_data_set(name='testsweep')" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'__class__': 'toymodel.MockMeter',\n", - " 'functions': {},\n", - " 'name': 'meter',\n", - " 'parameters': {'IDN': {'__class__': 'qcodes.instrument.parameter.StandardParameter',\n", - " 'instrument': 'toymodel.MockMeter',\n", - " 'instrument_name': 'meter',\n", - " 'label': 'IDN',\n", - " 'name': 'IDN',\n", - " 'ts': '2016-11-08 13:29:02',\n", - " 'units': '',\n", - " 'value': {'firmware': None,\n", - " 'model': 'MockMeter',\n", - " 'serial': 'meter',\n", - " 'vendor': None}},\n", - " 'amplitude': {'__class__': 'qcodes.instrument.parameter.StandardParameter',\n", - " 'instrument': 'toymodel.MockMeter',\n", - " 'instrument_name': 'meter',\n", - " 'label': 'Current (nA)',\n", - " 'name': 'amplitude',\n", - " 'ts': '2016-11-08 13:29:23',\n", - " 'units': '',\n", - " 'value': 0.117}}}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "meter.snapshot()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plotting the loop II\n", - "\n", - "QCodes supports both matplotlib inline plotting and pyqtgraph for plotting. \n", - "For a comparison see http://pyqtgraph.org/ (actually not that biased)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The same API works for plotting a measured dataset or an old dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, + "outputs": [], "source": [ - "`DataSet` objects are not intended to be instantiated directly, but\n", - "rather through the helper functions:\n", - "- `load_data` for existing data sets, including the data currently\n", - " being acquired.\n", - "- `new_data` to make an empty data set to be populated with new\n", - " measurements or simulation data. `new_data` is called internally by\n", - " `Loop.run()` so is also generally not needed directly." + "plot = qc.QtPlot()\n", + "plot.add(data.dac2_dac2)\n", + "_ = loop.with_bg_task(plot.update, plot.save).run()" ] }, { @@ -586,1024 +153,130 @@ "execution_count": null, "metadata": { "collapsed": false, - "scrolled": false + "deletable": true, + "editable": true }, "outputs": [], "source": [ - "loaded_data = qc.load_data(\"data/2016-10-10/#002_testsweep_10-08-32\")\n", - "plot = qc.MatPlot(loaded_data.meter_amplitude)" + "plot" ] }, { "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example: multiple 2D measurements with live plotting" - ] - }, - { - "cell_type": "code", - "execution_count": 6, "metadata": { - "collapsed": true + "deletable": true, + "editable": true }, - "outputs": [], "source": [ - "loop = qc.Loop(c1[-15:15:1], 0.01).loop(c0[-15:12:.5], 0.001).each(\n", - " meter.amplitude, # first measurement, at c2=0 -> amplitude_0 bcs it's action 0\n", - " qc.Task(c2.set, 1), # action 1 -> c2.set(1)\n", - " qc.Wait(0.001),\n", - " meter.amplitude, # second measurement, at c2=1 -> amplitude_4 bcs it's action 4\n", - " qc.Task(c2.set, 0)\n", - " )\n", - "data = loop.get_data_set(name='2D_test')" + "## Output of the loop\n", + "A loop returns a dataset. \n", + "The representation of the dataset shows what arrays it contains and where it is saved. \n", + "The dataset initially starts out empty (filled with NAN's) and get's filled while the Loop get's executed. " ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ - "### Plot with matplotlib " + "Once the measurement is done, take a look at the file in finder/explorer (the dataset.location should give you the relative path). \n", + "Note also the snapshot that captures the settings of all instruments at the start of the Loop. \n", + "This metadata is also accesible from the dataset and captures a snapshot of each instrument listed in the station. " ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "collapsed": false, - "scrolled": false + "deletable": true, + "editable": true }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('