Merge pull request #1052 from aiidateam/merge_v0.11.0_into_develop

Merge v0.11.0 into develop

.travis-data/set_daemon_times.py

@@ -1,4 +1,13 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 """
 For AiiDA 0.10.0, we set the times of the daemon
 to 5 seconds.

.travis-data/test_fixtures.py

@@ -1,3 +1,12 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 """Unittests for plugin test fixture manager"""
 import os
 import unittest

.travis-data/test_plugin_testcase.py

@@ -1,3 +1,12 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 """
 Test the plugin test case
 

AUTHORS.txt

@@ -50,7 +50,7 @@ And the following people for general improvements to the code, fixing bugs,
 corrections and improvements to the documentation and useful suggestions:
 
 Ivano E. Castelli, Ian Lee, Gianluca Prandini, Jianxing Huang, Antimo Marrazzo,
-Nicola Varini, Mario Zic, Vladimir Dikan, Michael Atambo
+Nicola Varini, Mario Zic, Vladimir Dikan, Michael Atambo, Ole Schütt, Marco Borelli
 
 ---
 

CHANGELOG.md

@@ -1,3 +1,42 @@
+## v0.11.0:
+
+### Improvements
+
+#### Core entities
+- `Computer`: the shebang line is now customizable [[#940]](https://github.com/aiidateam/aiida_core/pull/940)
+- `KpointsData`: deprecate buggy legacy implementation of k-point generation in favor of Seekpath [[#1015]](https://github.com/aiidateam/aiida_core/pull/1015)
+- `ParameterData`: `to_aiida_type` used on dictionaries now automatically converted to `ParameterData` [[#947]](https://github.com/aiidateam/aiida_core/pull/947)
+- `JobCalculation`: parsers can now specify files that are retrieved locally for parsing, but only temporarily, as they are deleted after parsing is completed [[#886]](https://github.com/aiidateam/aiida_core/pull/886) [[#894]](https://github.com/aiidateam/aiida_core/pull/894)
+
+#### Plugins
+- Plugin data hooks: plugins can now add custom commands to `verdi data` [[#993]](https://github.com/aiidateam/aiida_core/pull/993)
+- Plugin fixtures: simple-to-use decorators for writing tests of plugins [[#716]](https://github.com/aiidateam/aiida_core/pull/716) [[#865]](https://github.com/aiidateam/aiida_core/pull/865)
+- Plugin development: no longer swallow `ImportError` exception during import of plugins [[#1029]](https://github.com/aiidateam/aiida_core/pull/1029)
+
+#### Verdi
+- `verdi shell`: improve tab completion of imports in  [[#1008]](https://github.com/aiidateam/aiida_core/pull/1008)
+- `verdi work list`: projections for verdi work list [[#847]](https://github.com/aiidateam/aiida_core/pull/847)
+
+#### Miscellaneous
+- Supervisor removal: dependency on unix-only supervisor package removed [[#790]](https://github.com/aiidateam/aiida_core/pull/790)
+- REST API: add server info endpoint, structure endpoint can return different file formats [[#878]](https://github.com/aiidateam/aiida_core/pull/878)
+- REST API: update endpoints for structure visualization, calculation (includes retrieved input & output list), add endpoints for `UpfData` and more [[#977]](https://github.com/aiidateam/aiida_core/pull/977) [[#991]](https://github.com/aiidateam/aiida_core/pull/991)
+- Tests using daemon run faster [[#870]](https://github.com/aiidateam/aiida_core/pull/870)
+- Documentation: updated outdated workflow examples [[#948]](https://github.com/aiidateam/aiida_core/pull/948)
+- Documentation: updated import/export [[#994]](https://github.com/aiidateam/aiida_core/pull/994), 
+- Documentation: plugin quickstart [[#996]](https://github.com/aiidateam/aiida_core/pull/996), 
+- Documentation: parser example [[#1003]](https://github.com/aiidateam/aiida_core/pull/1003) 
+
+### Minor bug fixes
+- Fix bug with repository on external hard drive [[#982]](https://github.com/aiidateam/aiida_core/pull/982)
+- Fix bug in configuration of pre-commit hooks [[#863]](https://github.com/aiidateam/aiida_core/pull/863)
+- Fix and improve plugin loader tests [[#1025]](https://github.com/aiidateam/aiida_core/pull/1025)
+- Fix broken celery logging [[#1033]](https://github.com/aiidateam/aiida_core/pull/1033)
+
+### Deprecations
+- async from aiida.work.run has been deprecated because it can lead to race conditions and thereby unexpected behavior [[#1040]](https://github.com/aiidateam/aiida_core/pull/1040)
+
+
 ## v0.10.1:
 
 ### Improvements
@@ -54,8 +93,10 @@
 - Fixed query in the ICSD importer element that caused certain structures to be erroneously skipped [[#690]](https://github.com/aiidateam/aiida_core/pull/690)
 
 ### Miscellaneous
-- Added a "quickstart" to plugin development in the [documentation](http://aiida-core.readthedocs.io/en/v0.10.0/developer_guide/plugins/quickstart.html), structured around the new [plugin template](https://github.com/aiidateam/aiida-plugin-template)
-- Improved and restructured the documentation
+- Added a "quickstart" to plugin development in the [documentation](http://aiida-core.readthedocs.io/en/v0.10.0/developer_guide/plugins/quickstart.html),
+  structured around the new [plugintemplate](https://github.com/aiidateam/aiida-plugin-template) [[#818]](https://github.com/aiidateam/aiida_core/pull/818)
+- Improved and restructured the developer documentation [[#818]](https://github.com/aiidateam/aiida_core/pull/818)
+
 
 
 ## v0.9.1:

aiida/__init__.py

@@ -13,7 +13,7 @@
 
 __copyright__ = u"Copyright (c), This file is part of the AiiDA platform. For further information please visit http://www.aiida.net/. All rights reserved."
 __license__ = "MIT license, see LICENSE.txt file."
-__version__ = "0.10.1"
+__version__ = "0.11.0"
 __authors__ = "The AiiDA team."
 __paper__ = """G. Pizzi, A. Cepellotti, R. Sabatini, N. Marzari, and B. Kozinsky, "AiiDA: automated interactive infrastructure and database for computational science", Comp. Mat. Sci 111, 218-230 (2016); http://dx.doi.org/10.1016/j.commatsci.2015.09.013 - http://www.aiida.net."""
 __paper_short__ = """G. Pizzi et al., Comp. Mat. Sci 111, 218 (2016)."""

aiida/backends/tests/dataclasses.py

@@ -2640,6 +2640,53 @@ class TestKpointsData(AiidaTestCase):
     Tests the TrajectoryData objects.
     """
 
+    def test_set_kpoints_path_legacy(self):
+        """
+        Regression test for the deprecated KpointsData.set_kpoints_path method.
+        For certain formats of a direct kpoint list, it is not necessary to have defined a cell.
+        """
+        import numpy
+        from aiida.orm.data.array.kpoints import KpointsData
+
+        # Create a node with two arrays
+        kpoints_01 = KpointsData()
+        kpoints_02 = KpointsData()
+        kpoints_03 = KpointsData()
+        kpoints_04 = KpointsData()
+
+        # The various allowed formats
+        format_01 = [('G', 'M')]
+        format_02 = [('G', 'M', 30)]
+        format_03 = [('G', (0, 0, 0), 'M', (1, 1, 1))]
+        format_04 = [('G', (0, 0, 0), 'M', (1, 1, 1), 30)]
+
+        # Without a cell defined, the first two should fail, the last two should work
+        with self.assertRaises(ValueError):
+            kpoints_01.set_kpoints_path(format_01)
+
+        with self.assertRaises(ValueError):
+            kpoints_02.set_kpoints_path(format_02)
+
+        kpoints_03.set_kpoints_path(format_03)
+        kpoints_04.set_kpoints_path(format_04)
+
+        # Define a cell and settings it enable the usage of formats 1 and 2
+        alat = 4.
+        cell = numpy.array([
+            [alat, 0., 0.],
+            [0., alat, 0.],
+            [0., 0., alat],
+        ])
+
+        kpoints_01.set_cell(cell)
+        kpoints_02.set_cell(cell)
+
+        kpoints_01.set_kpoints_path(format_01)
+        kpoints_02.set_kpoints_path(format_02)
+        kpoints_03.set_kpoints_path(format_03)
+        kpoints_04.set_kpoints_path(format_04)
+
+
     def test_mesh(self):
         """
         Check the methods to set and retrieve a mesh.

aiida/backends/tests/work/util.py

@@ -7,16 +7,13 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-
 from aiida.backends.testbase import AiidaTestCase
-
-from aiida.work.process import Process
-from aiida.work.workfunction import workfunction
 from aiida.common.lang import override
-from aiida.work.run import async
 from aiida.orm.data.base import Int
+from aiida.work.process import Process
+from aiida.work.run import run
 from aiida.work.util import ProcessStack
-
+from aiida.work.workfunction import workfunction
 
 
 class StackTester(Process):
@@ -30,22 +27,25 @@ def _run(self):
 @workfunction
 def registry_tester():
     # Call a wf
-    future = async(nested_tester)
-    out = future.result()
-    assert future.pid == out['pid']
-    assert future.pid == out['node_pk']
+    result, pid = run(nested_tester, _return_pid=True)
+    assert pid == result['pid']
+    assert pid == result['node_pk']
 
     # Call a Process
     StackTester.run()
 
-    return {'pid': Int(ProcessStack.get_active_process_id()),
-            'node_pk': Int(ProcessStack.get_active_process_calc_node().pk)}
+    return {
+        'pid': Int(ProcessStack.get_active_process_id()),
+        'node_pk': Int(ProcessStack.get_active_process_calc_node().pk)
+    }
 
 
 @workfunction
 def nested_tester():
-    return {'pid': Int(ProcessStack.get_active_process_id()),
-            'node_pk': Int(ProcessStack.get_active_process_calc_node().pk)}
+    return {
+        'pid': Int(ProcessStack.get_active_process_id()),
+        'node_pk': Int(ProcessStack.get_active_process_calc_node().pk)
+    }
 
 
 class TestProcessRegistry(AiidaTestCase):
@@ -65,13 +65,7 @@ def test_process_pid_and_calc(self):
         StackTester.run()
 
     def test_wf_pid_and_calc(self):
-        future = async(registry_tester)
-        out = future.result()
-
-        self.assertEqual(out['pid'], future.pid)
-        self.assertEqual(out['node_pk'], future.pid)
-
-
-
-
+        result, pid = run(nested_tester, _return_pid=True)
 
+        self.assertEqual(result['pid'], pid)
+        self.assertEqual(result['node_pk'], pid)

aiida/backends/tests/work/workChain.py

@@ -21,7 +21,7 @@
     ToContext, _Block, _If, _While, if_, while_, return_
 from aiida.work.workchain import _WorkChainSpec, Outputs
 from aiida.work.workfunction import workfunction
-from aiida.work.run import run, async, legacy_workflow
+from aiida.work.run import run, legacy_workflow
 from aiida.orm.data.base import Int, Str, Bool
 import aiida.work.util as util
 from aiida.common.links import LinkType
@@ -236,18 +236,19 @@ def define(cls, spec):
                 spec.outline(cls.s1, cls.s2, cls.s3)
 
             def s1(self):
-                return ToContext(r1=Outputs(async(a)), r2=Outputs(async(b)))
+                self.ctx.r1 = run(a)
+                self.ctx.r2 = run(b)
 
             def s2(self):
-                assert self.ctx.r1['_return'] == A
-                assert self.ctx.r2['_return'] == B
+                assert self.ctx.r1 == A
+                assert self.ctx.r2 == B
 
                 # Try overwriting r1
-                return ToContext(r1=Outputs(async(b)))
+                self.ctx.r1 = run(b)
 
             def s3(self):
-                assert self.ctx.r1['_return'] == B
-                assert self.ctx.r2['_return'] == B
+                assert self.ctx.r1 == B
+                assert self.ctx.r2 == B
 
         Wf.run()
 
@@ -323,32 +324,6 @@ def after(self):
 
         WcWithReturn.run()
 
-    @unittest.skipIf(settings.BACKEND == u'sqlalchemy', "SQLA async functionality is in development")
-    def test_tocontext_async_workchain(self):
-        class MainWorkChain(WorkChain):
-            @classmethod
-            def define(cls, spec):
-                super(MainWorkChain, cls).define(spec)
-                spec.outline(cls.run, cls.check)
-                spec.dynamic_output()
-
-            def run(self):
-                return ToContext(subwc=async(SubWorkChain))
-
-            def check(self):
-                assert self.ctx.subwc.out.value == Int(5)
-
-        class SubWorkChain(WorkChain):
-            @classmethod
-            def define(cls, spec):
-                super(SubWorkChain, cls).define(spec)
-                spec.outline(cls.run)
-
-            def run(self):
-                self.out("value", Int(5))
-
-        run(MainWorkChain)
-
     def test_if_block_persistence(self):
         """ This test was created to capture issue #902 """
         wc = IfTest.new_instance()

aiida/backends/tests/work/workfunction.py

@@ -7,16 +7,14 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-
 import plum.process_monitor
 from aiida.backends.testbase import AiidaTestCase
 from aiida.work.workfunction import workfunction
 from aiida.orm.data.base import get_true_node
-from aiida.work.run import async, run
+from aiida.work.run import run
 import aiida.work.util as util
 
 
-
 @workfunction
 def simple_wf():
     return {'result': get_true_node()}
@@ -42,10 +40,6 @@ def test_blocking(self):
         self.assertTrue(simple_wf()['result'])
         self.assertTrue(return_input(get_true_node())['result'])
 
-    def test_async(self):
-        self.assertTrue(async(simple_wf).result()['result'])
-        self.assertTrue(async(return_input, get_true_node()).result()['result'])
-
     def test_run(self):
         self.assertTrue(run(simple_wf)['result'])
-        self.assertTrue(run(return_input, get_true_node())['result'])
+        self.assertTrue(run(return_input, get_true_node())['result'])

aiida/cmdline/baseclass.py

@@ -7,7 +7,7 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-
+import click
 
 
 class VerdiCommand(object):
@@ -137,6 +137,25 @@ def run(self, *args):
         function_to_call(*args[1:])
 
     def complete(self, subargs_idx, subargs):
+        """
+        Relay tab-completion requests to subcommand complete functions.
+
+        Click-implemented commands (filtered by isinstance check) pose a special case.
+        Here we use aiida.cmdline.commands.click_subcmd_complete to create a completion function
+        on the fly, leveraging click functionality. Since VerdiCommands with click subcommands must actually rout to the root
+        (aiida.cmdline.commands.verdi) for each subcommand, we have only access to the root as a python object through routed_subcommands,
+        Not to the active subcommand.
+        This means we must follow the chain up through cli parameters. Since this function is only passed subargs, we need to use sys.argv.
+
+        sys.argv in this case follows the signature of ``verdi completion``::
+
+            verdi completion <num_subargs> verdi [<subgroup>, ...] <subcommand> ([<subcmd param>, ...] | '')
+
+        Therefore we can disregard the first four and the last element of argv and loop through sys.argv[4:len(sys.argv)-1], for each
+        determine wether it's a subcommand, if so, retrieve it's python object until we find the last subcommand, for which we then build
+        the completion function. We can safely disregard the last one, because it is never a (complete) subcommand and it will be passed
+        to the completion function we build in any case (as part of ``subargs``).
+        """
         if subargs_idx == 0:
             print "\n".join(self.routed_subcommands.keys())
         elif subargs_idx >= 1:
@@ -146,8 +165,16 @@ def complete(self, subargs_idx, subargs):
                 first_subarg = ''
 
             try:
-                complete_function = self.routed_subcommands[
-                    first_subarg]().complete
+                cmd_or_class = self.routed_subcommands[first_subarg]
+                if isinstance(cmd_or_class, (click.Command, click.MultiCommand)):
+                    import sys
+                    from aiida.cmdline.commands import click_subcmd_complete
+
+                    for i in sys.argv[4:len(sys.argv)-1]:
+                        cmd_or_class = cmd_or_class.commands.get(i, cmd_or_class)
+                    complete_function = click_subcmd_complete(cmd_or_class)
+                else:
+                    complete_function = cmd_or_class().complete
             except KeyError:
                 print ""
                 return