Hashing, caching and fast-forwarding

.travis-data/test_daemon.py

@@ -98,6 +98,65 @@ def validate_workchains(expected_results):
 
     return valid
 
+def validate_cached(cached_calcs):
+    """
+    Check that the calculations with created with caching are indeed cached.
+    """
+    return all(
+        '_aiida_cached_from' in calc.extras() and
+        calc.get_hash() == calc.get_extra('_aiida_hash')
+        for calc in cached_calcs
+    )
+
+def create_calculation(code, counter, inputval, use_cache=False):
+    parameters = ParameterData(dict={'value': inputval})
+    template = ParameterData(dict={
+            ## The following line adds a significant sleep time.
+            ## I set it to 1 second to speed up tests
+            ## I keep it to a non-zero value because I want
+            ## To test the case when AiiDA finds some calcs
+            ## in a queued state
+            #'cmdline_params': ["{}".format(counter % 3)], # Sleep time
+            'cmdline_params': ["1"],
+            'input_file_template': "{value}", # File just contains the value to double
+            'input_file_name': 'value_to_double.txt',
+            'output_file_name': 'output.txt',
+            'retrieve_temporary_files': ['triple_value.tmp']
+            })
+    calc = code.new_calc()
+    calc.set_max_wallclock_seconds(5 * 60)  # 5 min
+    calc.set_resources({"num_machines": 1})
+    calc.set_withmpi(False)
+    calc.set_parser_name('simpleplugins.templatereplacer.test.doubler')
+
+    calc.use_parameters(parameters)
+    calc.use_template(template)
+    calc.store_all(use_cache=use_cache)
+    expected_result = {
+        'value': 2 * inputval,
+        'retrieved_temporary_files': {
+            'triple_value.tmp': str(inputval * 3)
+        }
+    }
+    print "[{}] created calculation {}, pk={}".format(
+        counter, calc.uuid, calc.dbnode.pk)
+    return calc, expected_result
+
+def submit_calculation(code, counter, inputval):
+    calc, expected_result = create_calculation(
+        code=code, counter=counter, inputval=inputval
+    )
+    calc.submit()
+    print "[{}] calculation submitted.".format(counter)
+    return calc, expected_result
+
+def create_cache_calc(code, counter, inputval):
+    calc, expected_result = create_calculation(
+        code=code, counter=counter, inputval=inputval, use_cache=True
+    )
+    print "[{}] created cached calculation.".format(counter)
+    return calc, expected_result
+
 def main():
 
     # Submitting the Calculations
@@ -106,39 +165,10 @@ def main():
     expected_results_calculations = {}
     for counter in range(1, number_calculations + 1):
         inputval = counter
-        parameters = ParameterData(dict={'value': inputval})
-        template = ParameterData(dict={
-                ## The following line adds a significant sleep time.
-                ## I set it to 1 second to speed up tests
-                ## I keep it to a non-zero value because I want
-                ## To test the case when AiiDA finds some calcs
-                ## in a queued state
-                #'cmdline_params': ["{}".format(counter % 3)], # Sleep time
-                'cmdline_params': ["1"],
-                'input_file_template': "{value}", # File just contains the value to double
-                'input_file_name': 'value_to_double.txt',
-                'output_file_name': 'output.txt',
-                'retrieve_temporary_files': ['triple_value.tmp']
-                })
-        calc = code.new_calc()
-        calc.set_max_wallclock_seconds(5 * 60)  # 5 min
-        calc.set_resources({"num_machines": 1})
-        calc.set_withmpi(False)
-        calc.set_parser_name('simpleplugins.templatereplacer.test.doubler')
-
-        calc.use_parameters(parameters)
-        calc.use_template(template)
-        calc.store_all()
-        print "[{}] created calculation {}, pk={}".format(
-            counter, calc.uuid, calc.dbnode.pk)
-        expected_results_calculations[calc.pk] = {
-            'value': inputval * 2,
-            'retrieved_temporary_files': {
-                'triple_value.tmp': str(inputval * 3)
-            }
-        }
-        calc.submit()
-        print "[{}] calculation submitted.".format(counter)
+        calc, expected_result = submit_calculation(
+            code=code, counter=counter, inputval=inputval
+        )
+        expected_results_calculations[calc.pk] = expected_result
 
     # Submitting the Workchains
     print "Submitting {} workchains to the daemon".format(number_workchains)
@@ -158,7 +188,7 @@ def main():
     exited_with_timeout = True
     while time.time() - start_time < timeout_secs:
         time.sleep(15) # Wait a few seconds
-        
+
         # Print some debug info, both for debugging reasons and to avoid
         # that the test machine is shut down because there is no output
 
@@ -168,7 +198,7 @@ def main():
         print "Output of 'verdi calculation list -a':"
         try:
             print subprocess.check_output(
-                ["verdi", "calculation", "list", "-a"], 
+                ["verdi", "calculation", "list", "-a"],
                 stderr=subprocess.STDOUT,
             )
         except subprocess.CalledProcessError as e:
@@ -177,7 +207,7 @@ def main():
         print "Output of 'verdi work list':"
         try:
             print subprocess.check_output(
-                ['verdi', 'work', 'list'], 
+                ['verdi', 'work', 'list'],
                 stderr=subprocess.STDOUT,
             )
         except subprocess.CalledProcessError as e:
@@ -186,7 +216,7 @@ def main():
         print "Output of 'verdi daemon status':"
         try:
             print subprocess.check_output(
-                ["verdi", "daemon", "status"], 
+                ["verdi", "daemon", "status"],
                 stderr=subprocess.STDOUT,
             )
         except subprocess.CalledProcessError as e:
@@ -204,8 +234,18 @@ def main():
             timeout_secs)
         sys.exit(2)
     else:
+        # create cached calculations -- these should be FINISHED immediately
+        cached_calcs = []
+        for counter in range(1, number_calculations + 1):
+            inputval = counter
+            calc, expected_result = create_cache_calc(
+                code=code, counter=counter, inputval=inputval
+            )
+            cached_calcs.append(calc)
+            expected_results_calculations[calc.pk] = expected_result
         if (validate_calculations(expected_results_calculations)
-            and validate_workchains(expected_results_workchains)):
+            and validate_workchains(expected_results_workchains)
+            and validate_cached(cached_calcs)):
             print_daemon_log()
             print ""
             print "OK, all calculations have the expected parsed result"

.travis.yml

@@ -44,9 +44,10 @@ install:
     # Install AiiDA with some optional dependencies
     - pip install .[REST,docs,atomic_tools,testing,dev_precommit]
 
+
 env:
 ## Build matrix to test both backends, and the docs
-## I still let it create the test backend for django 
+## I still let it create the test backend for django
 ## also when building the docs
 ## because otherwise the code would complain. Also, I need latex.
 - TEST_TYPE="pre-commit"

aiida/backends/djsite/db/subtests/generic.py

@@ -133,18 +133,22 @@ def test_replacement_1(self):
         DbExtra.set_value_for_node(n1.dbnode, "pippobis", [5, 6, 'c'])
         DbExtra.set_value_for_node(n2.dbnode, "pippo2", [3, 4, 'b'])
 
-        self.assertEquals(n1.dbnode.extras, {'pippo': [1, 2, 'a'],
-                                             'pippobis': [5, 6, 'c']})
-        self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n1.get_extras(), {'pippo': [1, 2, 'a'],
+                                             'pippobis': [5, 6, 'c'],
+                                             '_aiida_hash': n1.get_hash()
+                                             })
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'],
+                                            '_aiida_hash': n2.get_hash()
+                                            })
 
         new_attrs = {"newval1": "v", "newval2": [1, {"c": "d", "e": 2}]}
 
         DbExtra.reset_values_for_node(n1.dbnode, attributes=new_attrs)
-        self.assertEquals(n1.dbnode.extras, new_attrs)
-        self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n1.get_extras(), new_attrs)
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()})
 
         DbExtra.del_value_for_node(n1.dbnode, key='newval2')
         del new_attrs['newval2']
-        self.assertEquals(n1.dbnode.extras, new_attrs)
+        self.assertEquals(n1.get_extras(), new_attrs)
         # Also check that other nodes were not damaged
-        self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()})

aiida/backends/sqlalchemy/tests/generic.py

@@ -141,18 +141,18 @@ def test_replacement_1(self):
 
         n2.set_extra("pippo2", [3, 4, u'b'])
 
-        self.assertEqual(n1.get_extras(),{'pippo': [1, 2, u'a'], 'pippobis': [5, 6, u'c']})
+        self.assertEqual(n1.get_extras(),{'pippo': [1, 2, u'a'], 'pippobis': [5, 6, u'c'], '_aiida_hash': n1.get_hash()})
 
-        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()})
 
         new_attrs = {"newval1": "v", "newval2": [1, {"c": "d", "e": 2}]}
 
         n1.reset_extras(new_attrs)
         self.assertEquals(n1.get_extras(), new_attrs)
-        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()})
 
         n1.del_extra('newval2')
         del new_attrs['newval2']
         self.assertEquals(n1.get_extras(), new_attrs)
         # Also check that other nodes were not damaged
-        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']})
+        self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()})

aiida/backends/testbase.py

@@ -144,7 +144,7 @@ def run_aiida_db_tests(tests_to_run, verbose=False):
 
     actually_run_tests = []
     num_tests_expected = 0
-    
+
     # To avoid adding more than once the same test
     # (e.g. if you type both db and db.xxx)
     found_modulenames = set()

aiida/backends/tests/__init__.py

@@ -66,6 +66,8 @@
         'pluginloader': ['aiida.backends.tests.test_plugin_loader'],
         'daemon': ['aiida.backends.tests.daemon'],
         'verdi_commands': ['aiida.backends.tests.verdi_commands'],
+        'caching_config': ['aiida.backends.tests.test_caching_config'],
+        'inline_calculation': ['aiida.backends.tests.inline_calculation'],
     }
 }
 

aiida/backends/tests/inline_calculation.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""
+Tests for inline calculations.
+"""
+
+from aiida.orm.data.base import Int
+from aiida.common.caching import enable_caching
+from aiida.orm.calculation.inline import make_inline, InlineCalculation
+from aiida.backends.testbase import AiidaTestCase
+
+class TestInlineCalculation(AiidaTestCase):
+    """
+    Tests for the InlineCalculation calculations.
+    """
+    def setUp(self):
+        @make_inline
+        def incr_inline(inp):
+            return {'res': Int(inp.value + 1)}
+
+        self.incr_inline = incr_inline
+
+    def test_incr(self):
+        """
+        Simple test for the inline increment function.
+        """
+        for i in [-4, 0, 3, 10]:
+            calc, res = self.incr_inline(inp=Int(i))
+            self.assertEqual(res['res'].value, i + 1)
+
+    def test_caching(self):
+        with enable_caching(InlineCalculation):
+            calc1, res1 = self.incr_inline(inp=Int(11))
+            calc2, res2 = self.incr_inline(inp=Int(11))
+            self.assertEquals(res1['res'].value, res2['res'].value, 12)
+            self.assertEquals(calc1.get_extra('_aiida_cached_from', calc1.uuid), calc2.get_extra('_aiida_cached_from'))
+
+    def test_caching_change_code(self):
+        with enable_caching(InlineCalculation):
+            calc1, res1 = self.incr_inline(inp=Int(11))
+
+            @make_inline
+            def incr_inline(inp):
+                return {'res': Int(inp.value + 2)}
+
+            calc2, res2 = incr_inline(inp=Int(11))
+            self.assertNotEquals(res1['res'].value, res2['res'].value)
+            self.assertFalse('_aiida_cached_from' in calc2.extras())