Merge branch 'develop' of github.com:aiidateam/aiida_core into add_re…

…mote_listdir

aiida/backends/djsite/queries.py

@@ -84,3 +84,52 @@ def query_jobcalculations_by_computer_user_state(
             return queryresults[:limit]
         else:
             return queryresults
+
+    def get_creation_statistics(
+            self,
+            user_email=None
+    ):
+        """
+        Return a dictionary with the statistics of node creation, summarized by day,
+        optimized for the Django backend.
+
+        :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary.
+
+        :param user_email: If None (default), return statistics for all users.
+            If an email is specified, return only the statistics for the given user.
+
+        :return: a dictionary as
+            follows::
+
+                {
+                   "total": TOTAL_NUM_OF_NODES,
+                   "types": {TYPESTRING1: count, TYPESTRING2: count, ...},
+                   "ctime_by_day": {'YYYY-MMM-DD': count, ...}
+
+            where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is
+            an integer with the number of nodes created that day.
+        """
+        import sqlalchemy as sa
+        from aiida.backends.djsite.querybuilder_django import dummy_model
+
+        # Get the session (uses internally aldjemy - so, sqlalchemy) also for the Djsite backend
+        s = dummy_model.get_aldjemy_session()
+
+        retdict = {}
+
+        # Total number of nodes
+        retdict["total"] = s.query(dummy_model.DbNode).count()
+
+        # Nodes per type
+        retdict["types"] = dict(s.query(dummy_model.DbNode.type.label('typestring'),
+            sa.func.count(dummy_model.DbNode.id)).group_by('typestring').all())
+
+        # Nodes created per day
+        stat = s.query(sa.func.date_trunc('day', dummy_model.DbNode.ctime).label('cday'),
+                       sa.func.count(dummy_model.DbNode.id)).group_by('cday').order_by('cday').all()
+
+        ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat}
+        retdict["ctime_by_day"] = ctime_by_day
+
+        return retdict
+        # Still not containing all dates

aiida/backends/general/abstractqueries.py

@@ -16,6 +16,8 @@ class AbstractQueryManager(object):
 
     def __init__(self,  *args, **kwargs):
         pass
+
+
     # This is an example of a query that could be overriden by a better implementation,
     # for performance reasons:
     def query_jobcalculations_by_computer_user_state(
@@ -106,3 +108,83 @@ def query_jobcalculations_by_computer_user_state(
             returnresult = zip(*returnresult)[0]
         return returnresult
 
+
+    def get_creation_statistics(
+            self,
+            user_email=None
+    ):
+        """
+        Return a dictionary with the statistics of node creation, summarized by day.
+
+        :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary.
+
+        :param user_email: If None (default), return statistics for all users.
+            If an email is specified, return only the statistics for the given user.
+
+        :return: a dictionary as
+            follows::
+
+                {
+                   "total": TOTAL_NUM_OF_NODES,
+                   "types": {TYPESTRING1: count, TYPESTRING2: count, ...},
+                   "ctime_by_day": {'YYYY-MMM-DD': count, ...}
+
+            where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is
+            an integer with the number of nodes created that day.
+        """
+        from aiida.orm.querybuilder import QueryBuilder as QB
+        from aiida.orm import User, Node
+        from collections import Counter
+        import datetime
+
+        def count_statistics(dataset):
+
+            def get_statistics_dict(dataset):
+                results = {}
+                for count, typestring in sorted(
+                        (v, k) for k, v in dataset.iteritems())[::-1]:
+                    results[typestring] = count
+                return results
+
+            count_dict = {}
+
+            types = Counter([r[2] for r in dataset])
+            count_dict["types"] = get_statistics_dict(types)
+
+            ctimelist = [r[1].strftime("%Y-%m-%d") for r in dataset]
+            ctime = Counter(ctimelist)
+
+            if len(ctimelist) > 0:
+
+                # For the way the string is formatted, we can just sort it alphabetically
+                firstdate = datetime.datetime.strptime(sorted(ctimelist)[0], '%Y-%m-%d')
+                lastdate = datetime.datetime.strptime(sorted(ctimelist)[-1], '%Y-%m-%d')
+
+                curdate = firstdate
+                outdata = {}
+
+                while curdate <= lastdate:
+                    curdatestring = curdate.strftime('%Y-%m-%d')
+                    outdata[curdatestring] = ctime.get(curdatestring, 0)
+                    curdate += datetime.timedelta(days=1)
+                count_dict["ctime_by_day"] = outdata
+
+            else:
+                count_dict["ctime_by_day"] = {}
+
+            return count_dict
+
+        statistics = {}
+
+        q = QB()
+        q.append(Node, project=['id', 'ctime', 'type'], tag='node')
+        if user_email is not None:
+            q.append(User, creator_of='node', project='email', filters={'email': user_email})
+        qb_res = q.all()
+
+        # total count
+        statistics["total"] = len(qb_res)
+        statistics.update(count_statistics(qb_res))
+
+        return statistics
+

aiida/backends/sqlalchemy/queries.py

@@ -11,5 +11,58 @@
 
 
 class QueryManagerSQLA(AbstractQueryManager):
-    pass
+    """
+    SQLAlchemy implementation of custom queries, for efficiency reasons
+    """
+
+    def get_creation_statistics(
+            self,
+            user_email=None
+    ):
+        """
+        Return a dictionary with the statistics of node creation, summarized by day,
+        optimized for the Django backend.
+
+        :note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary.
+
+        :param user_email: If None (default), return statistics for all users.
+            If an email is specified, return only the statistics for the given user.
+
+        :return: a dictionary as
+            follows::
+
+                {
+                   "total": TOTAL_NUM_OF_NODES,
+                   "types": {TYPESTRING1: count, TYPESTRING2: count, ...},
+                   "ctime_by_day": {'YYYY-MMM-DD': count, ...}
+
+            where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is
+            an integer with the number of nodes created that day.
+        """
+        import sqlalchemy as sa
+        import aiida.backends.sqlalchemy
+        from aiida.backends.sqlalchemy import models as m
+
+        # Get the session (uses internally aldjemy - so, sqlalchemy) also for the Djsite backend
+        s = aiida.backends.sqlalchemy.get_scoped_session()
+
+        retdict = {}
+
+        # Total number of nodes
+        retdict["total"] = s.query(m.node.DbNode).count()
+
+        # Nodes per type
+        retdict["types"] = dict(s.query(m.node.DbNode.type.label('typestring'),
+            sa.func.count(m.node.DbNode.id)).group_by('typestring').all())
+
+        # Nodes created per day
+        stat = s.query(sa.func.date_trunc('day', m.node.DbNode.ctime).label('cday'),
+                       sa.func.count(m.node.DbNode.id)).group_by('cday').order_by('cday').all()
+
+        ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat}
+        retdict["ctime_by_day"] = ctime_by_day
+
+        return retdict
+        # Still not containing all dates
+
 

aiida/backends/tests/export_and_import.py

@@ -197,6 +197,42 @@ def setUp(self):
     def tearDown(self):
         pass
 
+    def test_0(self):
+        import os
+        import shutil
+        import tempfile
+
+        from aiida.orm import DataFactory
+        from aiida.orm import load_node
+        from aiida.orm.data.base import Str, Int, Float, Bool
+        from aiida.orm.calculation.job import JobCalculation
+        from aiida.orm.importexport import export
+
+        # Creating a folder for the import/export files
+        temp_folder = tempfile.mkdtemp()
+        try:
+            # producing values for each base type
+            values = ("Hello", 6, -1.2399834e12, False) #, ["Bla", 1, 1e-10])
+            filename = os.path.join(temp_folder, "export.tar.gz")
+
+            # producing nodes:
+            nodes = [cls(val).store() for val, cls in zip(values, (Str, Int, Float, Bool))]
+            # my uuid - list to reload the node:
+            uuids = [n.uuid for n in nodes]
+            # exporting the nodes:
+            export([n.dbnode for n in nodes], outfile=filename, silent=True)
+            # cleaning:
+            self.clean_db()
+            # Importing back the data:
+            import_data(filename, silent=True)
+            # Checking whether values are preserved:
+            for uuid, refval in zip(uuids, values):
+                self.assertEquals(load_node(uuid).value, refval)
+        finally:
+            # Deleting the created temporary folder
+            shutil.rmtree(temp_folder, ignore_errors=True)
+
+
     def test_1(self):
         import os
         import shutil

aiida/backends/tests/query.py

@@ -861,5 +861,99 @@ def test_create_node_and_query(self):
         self.assertEqual(idx,99)
         self.assertTrue(len(QueryBuilder().append(Node,project=['id','label']).all(batch_size=10)) > 99)
 
+class TestStatisticsQuery(AiidaTestCase):
+    def test_statistics(self):
+        """
+        Test if the statistics query works properly.
+
+        I try to implement it in a way that does not depend on the past state.
+        """
+        from aiida.backends.utils import QueryFactory
+        from aiida.orm import Node, DataFactory, Calculation
+        from collections import defaultdict
+
+        def store_and_add(n, statistics):
+            n.store()
+            statistics['total'] += 1
+            statistics['types'][n._plugin_type_string] += 1
+            statistics['ctime_by_day'][n.ctime.strftime('%Y-%m-%d')] += 1
+
+        qmanager = QueryFactory()()
+        current_db_statistics = qmanager.get_creation_statistics()
+        types = defaultdict(int)
+        types.update(current_db_statistics['types'])
+        ctime_by_day = defaultdict(int)
+        ctime_by_day.update(current_db_statistics['ctime_by_day'])
+
+        expected_db_statistics = {
+            'total': current_db_statistics['total'],
+            'types': types,
+            'ctime_by_day': ctime_by_day
+            }
+
+        ParameterData = DataFactory('parameter')
+
+        store_and_add(Node(), expected_db_statistics)
+        store_and_add(ParameterData(), expected_db_statistics)
+        store_and_add(ParameterData(), expected_db_statistics)
+        store_and_add(Calculation(), expected_db_statistics)
+
+        new_db_statistics = qmanager.get_creation_statistics()
+        # I only check a few fields
+        new_db_statistics = {k: v for k, v in new_db_statistics.iteritems() if k in expected_db_statistics}
+
+        expected_db_statistics = {k: dict(v) if isinstance(v, defaultdict) else v
+            for k, v in expected_db_statistics.iteritems()}
+
+        self.assertEquals(new_db_statistics, expected_db_statistics)
+
+
+    def test_statistics_default_class(self):
+        """
+        Test if the statistics query works properly.
+
+        I try to implement it in a way that does not depend on the past state.
+        """
+        from aiida.orm import Node, DataFactory, Calculation
+        from collections import defaultdict
+        from aiida.backends.general.abstractqueries import AbstractQueryManager
+
+        def store_and_add(n, statistics):
+            n.store()
+            statistics['total'] += 1
+            statistics['types'][n._plugin_type_string] += 1
+            statistics['ctime_by_day'][n.ctime.strftime('%Y-%m-%d')] += 1
+
+        class QueryManagerDefault(AbstractQueryManager):
+            pass
+
+        qmanager_default = QueryManagerDefault()
+
+        current_db_statistics = qmanager_default.get_creation_statistics()
+        types = defaultdict(int)
+        types.update(current_db_statistics['types'])
+        ctime_by_day = defaultdict(int)
+        ctime_by_day.update(current_db_statistics['ctime_by_day'])
+
+        expected_db_statistics = {
+            'total': current_db_statistics['total'],
+            'types': types,
+            'ctime_by_day': ctime_by_day
+            }
+
+        ParameterData = DataFactory('parameter')
+
+        store_and_add(Node(), expected_db_statistics)
+        store_and_add(ParameterData(), expected_db_statistics)
+        store_and_add(ParameterData(), expected_db_statistics)
+        store_and_add(Calculation(), expected_db_statistics)
+
+        new_db_statistics = qmanager_default.get_creation_statistics()
+        # I only check a few fields
+        new_db_statistics = {k: v for k, v in new_db_statistics.iteritems() if k in expected_db_statistics}
+
+        expected_db_statistics = {k: dict(v) if isinstance(v, defaultdict) else v
+            for k, v in expected_db_statistics.iteritems()}
 
+        self.assertEquals(new_db_statistics, expected_db_statistics)
 

aiida/cmdline/commands/__init__.py

@@ -11,7 +11,8 @@
 
 
 @click.group()
-def verdi():
+@click.option('--profile', '-p')
+def verdi(profile):
     pass
 
 

aiida/cmdline/commands/profile.py

@@ -75,13 +75,23 @@ def profile_setdefault(self, *args):
 
     def profile_list(self, *args):
         from aiida.common.setup import get_profiles_list, get_default_profile, AIIDA_CONFIG_FOLDER
+        from aiida.common.exceptions import ConfigurationError
+
         from aiida.backends import settings
 
         print('Configuration folder: {}'.format(AIIDA_CONFIG_FOLDER))
 
         current_profile = settings.AIIDADB_PROFILE
-        default_profile = get_default_profile(
+        try:
+            default_profile = get_default_profile(
                 settings.CURRENT_AIIDADB_PROCESS)
+        except ConfigurationError as e:
+            err_msg = (
+                "Stopping: {}\n"
+                "Note: if no configuration file was found, it means that you have not run\n" 
+            "'verdi setup' yet to configure at least one AiiDA profile.".format(e.message))
+            click.echo(err_msg, err=True)
+            sys.exit(1)
         default_daemon_profile = get_default_profile("daemon")
         if current_profile is None:
             current_profile = default_profile