diff --git a/.prospector.yaml b/.prospector.yaml index 07d7db638c..236c2b4104 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -1,14 +1,16 @@ +max-line-length: 120 + ignore-paths: - doc - examples - test - utils -pylint: - max-line-length: 140 - pyflakes: run: false +pep8: + run: false + mccabe: run: false diff --git a/aiida/backends/tests/verdi_commands.py b/aiida/backends/tests/verdi_commands.py index 1aad2b21b9..6ab3256787 100644 --- a/aiida/backends/tests/verdi_commands.py +++ b/aiida/backends/tests/verdi_commands.py @@ -125,23 +125,27 @@ def test_calculation_list(self): calc_cmd.calculation_list() out_str = ''.join(output) - self.assertTrue(calc_states.TOSUBMIT in out_str, - "The TOSUBMIT calculations should be part fo the " - "simple calculation list.") - self.assertTrue(calc_states.COMPUTED in out_str, - "The COMPUTED calculations should be part fo the " - "simple calculation list.") - self.assertFalse(calc_states.FINISHED in out_str, - "The FINISHED calculations should not be part fo the " - "simple calculation list.") + self.assertTrue( + calc_states.TOSUBMIT in out_str, + "The TOSUBMIT calculations should be part fo the " + "simple calculation list.") + self.assertTrue( + calc_states.COMPUTED in out_str, + "The COMPUTED calculations should be part fo the " + "simple calculation list.") + self.assertFalse( + calc_states.FINISHED in out_str, + "The FINISHED calculations should not be part fo the " + "simple calculation list.") with Capturing() as output: calc_cmd.calculation_list(*['-a']) out_str = ''.join(output) - self.assertTrue(calc_states.FINISHED in out_str, - "The FINISHED calculations should be part fo the " - "simple calculation list.") + self.assertTrue( + calc_states.FINISHED in out_str, + "The FINISHED calculations should be part fo the " + "simple calculation list.") class TestVerdiCodeCommands(AiidaTestCase): @@ -193,12 +197,14 @@ def test_code_list(self): with Capturing() as output: code_cmd.code_list() out_str_1 = ''.join(output) - self.assertTrue(computer_name_1 in out_str_1, - "The computer 1 name should be included into this list") + self.assertTrue( + computer_name_1 in out_str_1, + "The computer 1 name should be included into this list") self.assertTrue(code_name_1 in out_str_1, "The code 1 name should be included into this list") - self.assertTrue(computer_name_2 in out_str_1, - "The computer 2 name should be included into this list") + self.assertTrue( + computer_name_2 in out_str_1, + "The computer 2 name should be included into this list") self.assertTrue(code_name_2 in out_str_1, "The code 2 name should be included into this list") @@ -207,16 +213,18 @@ def test_code_list(self): with Capturing() as output: code_cmd.code_list(*['-a']) out_str_2 = ''.join(output) - self.assertEqual(out_str_1, out_str_2, - "verdi code list & verdi code list -a should provide " - "the same output in this experiment.") + self.assertEqual( + out_str_1, out_str_2, + "verdi code list & verdi code list -a should provide " + "the same output in this experiment.") # Run a verdi code list -c, capture the output and check the result with Capturing() as output: code_cmd.code_list(*['-c', computer_name_1]) out_str = ''.join(output) - self.assertTrue(computer_name_1 in out_str, - "The computer 1 name should be included into this list") + self.assertTrue( + computer_name_1 in out_str, + "The computer 1 name should be included into this list") self.assertFalse( computer_name_2 in out_str, "The computer 2 name should not be included into this list") @@ -228,8 +236,9 @@ def test_code_list(self): with Capturing() as output: code_cmd.code_list() out_str_3 = ''.join(output) - self.assertTrue(computer_name_1 in out_str_3, - "The computer 1 name should be included into this list") + self.assertTrue( + computer_name_1 in out_str_3, + "The computer 1 name should be included into this list") self.assertTrue(code_name_1 in out_str_3, "The code 1 name should be included into this list") self.assertFalse( @@ -665,12 +674,12 @@ def test_trajectory_simple_listing(self): for nid in self.cmd_to_nodeid_map[sub_cmd]: if str(nid) not in out_str: - self.fail("The data objects ({}) with ids {} and {} " - "were not found. " - .format(sub_cmd, - str(self.cmd_to_nodeid_map[sub_cmd][0]), - str(self.cmd_to_nodeid_map[sub_cmd][1])) + - "The output was {}".format(out_str)) + self.fail( + "The data objects ({}) with ids {} and {} " + "were not found. ".format( + sub_cmd, str(self.cmd_to_nodeid_map[sub_cmd][0]), + str(self.cmd_to_nodeid_map[sub_cmd][1])) + + "The output was {}".format(out_str)) def test_trajectory_all_user_listing(self): from aiida.cmdline.commands.data import _Bands @@ -742,10 +751,10 @@ def test_trajectory_group_listing(self): from aiida.cmdline.commands.data import _Cif from aiida.cmdline.commands.data import _Trajectory - args_to_test = [['-g', self.group_name], [ - '--group-name', self.group_name - ], ['-G', str(self.group_id)], ['--group-pk', - str(self.group_id)]] + args_to_test = [['-g', self.group_name], + ['--group-name', self.group_name], + ['-G', str(self.group_id)], + ['--group-pk', str(self.group_id)]] sub_cmds = [_Bands, _Structure, _Cif, _Trajectory] for sub_cmd in sub_cmds: @@ -755,8 +764,8 @@ def test_trajectory_group_listing(self): curr_scmd.list(*arg) out_str = ' '.join(output) - if str(self.cmd_to_nodeid_map_for_groups[ - sub_cmd]) not in out_str: + if str(self. + cmd_to_nodeid_map_for_groups[sub_cmd]) not in out_str: self.fail( "The data object ({}) with id {} " "was not found. ".format( diff --git a/aiida/control/postgres.py b/aiida/control/postgres.py index 36d3eb38af..b988185809 100644 --- a/aiida/control/postgres.py +++ b/aiida/control/postgres.py @@ -15,6 +15,7 @@ installed by default on various systems. If the postgres setup is not the default installation, additional information needs to be provided. """ + try: import subprocess32 as subprocess except ImportError: @@ -64,17 +65,24 @@ class Postgres(object): print('setup sucessful!') """ - def __init__(self, port=None, interactive=False, quiet=True): + def __init__(self, + host='localhost', + port=None, + interactive=False, + quiet=True): self.interactive = interactive self.quiet = quiet self.pg_execute = _pg_execute_not_connected self.dbinfo = {} - if port: - self.set_port(port) self.setup_fail_callback = None self.setup_fail_counter = 0 self.setup_max_tries = 1 + if host: + self.set_host(host) + if port: + self.set_port(port) + def set_setup_fail_callback(self, callback): """ Set a callback to be called when setup cannot be determined automatically @@ -83,6 +91,10 @@ def set_setup_fail_callback(self, callback): """ self.setup_fail_callback = callback + def set_host(self, host): + """Set the host manually""" + self.dbinfo['host'] = host + def set_port(self, port): """Set the port manually""" self.dbinfo['port'] = str(port) @@ -106,13 +118,19 @@ def determine_setup(self): self.dbinfo = local_dbinfo break - # This will work for the default Debian postgres setup + # This will work for the default Debian postgres setup, assuming that sudo is available to the user if self.pg_execute == _pg_execute_not_connected: - dbinfo['user'] = 'postgres' - if _try_subcmd( - non_interactive=bool(not self.interactive), **dbinfo): - self.pg_execute = _pg_execute_sh - self.dbinfo = dbinfo + # Check if the user can find the sudo command + if _sudo_exists(): + dbinfo['user'] = 'postgres' + if _try_subcmd( + non_interactive=bool(not self.interactive), **dbinfo): + self.pg_execute = _pg_execute_sh + self.dbinfo = dbinfo + else: + click.echo( + 'Warning: Could not find `sudo`. No way of connecting to the database could be found.' + ) # This is to allow for any other setup if self.pg_execute == _pg_execute_not_connected: @@ -192,9 +210,9 @@ def db_exists(self, dbname): def _no_setup_detected(self): """Print a warning message and calls the failed setup callback""" message = '\n'.join([ - 'Detected no known postgres setup, some information is needed to create the aiida database and grant aiida access to it.', - 'If you feel unsure about the following parameters, first check if postgresql is installed.', - 'If postgresql is not installed please exit and install it, then run verdi quicksetup again.', + 'Detected no known postgres setup, some information is needed to create the aiida database and grant ', + 'aiida access to it. If you feel unsure about the following parameters, first check if postgresql is ', + 'installed. If postgresql is not installed please exit and install it, then run verdi quicksetup again.', 'If postgresql is installed, please ask your system manager to provide you with the following parameters:' ]) if not self.quiet: @@ -269,6 +287,22 @@ def _try_connect(**kwargs): return success +def _sudo_exists(): + """ + Check that the sudo command can be found + + :return: True if successful, False otherwise + """ + try: + subprocess.check_output(['sudo', '-V']) + except subprocess.CalledProcessError: + return False + except OSError: + return False + + return True + + def _try_subcmd(**kwargs): """ try to run psql in a subprocess. @@ -286,12 +320,12 @@ def _try_subcmd(**kwargs): def _pg_execute_psyco(command, **kwargs): - ''' + """ executes a postgres commandline through psycopg2 :param command: A psql command line as a str :param kwargs: will be forwarded to psycopg2.connect - ''' + """ from psycopg2 import connect, ProgrammingError conn = connect(**kwargs) conn.autocommit = True @@ -316,7 +350,7 @@ def _pg_execute_sh(command, user='postgres', **kwargs): :param kwargs: connection details to forward to psql, signature as in psycopg2.connect To stop `sudo` from asking for a password and fail if one is required, - pass `noninteractive=True` as a kwarg. + pass `non_interactive=True` as a kwarg. """ options = '' database = kwargs.pop('database', None) @@ -330,7 +364,7 @@ def _pg_execute_sh(command, user='postgres', **kwargs): if port: options += '-p {}'.format(port) - ## build command line + # Build command line sudo_cmd = ['sudo'] non_interactive = kwargs.pop('non_interactive', None) if non_interactive: @@ -343,10 +377,9 @@ def _pg_execute_sh(command, user='postgres', **kwargs): ] sudo_su_psql = sudo_cmd + su_cmd + psql_cmd result = subprocess.check_output(sudo_su_psql, **kwargs) + result = result.decode('utf-8').strip().split('\n') + result = [i for i in result if i] - if isinstance(result, str): - result = result.strip().split('\n') - result = [i for i in result if i] return result diff --git a/aiida/orm/data/cif.py b/aiida/orm/data/cif.py index 169820ba4a..a1884130a0 100644 --- a/aiida/orm/data/cif.py +++ b/aiida/orm/data/cif.py @@ -364,8 +364,8 @@ class CifData(SinglefileData): when setting ``ase`` or ``values``, a physical CIF file is generated first, the values are updated from the physical CIF file. """ - _set_incompatibilities = [('ase', 'file'), ('ase', 'values'), ('file', - 'values')] + _set_incompatibilities = [('ase', 'file'), ('ase', 'values'), + ('file', 'values')] _scan_types = ['standard', 'flex'] _parse_policies = ['eager', 'lazy'] @@ -453,8 +453,8 @@ def get_or_create(cls, filename, use_first=False, store_cif=True): else: raise ValueError("More than one copy of a CIF file " "with the same MD5 has been found in " - "the DB. pks={}".format( - ",".join([str(i.pk) for i in cifs]))) + "the DB. pks={}".format(",".join( + [str(i.pk) for i in cifs]))) else: return cifs[0], False diff --git a/aiida/restapi/resources.py b/aiida/restapi/resources.py index f34d687982..6dbf20cef0 100644 --- a/aiida/restapi/resources.py +++ b/aiida/restapi/resources.py @@ -24,9 +24,7 @@ def __init__(self, **kwargs): # Configure utils utils_conf_keys = ('PREFIX', 'PERPAGE_DEFAULT', 'LIMIT_DEFAULT') self.utils_confs = { - k: kwargs[k] - for k in utils_conf_keys - if k in kwargs + k: kwargs[k] for k in utils_conf_keys if k in kwargs } self.utils = Utils(**self.utils_confs) @@ -103,9 +101,7 @@ def __init__(self, **kwargs): # Configure utils utils_conf_keys = ('PREFIX', 'PERPAGE_DEFAULT', 'LIMIT_DEFAULT') self.utils_confs = { - k: kwargs[k] - for k in utils_conf_keys - if k in kwargs + k: kwargs[k] for k in utils_conf_keys if k in kwargs } self.utils = Utils(**self.utils_confs) self.method_decorators = {'get': kwargs.get('get_decorators', [])} @@ -205,9 +201,7 @@ def __init__(self, **kwargs): # Configure utils utils_conf_keys = ('PREFIX', 'PERPAGE_DEFAULT', 'LIMIT_DEFAULT') self.utils_confs = { - k: kwargs[k] - for k in utils_conf_keys - if k in kwargs + k: kwargs[k] for k in utils_conf_keys if k in kwargs } self.utils = Utils(**self.utils_confs) self.method_decorators = {'get': kwargs.get('get_decorators', [])} diff --git a/docs/update_req_for_rtd.py b/docs/update_req_for_rtd.py index 71703ff3fe..7617b582a2 100644 --- a/docs/update_req_for_rtd.py +++ b/docs/update_req_for_rtd.py @@ -26,15 +26,15 @@ def update_req_for_rtd(pre_commit): import setup_requirements - reqs = set( - setup_requirements.extras_require['docs'] + setup_requirements. - extras_require['rest'] + setup_requirements.extras_require['testing'] + - setup_requirements.extras_require[':python_version < "3"'] + - # To avoid that it requires also the postgres libraries - [ - p for p in setup_requirements.install_requires - if not p.startswith('psycopg2') - ]) + reqs = set(setup_requirements.extras_require['docs'] + + setup_requirements.extras_require['rest'] + + setup_requirements.extras_require['testing'] + + setup_requirements.extras_require[':python_version < "3"'] + + # To avoid that it requires also the postgres libraries + [ + p for p in setup_requirements.install_requires + if not p.startswith('psycopg2') + ]) reqs_str = "\n".join(sorted(reqs)) basename = 'requirements_for_rtd.txt' diff --git a/setup_requirements.py b/setup_requirements.py index 08d9538220..02334609b4 100644 --- a/setup_requirements.py +++ b/setup_requirements.py @@ -119,7 +119,7 @@ ], 'dev_precommit': [ 'pre-commit==1.13.0', - 'yapf==0.21.0', + 'yapf==0.24.0', 'prospector==0.12.7', 'pylint==1.8.4', 'toml==0.9.4'