From 619d3aaecc33ffba9c8d2cb5a9e71003b0a4e652 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Mon, 30 Aug 2021 10:13:24 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20CREATE=20DATABASE=20with?= =?UTF-8?q?=20psycopg2=20v2.9=20(#27)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses https://github.com/aiidateam/aiida-core/issues/5002: In v2.9, the context manager opens a transaction, but databases cannot be created within them. The fix is taken from: https://github.com/psycopg/psycopg2/issues/941#issuecomment-864019726 --- .github/workflows/ci.yml | 24 +++++++------------ .pre-commit-config.yaml | 5 ++++ pgsu/__init__.py | 18 +++++++------- pgsu/cli.py | 2 +- pyproject.toml | 4 ++++ setup.cfg | 51 +++++++++++++++++++++++++++++++++++++++- setup.py | 45 +++++------------------------------ tox.ini | 16 +++++++++++++ 8 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 tox.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e810c5b..71a8654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: pre-commit: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -17,15 +17,9 @@ jobs: with: python-version: 3.8 - - name: Install system dependencies - run: | - sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list - sudo apt update - sudo apt install libkrb5-dev ruby ruby-dev - - name: Install python dependencies run: | - pip install -e .[pre-commit,testing] + pip install -e .[pre_commit,testing] pip freeze - name: Run pre-commit @@ -40,7 +34,7 @@ jobs: strategy: matrix: - python-version: [3.5, 3.9] + python-version: [3.6, 3.9] steps: - uses: actions/checkout@v2 @@ -76,14 +70,14 @@ jobs: file: ./coverage.xml fail_ci_if_error: true - ubuntu-apt-1604: + ubuntu-apt-1804: # ubuntu using postgresql installed via apt (+ pgtest) - runs-on: ubuntu-16.04 + runs-on: ubuntu-18.04 timeout-minutes: 5 strategy: matrix: - python-version: [3.5] + python-version: [3.6] steps: - uses: actions/checkout@v2 @@ -97,9 +91,9 @@ jobs: run: | sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list sudo apt update - sudo apt install postgresql-12 - sudo cat /etc/postgresql/12/main/pg_hba.conf - sudo pg_ctlcluster 12 main start + sudo apt install postgresql-10 + sudo cat /etc/postgresql/10/main/pg_hba.conf + sudo pg_ctlcluster 10 main start - name: Install pgsu run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d4c97a..f9ee9d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,11 @@ repos: - id: mixed-line-ending - id: trailing-whitespace +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt + # yapf = yet another python formatter - repo: https://github.com/pre-commit/mirrors-yapf rev: v0.30.0 diff --git a/pgsu/__init__.py b/pgsu/__init__.py index 7c03db3..84cf4d3 100644 --- a/pgsu/__init__.py +++ b/pgsu/__init__.py @@ -250,16 +250,18 @@ def _execute_psyco(command, dsn): """ import psycopg2 # pylint: disable=import-outside-toplevel + conn = None output = None - with psycopg2.connect(**dsn) as conn: + try: + conn = psycopg2.connect(**dsn) conn.autocommit = True with conn.cursor() as cursor: cursor.execute(command) if cursor.description is not None: output = cursor.fetchall() - - # see http://initd.org/psycopg/docs/usage.html#with-statement - conn.close() + finally: + if conn: + conn.close() return output @@ -314,7 +316,7 @@ def _execute_su_psql(command, dsn, interactive=False, stderr=None): database = dsn.get('database') if database: - psql_option_str += '-d {}'.format(database) + psql_option_str += f'-d {database}' # to do: Forward password to psql; ignore host only when the password is None. # pylint: disable=fixme # Note: There is currently no known postgresql setup that needs this, though @@ -322,7 +324,7 @@ def _execute_su_psql(command, dsn, interactive=False, stderr=None): host = dsn.pop('host', 'localhost') if host and host != 'localhost': - psql_option_str += ' -h {}'.format(host) + psql_option_str += f' -h {host}' else: LOGGER.debug( "Found host 'localhost' but dropping '-h localhost' option for psql " @@ -331,7 +333,7 @@ def _execute_su_psql(command, dsn, interactive=False, stderr=None): port = dsn.get('port') if port: - psql_option_str += ' -p {}'.format(port) + psql_option_str += f' -p {port}' # Note: This is *both* the UNIX user to become *and* the database user user = dsn.get('user') @@ -382,7 +384,7 @@ def escape_for_bash(str_to_escape): string found below. """ escaped_quotes = str_to_escape.replace("'", """'"'"'""") - return "'{}'".format(escaped_quotes) + return f"'{escaped_quotes}'" def unique_list(non_unique_list): diff --git a/pgsu/cli.py b/pgsu/cli.py index 8066f78..62200e7 100644 --- a/pgsu/cli.py +++ b/pgsu/cli.py @@ -14,6 +14,6 @@ def run(query): """Execute SQL command as PostrgreSQL superuser.""" pgsu = PGSU(interactive=True, quiet=False) - click.echo('Executing query: {}'.format(query)) + click.echo(f'Executing query: {query}') dbs = pgsu.execute(query) click.echo(pprint.pformat(dbs)) diff --git a/pyproject.toml b/pyproject.toml index 3444549..f51938e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools>=46.4.0", "wheel"] +build-backend = "setuptools.build_meta" + [tool.pylint.format] max-line-length = 120 max-args = 7 diff --git a/setup.cfg b/setup.cfg index a158dc6..1809f01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,51 @@ +[metadata] +name = pgsu +version = 0.2.0 +description = Connect to an existing PostgreSQL cluster as a postgres superuser and execute SQL commands. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/aiidateam/pgsu +author = AiiDA Team +author_email = aiidateam@gmail.com +license = MIT +license_file = LICENSE +classifiers = + License :: OSI Approved :: MIT License + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: Implementation :: CPython + +[options] +packages = find: +install_requires = + click + psycopg2-binary>=2.8.3 +python_requires = ~=3.6 + +[options.packages.find] +exclude = + test* + +[options.entry_points] +console_scripts = + pgsu = pgsu.cli:run + +[options.extras_require] +pre_commit = + pre-commit~=2.2 + pylint~=2.5.0 +testing = + pgtest>=1.3.1 + pytest + pytest-cov + [coverage:run] -parallel=true +parallel = true diff --git a/setup.py b/setup.py index 3b15a8d..673763c 100644 --- a/setup.py +++ b/setup.py @@ -1,42 +1,9 @@ # -*- coding: utf-8 -*- -"""Setup script for pgsu python package""" -import os -from setuptools import setup, find_packages +"""This file is needed for editable installs (`pip install -e .`). -THIS_FOLDER = os.path.split(os.path.abspath(__file__))[0] +Can be removed once the following is resolved +https://github.com/pypa/packaging-problems/issues/256 +""" +from setuptools import setup -setup( - name='pgsu', - version='0.2.0', - description= - ('Connect to an existing PostgreSQL cluster as a postgres superuser and execute SQL commands.' - ), - long_description=open(os.path.join(THIS_FOLDER, 'README.md')).read(), - long_description_content_type='text/markdown', - url='https://github.com/aiidateam/pgsu', - author='AiiDA Team', - author_email='aiidateam@gmail.com', - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Operating System :: POSIX :: Linux', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - license='MIT', - packages=find_packages(), - install_requires=[ - 'psycopg2-binary>=2.8.3', - 'click', - ], - extras_require={ - 'testing': ['pytest', 'pgtest>=1.3.1', 'pytest-cov'], - # note: pre-commit hooks require python3 - 'pre-commit': ['pre-commit~=2.2', 'pylint~=2.5.0'] - }, - entry_points={'console_scripts': ['pgsu=pgsu.cli:run']}) +setup() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..84a1e9e --- /dev/null +++ b/tox.ini @@ -0,0 +1,16 @@ +# To use tox, see https://tox.readthedocs.io +# Simply pip or conda install tox +# If you use conda, you may also want to install tox-conda +# then run `tox` or `tox -- {pytest args}` +# run in parallel using `tox -p` +[tox] +envlist = py38 + +[testenv] +usedevelop = true + +[testenv:py{37,38,39}] +description = Run unit tests with this Python version +extras = + testing +commands = pytest {posargs}