From 616dc9cc22fedb7a6383fc4189539a74a993dd3d Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 1 Sep 2017 01:40:17 -0600 Subject: [PATCH] Leverage `pex.common.open_zip`. (#409) Also use in one spot in the `pex.testing` module where we weren't before. --- .travis.yml | 2 ++ pex/common.py | 2 +- pex/pex_bootstrapper.py | 6 +++--- pex/testing.py | 9 ++++----- tests/test_pex_bootstrapper.py | 5 ++--- tests/test_pex_builder.py | 5 ++--- tests/test_util.py | 6 ++---- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index a373b37dd..ede3e4f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ # "machines". # See: http://docs.travis-ci.com/user/workers/container-based-infrastructure/ sudo: false +# The pypy test shard fails with "too many open files" under Trusty so we pin to Precise. +dist: precise # TRAVIS_PYTHON_VERSION diff --git a/pex/common.py b/pex/common.py index d62c2da0a..546e4a31f 100644 --- a/pex/common.py +++ b/pex/common.py @@ -332,6 +332,6 @@ def delete(self): shutil.rmtree(self.chroot) def zip(self, filename, mode='wb'): - with contextlib.closing(zipfile.ZipFile(filename, mode)) as zf: + with open_zip(filename, mode) as zf: for f in sorted(self.files()): zf.write(os.path.join(self.chroot, f), arcname=f, compress_type=zipfile.ZIP_DEFLATED) diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 65a481f31..c3510a84b 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -1,10 +1,10 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import contextlib import os import sys -import zipfile + +from .common import open_zip __all__ = ('bootstrap_pex',) @@ -24,7 +24,7 @@ def read_pexinfo_from_directory(entry_point): def read_pexinfo_from_zip(entry_point): - with contextlib.closing(zipfile.ZipFile(entry_point)) as zf: + with open_zip(entry_point) as zf: return zf.read('PEX-INFO') diff --git a/pex/testing.py b/pex/testing.py index d49b52f38..215071eb4 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -6,12 +6,11 @@ import random import sys import tempfile -import zipfile from collections import namedtuple from textwrap import dedent from .bin.pex import log, main -from .common import safe_mkdir, safe_rmtree +from .common import open_zip, safe_mkdir, safe_rmtree from .compatibility import nested from .executor import Executor from .installer import EggInstaller, Packager @@ -48,7 +47,7 @@ def random_bytes(length): def get_dep_dist_names_from_pex(pex_path, match_prefix=''): """Given an on-disk pex, extract all of the unique first-level paths under `.deps`.""" - with zipfile.ZipFile(pex_path) as pex_zip: + with open_zip(pex_path) as pex_zip: dep_gen = (f.split(os.sep)[1] for f in pex_zip.namelist() if f.startswith('.deps/')) return set(item for item in dep_gen if item.startswith(match_prefix)) @@ -80,7 +79,7 @@ def yield_files(directory): def write_zipfile(directory, dest, reverse=False): - with contextlib.closing(zipfile.ZipFile(dest, 'w')) as zf: + with open_zip(dest, 'w') as zf: for filename, rel_filename in sorted(yield_files(directory), reverse=reverse): zf.write(filename, arcname=rel_filename) return dest @@ -152,7 +151,7 @@ def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, else: with temporary_dir() as td: extract_path = os.path.join(td, os.path.basename(dist_location)) - with contextlib.closing(zipfile.ZipFile(dist_location)) as zf: + with open_zip(dist_location) as zf: zf.extractall(extract_path) yield DistributionHelper.distribution_from_path(extract_path) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index f1aa11ec3..27b38f7f9 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -2,11 +2,10 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import zipfile -from contextlib import closing from twitter.common.contextutil import temporary_dir +from pex.common import open_zip from pex.pex_bootstrapper import get_pex_info from pex.testing import write_simple_pex @@ -21,7 +20,7 @@ def test_get_pex_info(): pex_info = get_pex_info(pex_path) with temporary_dir() as pex_td: - with closing(zipfile.ZipFile(pex_path, 'r')) as zf: + with open_zip(pex_path, 'r') as zf: zf.extractall(pex_td) # from dir diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index 9521ab371..af70830f9 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -4,13 +4,12 @@ import os import stat import sys -import zipfile -from contextlib import closing import pytest from twitter.common.contextutil import temporary_dir from twitter.common.dirutil import safe_mkdir +from pex.common import open_zip from pex.compatibility import WINDOWS, nested from pex.pex import PEX from pex.pex_builder import PEXBuilder @@ -55,7 +54,7 @@ def test_pex_builder(): td1, td2, p1): target_egg_dir = os.path.join(td2, os.path.basename(p1.location)) safe_mkdir(target_egg_dir) - with closing(zipfile.ZipFile(p1.location, 'r')) as zf: + with open_zip(p1.location, 'r') as zf: zf.extractall(target_egg_dir) p1 = DistributionHelper.distribution_from_path(target_egg_dir) diff --git a/tests/test_util.py b/tests/test_util.py index 7cf11e542..e0377a450 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,16 +1,14 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import contextlib import functools import os -import zipfile from hashlib import sha1 from textwrap import dedent from twitter.common.contextutil import temporary_dir -from pex.common import safe_mkdir +from pex.common import open_zip, safe_mkdir from pex.compatibility import nested, to_bytes from pex.installer import EggInstaller, WheelInstaller from pex.pex_builder import PEXBuilder @@ -60,7 +58,7 @@ def test_hash_consistency(): dir_hash = CacheHelper.dir_hash(td) with named_temporary_file() as tf: write_zipfile(td, tf.name, reverse=reverse) - with contextlib.closing(zipfile.ZipFile(tf.name, 'r')) as zf: + with open_zip(tf.name, 'r') as zf: zip_hash = CacheHelper.zip_hash(zf) assert zip_hash == dir_hash assert zip_hash != sha1().hexdigest() # make sure it's not an empty hash