diff --git a/pip/index.py b/pip/index.py index 64059bc235e..415c2ff7a74 100644 --- a/pip/index.py +++ b/pip/index.py @@ -40,7 +40,7 @@ class PackageFinder(object): """ def __init__(self, find_links, index_urls, - use_mirrors=False, mirrors=None, main_mirror_url=None): + use_mirrors=False, mirrors=None, main_mirror_url=None): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] @@ -53,6 +53,13 @@ def __init__(self, find_links, index_urls, else: self.mirror_urls = [] + # Assign a preference value to all index, mirror, or find_links + # package sources. The lower the number, the higher the preference. + all_origins = self.find_links + self.mirror_urls + self.index_urls + self.origin_preferences = dict([(str(e[1]), e[0]) for e in enumerate(all_origins)]) + logger.debug('Origins: %s, %s, %s' % (self.index_urls, self.mirror_urls, self.find_links)) + logger.debug('Origin preferences: %s' % self.origin_preferences) + def add_dependency_links(self, links): ## FIXME: this shouldn't be global list this, it should only ## apply to requirements of the package that specifies the @@ -102,7 +109,7 @@ def sort_path(path): def find_requirement(self, req, upgrade): - def mkurl_pypi_url(url): + def mkurl_pypi_url(url, url_name): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec @@ -117,7 +124,7 @@ def mkurl_pypi_url(url): main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: - main_index_url = Link(mkurl_pypi_url(self.index_urls[0])) + main_index_url = Link(mkurl_pypi_url(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: @@ -129,26 +136,30 @@ def mkurl_pypi_url(url): if url_name is not None: locations = [ - mkurl_pypi_url(url) + mkurl_pypi_url(url, url_name) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: - locations = [ - posixpath.join(main_index_url.url, version)] + locations + locations = [posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] + logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( - [Link(url, '-f') for url in self.find_links], req.name.lower())) + [Link(url, '-f') for url in self.find_links], + req.name.lower() + ) + ) + page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) @@ -157,56 +168,80 @@ def mkurl_pypi_url(url): page_versions.extend(self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 + + # sort page versions by origin + page_versions = sorted(page_versions, key=self._origin_key) + dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) file_versions = list(self._package_versions( - [Link(url) for url in file_locations], req.name.lower())) + [Link(url) for url in file_locations], req.name.lower()) + ) + if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) + installed_version = [] if req.satisfied_by is not None: - installed_version = [(req.satisfied_by.parsed_version, InfLink, req.satisfied_by.version)] + installed_version = [(req.satisfied_by.parsed_version, InfLink, + req.satisfied_by.version)] + if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) - #this is an intentional priority ordering + + # This is an intentional priority ordering: any installed version, then + # archives, packages found via find_links, packages found in indexes, + # and finally dependencies. all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] + for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue + applicable_versions.append((parsed_version, link, version)) - #bring the latest version to the front, but maintains the priority ordering as secondary + + # prioritize files over eggs + applicable_versions = sorted(applicable_versions, key=self._egg_key) + # sort by version applicable_versions = sorted(applicable_versions, key=lambda v: v[0], reverse=True) - existing_applicable = bool([link for parsed_version, link, version in applicable_versions if link is InfLink]) - if not upgrade and existing_applicable: - if applicable_versions[0][1] is InfLink: - logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' - % req.satisfied_by.version) - else: - logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' - % (req.satisfied_by.version, applicable_versions[0][2])) + + logger.debug("Applicable versions: %s" % applicable_versions) + + if req.satisfied_by is not None and not upgrade: + logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' + % (req.satisfied_by.version, applicable_versions[0][2])) return None + if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in all_versions]))) raise DistributionNotFound('No distributions matching the version for %s' % req) - if applicable_versions[0][1] is InfLink: - # We have an existing version, and its the best version - logger.info('Installed version (%s) is most up-to-date (past versions: %s)' - % (req.satisfied_by.version, ', '.join([version for parsed_version, link, version in applicable_versions[1:]]) or 'none')) + + if req.satisfied_by and req.satisfied_by.parsed_version == applicable_versions[0][0]: + # We have an existing version, and it's the best version + past_versions = ', '.join( + [version for parsed_version, link, version in applicable_versions[1:]] + ) + logger.info( + 'Installed version (%s) is most up-to-date (past versions: %s)' % ( + req.satisfied_by.version, past_versions or 'none' + ) + ) raise BestVersionAlreadyInstalled + + version_choices = '' if len(applicable_versions) > 1: - logger.info('Using version %s (newest of versions: %s)' % - (applicable_versions[0][2], ', '.join([version for parsed_version, link, version in applicable_versions]))) + version_choices = ' (best of:\n %s\n )' % '\n '.join([str(v) for v in applicable_versions]) + logger.info('Using version %s from %s%s\n' % (applicable_versions[0][1].comes_from, applicable_versions[0][0], version_choices)) return applicable_versions[0][1] - def _find_url_name(self, index_url, url_name, req): """Finds the true URL name of a package, when the given name isn't quite correct. This is usually used to implement case-insensitivity.""" @@ -266,21 +301,35 @@ def _get_queued_page(self, req, pending_queue, done, seen): _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I) _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') - def _sort_links(self, links): - "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" - eggs, no_eggs = [], [] - seen = set() - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs + def _egg_key(self, v): + """ + Sorts available versions, giving preference to files over eggs. + """ + parsed_version, link, version = v + if link is InfLink: + return True + return bool(link.egg_fragment) + + def _origin_key(self, v): + """ + Sorts versions found in indexes, mirrors, or pages searched with + find_links by their origin. + """ + + lowest_preference = float('-inf') + preference = lowest_preference + + parsed_version, link, version = v + + for origin in self.origin_preferences.keys(): + if link.url.startswith(origin): + preference = self.origin_preferences[origin] + break + + return preference def _package_versions(self, links, search_name): - for link in self._sort_links(links): + for link in set(links): for v in self._link_package_versions(link, search_name): yield v @@ -327,9 +376,9 @@ def _link_package_versions(self, link, search_name): logger.debug('Skipping %s because Python version is incorrect' % link) return [] logger.debug('Found link %s, version: %s' % (link, version)) - return [(pkg_resources.parse_version(version), - link, - version)] + return [ + (pkg_resources.parse_version(version), link, version) + ] def _egg_info_matches(self, egg_info, search_name, link): match = self._egg_info_re.search(egg_info) @@ -393,7 +442,7 @@ def set_is_archive(self, url, value=True): self._archives[url] = value def add_page_failure(self, url, level): - self._failures[url] = self._failures.get(url, 0)+level + self._failures[url] = self._failures.get(url, 0) + level def add_page(self, urls, page): for url in urls: @@ -408,7 +457,7 @@ class HTMLPage(object): _download_re = re.compile(r'\s*download\s+url', re.I) ## These aren't so aweful: _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) - _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) + _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I | re.S) _base_re = re.compile(r"""]+)""", re.I) def __init__(self, content, url, headers=None): @@ -482,7 +531,7 @@ def get_page(cls, link, req, cache=None, skip_archives=True): desc = str(e) if isinstance(e, socket.timeout): log_meth = logger.info - level =1 + level = 1 desc = 'timed out' elif isinstance(e, URLError): log_meth = logger.info @@ -677,8 +726,8 @@ def hash_name(self): def show_url(self): return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) -#An "Infinite Link" that compares greater than other links -InfLink = Link(Inf) #this object is not currently used as a sortable +# An "Infinite Link" that compares greater than other links +InfLink = Link(Inf) # this object is not currently used as a sortable def get_requirement_from_url(url): @@ -740,9 +789,8 @@ def string_range(last): This works for simple "a to z" lists, but also for "a to zz" lists. """ for k in range(len(last)): - for x in product(string.ascii_lowercase, repeat=k+1): + for x in product(string.ascii_lowercase, repeat=k + 1): result = ''.join(x) yield result if result == last: return - diff --git a/pip/req.py b/pip/req.py index 16c3cff6abc..ea5e340ce89 100644 --- a/pip/req.py +++ b/pip/req.py @@ -45,6 +45,12 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, self.extras = req.extras self.req = req self.comes_from = comes_from + + # origin is used just to record provenance; comes_from is also used to + # make decisions about the type of packaging and setting it can break + # things + self.origin = None + self.source_dir = source_dir self.editable = editable self.url = url @@ -1031,6 +1037,7 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): if url: try: self.unpack_url(url, location, self.is_download) + req_to_install.origin = url.url except HTTPError: e = sys.exc_info()[1] logger.fatal('Could not install requirement %s because of error %s' @@ -1173,6 +1180,7 @@ def install(self, install_options, global_options=(), *args, **kwargs): logger.indent += 2 try: for requirement in to_install: + logger.debug('Installing %s%s' % (requirement.name, requirement.origin and (' (origin: %s)' % requirement.origin) or requirement.comes_from and ' %s' % requirement.comes_from or '')) if requirement.conflicts_with: logger.notify('Found existing installation: %s' % requirement.conflicts_with) diff --git a/tests/test_finder.py b/tests/test_finder.py index 415e9a9dce9..f743e09e3c2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -48,7 +48,7 @@ def test_finder_detects_latest_find_links(): def test_finder_detects_latest_already_satisfied_find_links(): - """Test PackageFinder detects latest already satisified using find-links""" + """Test PackageFinder detects latest already satisfied using find-links""" req = InstallRequirement.from_line('simple', None) #the latest simple in local pkgs is 3.0 latest_version = "3.0" @@ -63,7 +63,7 @@ def test_finder_detects_latest_already_satisfied_find_links(): def test_finder_detects_latest_already_satisfied_pypi_links(): - """Test PackageFinder detects latest already satisified using pypi links""" + """Test PackageFinder detects latest already satisfied using pypi links""" req = InstallRequirement.from_line('initools', None) #the latest initools on pypi is 0.3.1 latest_version = "0.3.1" @@ -97,7 +97,7 @@ def test_finder_priority_page_over_deplink(): def test_finder_priority_nonegg_over_eggfragments(): """Test PackageFinder prefers non-egg links over "#egg=" links""" req = InstallRequirement.from_line('bar==1.0', None) - links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz'] + links = ['file://foo/bar.py#egg=bar-1.0', 'file://foo/bar-1.0.tar.gz'] finder = PackageFinder(links, []) link = finder.find_requirement(req, False) diff --git a/tests/test_freeze.py b/tests/test_freeze.py index 7e2a8b3eae5..d2269f8d7b1 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -46,7 +46,7 @@ def test_freeze_basic(): write_file('initools-req.txt', textwrap.dedent("""\ INITools==0.2 # and something else to test out: - MarkupSafe<=0.12 + MarkupSafe==0.12 """)) result = run_pip('install', '-r', env.scratch_path/'initools-req.txt') result = run_pip('freeze', expect_stderr=True) diff --git a/tests/test_index_preference.py b/tests/test_index_preference.py new file mode 100644 index 00000000000..da6cafa4fdd --- /dev/null +++ b/tests/test_index_preference.py @@ -0,0 +1,52 @@ +import os +import re +import shutil + +from test_pip import here, reset_env, run_pip + + +index_template = """ + + simple-1.0.tar.gz + + +""" + + +def test_index_preference(): + """ + Verify that indexes will be used in the order defined. + """ + e = reset_env() + + # create two indexes + index1 = e.scratch_path / 'index1' + os.makedirs(index1 / 'simple') + index = open(index1 / 'simple/index.html', 'w') + index.write(index_template) + index.close() + shutil.copy(here / 'packages/simple-1.0.tar.gz', index1 / 'simple') + + index2 = e.scratch_path / 'index2' + os.makedirs(index2 / 'simple') + index = open(index2 / 'simple/index.html', 'w') + index.write(index_template) + index.close() + shutil.copy(here / 'packages/simple-1.0.tar.gz', index2 / 'simple') + + # verify that the package is installed from the main index (index1) + result = run_pip('install', '-vv', '--index-url', 'file://' + index1, '--extra-index-url', 'file://' + index2, 'simple==1.0', expect_error=False) + output = result.stdout + + index1_re = re.compile('^\s*Installing simple \(origin: file://.*/scratch/index1/simple/simple-1.0.tar.gz', re.MULTILINE) + assert index1_re.search(output) is not None + + # uninstall the package + result = run_pip('uninstall', '--yes', 'simple', expect_error=False) + + # verify that the package is installed from the main index (index2, now) + result = run_pip('install', '-vv', '--index-url', 'file://' + index2, '--extra-index-url', 'file://' + index1, 'simple==1.0', expect_error=False) + output = result.stdout + + index2_re = re.compile('^\s*Installing simple \(origin: file://.*/scratch/index2/simple/simple-1.0.tar.gz', re.MULTILINE) + assert index2_re.search(output) is not None diff --git a/tests/test_upgrade.py b/tests/test_upgrade.py index d67c02865fd..0bd4d302557 100644 --- a/tests/test_upgrade.py +++ b/tests/test_upgrade.py @@ -1,4 +1,6 @@ import textwrap +import time + from os.path import join from nose.tools import nottest from tests.test_pip import (here, reset_env, run_pip, assert_all_changes, @@ -66,6 +68,9 @@ def test_upgrade_force_reinstall_newest(): env = reset_env() result = run_pip('install', 'INITools') assert env.site_packages/ 'initools' in result.files_created, sorted(result.files_created.keys()) + + time.sleep(1) # ensure mtimes change, so files_updated is accurate + result2 = run_pip('install', '--upgrade', '--force-reinstall', 'INITools') assert result2.files_updated, 'upgrade to INITools 0.3 failed' result3 = run_pip('uninstall', 'initools', '-y', expect_error=True) @@ -109,6 +114,9 @@ def test_upgrade_to_same_version_from_url(): env = reset_env() result = run_pip('install', 'INITools==0.3', expect_error=True) assert env.site_packages/ 'initools' in result.files_created, sorted(result.files_created.keys()) + + time.sleep(1) # ensure mtimes change, so files_updated is accurate + result2 = run_pip('install', 'http://pypi.python.org/packages/source/I/INITools/INITools-0.3.tar.gz', expect_error=True) assert not result2.files_updated, 'INITools 0.3 reinstalled same version' result3 = run_pip('uninstall', 'initools', '-y', expect_error=True)