Read the existing MANIFEST.in file for files to ignore.

CHANGES.rst

@@ -5,6 +5,8 @@ Changelog
 0.17 (unreleased)
 -----------------
 
+* Read the existing MANIFEST.in file for files to ignore.
+
 
 0.16 (2013-10-01)
 -----------------

check_manifest.py

@@ -350,6 +350,12 @@ def add_directories(names):
     '*.mo',
 ]
 
+IGNORE_REGEXPS = [
+    # Regular expressions for filename to ignore.  This is useful for
+    # filename patterns where the '*' part must not search in
+    # directories.
+]
+
 WARN_ABOUT_FILES_IN_VCS = [
     # generated files should not be committed into the VCS
     'PKG-INFO',
@@ -394,15 +400,86 @@ def read_config():
         IGNORE.extend(p for p in patterns if p)
 
 
+def read_manifest():
+    """Read existing configuration from MANIFEST.in.
+
+    We use that to ignore anything the MANIFEST.in ignores.
+    """
+    # XXX modifies global state, which is kind of evil
+    if not os.path.isfile('MANIFEST.in'):
+        return
+    with open('MANIFEST.in') as manifest:
+        contents = manifest.read()
+    ignore, ignore_regexps = _get_ignore_from_manifest(contents)
+    IGNORE.extend(ignore)
+    IGNORE_REGEXPS.extend(ignore_regexps)
+
+
+def _get_ignore_from_manifest(contents):
+    """Gather the various ignore patterns from MANIFEST.in.
+
+    'contents' should be a string, which may contain newlines.
+
+    Returns a list of standard ignore patterns and a list of regular
+    expressions to ignore.
+    """
+    ignore = []
+    ignore_regexps = []
+    for line in contents.splitlines():
+        try:
+            cmd, rest = line.split(None, 1)
+        except ValueError:
+            # no whitespace, so not interesting
+            continue
+        if cmd == 'exclude':
+            # An exclude of 'dirname/*css' can match 'dirname/foo.css'
+            # but not 'dirname/subdir/bar.css'.  We need a regular
+            # expression for that.
+            for pat in rest.split():
+                if '*' in pat:
+                    pat = pat.replace('*', '[^/]*')
+                    # Do not make a dot into a magical wildcard character.
+                    pat = pat.replace('.', '\.')
+                    ignore_regexps.append(pat)
+                else:
+                    # No need for special handling.
+                    ignore.append(pat)
+        elif cmd == 'global-exclude':
+            ignore.extend(rest.split())
+        elif cmd == 'recursive-exclude':
+            dirname, patterns = rest.split(None, 1)
+            for pattern in patterns.split():
+                ignore.append(dirname + os.path.sep + pattern)
+        elif cmd == 'prune':
+            # rest is considered to be a directory name.  It should
+            # not contain a path separator, as it actually has no
+            # effect in that case, but that could differ per python
+            # version.  We strip it here to avoid double separators.
+            rest = rest.rstrip(os.path.sep)
+            ignore.append(rest)
+            ignore.append(rest + os.path.sep + '*')
+    return ignore, ignore_regexps
+
+
 def file_matches(filename, patterns):
     """Does this filename match any of the patterns?"""
     return any(fnmatch.fnmatch(filename, pat) for pat in patterns)
 
 
+def file_matches_regexps(filename, patterns):
+    """Does this filename match any of the regular expressions?"""
+    return any(re.match(pat, filename) for pat in patterns)
+
+
 def strip_sdist_extras(filelist):
-    """Strip generated files that are only present in source distributions."""
+    """Strip generated files that are only present in source distributions.
+
+    We also strip files that are ignored for other reasons, like
+    command line arguments, setup.cfg rules or MANIFEST.in rules.
+    """
     return [name for name in filelist
-            if not file_matches(name, IGNORE)]
+            if not file_matches(name, IGNORE)
+            and not file_matches_regexps(name, IGNORE_REGEXPS)]
 
 
 def find_bad_ideas(filelist):
@@ -451,6 +528,7 @@ def check_manifest(source_tree='.', create=False, update=False,
         if not is_package(source_tree):
             raise Failure('This is not a Python project (no setup.py).')
         read_config()
+        read_manifest()
         info_begin("listing source files under version control")
         all_source_files = sorted(get_vcs_files())
         source_files = strip_sdist_extras(all_source_files)

tests.py

@@ -3,6 +3,7 @@
 import shutil
 import subprocess
 import tempfile
+import textwrap
 import unittest
 
 
@@ -82,6 +83,72 @@ def test_strip_sdist_extras(self):
         ]
         self.assertEqual(strip_sdist_extras(filelist), expected)
 
+    def test_strip_sdist_extras_with_manifest(self):
+        import check_manifest
+        from check_manifest import strip_sdist_extras
+        from check_manifest import _get_ignore_from_manifest as parse
+        orig_ignore = check_manifest.IGNORE
+        orig_ignore_regexps = check_manifest.IGNORE_REGEXPS
+        manifest_in = """
+        graft src
+        exclude *.cfg
+        global-exclude *.mo
+        prune src/dump
+        recursive-exclude src/zope *.sh
+        """
+        # Keep the indentation visually clear in the test, but remove
+        # leading whitespace programmatically.
+        manifest_in = textwrap.dedent(manifest_in)
+        filelist = [
+            '.gitignore',
+            'setup.py',
+            'setup.cfg',
+            'MANIFEST.in',
+            'README.txt',
+            'src',
+            'src/helper.sh',
+            'src/dump',
+            'src/dump/__init__.py',
+            'src/zope',
+            'src/zope/__init__.py',
+            'src/zope/zopehelper.sh',
+            'src/zope/foo',
+            'src/zope/foo/__init__.py',
+            'src/zope/foo/language.po',
+            'src/zope/foo/language.mo',
+            'src/zope/foo/config.cfg',
+            'src/zope/foo/foohelper.sh',
+            'src/zope.foo.egg-info',
+            'src/zope.foo.egg-info/SOURCES.txt',
+        ]
+        expected = [
+            'setup.py',
+            'MANIFEST.in',
+            'README.txt',
+            'src',
+            'src/helper.sh',
+            'src/zope',
+            'src/zope/__init__.py',
+            'src/zope/foo',
+            'src/zope/foo/__init__.py',
+            'src/zope/foo/language.po',
+            'src/zope/foo/config.cfg',
+        ]
+
+        # This will change the definitions.
+        try:
+            # This is normally done in read_manifest:
+            ignore, ignore_regexps = parse(manifest_in)
+            check_manifest.IGNORE.extend(ignore)
+            check_manifest.IGNORE_REGEXPS.extend(ignore_regexps)
+            # Filter the file list.
+            result = strip_sdist_extras(filelist)
+        finally:
+            # Restore the original definitions
+            check_manifest.IGNORE = orig_ignore
+            check_manifest.IGNORE_REGEXPS = orig_ignore_regexps
+        self.assertEqual(result, expected)
+
     def test_find_bad_ideas(self):
         from check_manifest import find_bad_ideas
         filelist = [
@@ -137,6 +204,77 @@ def test_find_suggestions_generic_fallback_rules(self):
         self.assertEqual(find_suggestions(['src/id-lang.map']),
                          (['recursive-include src *.map'], []))
 
+    def test_get_ignore_from_manifest(self):
+        from check_manifest import _get_ignore_from_manifest as parse
+        # The return value is a tuple with two lists:
+        # ([<list of filename ignores>], [<list of regular expressions>])
+        self.assertEqual(parse(''),
+                         ([], []))
+        self.assertEqual(parse('      \n        '),
+                         ([], []))
+        self.assertEqual(parse('exclude *.cfg'),
+                         ([], ['[^/]*\.cfg']))
+        self.assertEqual(parse('#exclude *.cfg'),
+                         ([], []))
+        self.assertEqual(parse('exclude          *.cfg'),
+                         ([], ['[^/]*\.cfg']))
+        self.assertEqual(parse('\texclude\t*.cfg foo.*   bar.txt'),
+                         (['bar.txt'], ['[^/]*\.cfg', 'foo\.[^/]*']))
+        self.assertEqual(parse('exclude some/directory/*.cfg'),
+                         ([], ['some/directory/[^/]*\.cfg']))
+        self.assertEqual(parse('include *.cfg'),
+                         ([], []))
+        self.assertEqual(parse('global-exclude *.pyc'),
+                         (['*.pyc'], []))
+        self.assertEqual(parse('global-exclude *.pyc *.sh'),
+                         (['*.pyc', '*.sh'], []))
+        self.assertEqual(parse('recursive-exclude dir *.pyc'),
+                         (['dir/*.pyc'], []))
+        self.assertEqual(parse('recursive-exclude dir *.pyc *.sh'),
+                         (['dir/*.pyc', 'dir/*.sh'], []))
+        self.assertEqual(parse('prune dir'),
+                         (['dir', 'dir/*'], []))
+        # You should not add a slash at the end of a prune, but let's
+        # not fail over it or end up with double slashes.
+        self.assertEqual(parse('prune dir/'),
+                         (['dir', 'dir/*'], []))
+        text = """
+        #exclude *.01
+        exclude *.02
+        exclude *.03 04.*   bar.txt
+        exclude          *.05
+        exclude some/directory/*.cfg
+        global-exclude *.10 *.11
+        global-exclude *.12
+        include *.20
+        prune 30
+        recursive-exclude    40      *.41
+        recursive-exclude 42 *.43 44.*
+        """
+        # Keep the indentation visually clear in the test, but remove
+        # leading whitespace programmatically.
+        text = textwrap.dedent(text)
+        self.assertEqual(
+            parse(text),
+            ([
+             'bar.txt',
+             '*.10',
+             '*.11',
+             '*.12',
+             '30',
+             '30/*',
+             '40/*.41',
+             '42/*.43',
+             '42/44.*',
+             ],
+             [
+             '[^/]*\.02',
+             '[^/]*\.03',
+             '04\.[^/]*',
+             '[^/]*\.05',
+             'some/directory/[^/]*\.cfg',
+             ]))
+
 
 class VCSMixin(object):