Merge branch 'rename-map'

* rename-map:
  Preprocess: add unit tests for file renaming
  Preprocess: rename find_files_to_be_renamed
  Preprocess: automate renaming of case conflicts
  Preprocess: simplify file renaming

Author: PeterFeicht

commands/preprocess.py

@@ -84,13 +84,6 @@ def rearrange_archive(root):
     for fn in fnmatch.filter(os.listdir(root), 'cppreference-export*.xml'):
         os.remove(os.path.join(root, fn))
 
-def add_file_to_rename_map(rename_map, dir, fn, new_fn):
-    path = os.path.join(dir, fn)
-    if not os.path.isfile(path):
-        print("ERROR: Not renaming '{0}' because path does not exist".format(path))
-        return
-    rename_map.append((dir, fn, new_fn))
-
 # Converts complex URL to resources supplied by MediaWiki loader to a simplified name
 def convert_loader_name(fn):
     if "modules=site&only=scripts" in fn:
@@ -106,56 +99,53 @@ def convert_loader_name(fn):
     else:
         raise Exception('Loader file {0} does not match any known files'.format(fn))
 
-def find_files_to_be_renamed(root):
-    # Returns a rename map: array of tuples each of which contain three strings:
-    # the directory the file resides in, the source and destination filenames.
+def build_rename_map(root):
+    # Returns a rename map: a map from old to new file name
+    loader = re.compile(r'load\.php\?.*')
+    query = re.compile(r'\?.*')
+    result = dict()
 
-    # The rename map specifies files to be renamed in order to support them on
-    # windows filesystems which don't support certain characters in file names
-    rename_map = []
+    # find files with invalid names -> rename all occurrences
+    for fn in set(fn for _, _, filenames in os.walk(root) for fn in filenames):
+        if loader.match(fn):
+            result[fn] = convert_loader_name(fn)
 
-    files_rename = []           # general files to be renamed
-    files_loader = []           # files served by load.php. These should map to
-                                # consistent and short file names because we
-                                # modify some of them later in the pipeline
+        elif any((c in fn) for c in '?*"'):
+            new_fn = query.sub('', fn)
+            new_fn = new_fn.replace('"', '_q_')
+            new_fn = new_fn.replace('*', '_star_')
+            result[fn] = new_fn
 
+    # find files that conflict on case-insensitive filesystems
     for dir, _, filenames in os.walk(root):
-        filenames_loader = set(fnmatch.filter(filenames, 'load.php[?]*'))
-        # match any filenames with '?"*' characters
-        filenames_rename = set(fnmatch.filter(filenames, '*[?"*]*'))
-
-        # don't process load.php files in general rename handler
-        filenames_rename -= filenames_loader
-
-        for fn in filenames_loader:
-            files_loader.append((dir, fn))
-        for fn in filenames_rename:
-            files_rename.append((dir, fn))
-
-    for dir, orig_fn in files_rename:
-        fn = orig_fn
-        fn = re.sub(r'\?.*', '', fn)
-        fn = fn.replace('"', '_q_')
-        fn = fn.replace('*', '_star_')
-        add_file_to_rename_map(rename_map, dir, orig_fn, fn)
-
-    # map loader names to more recognizable names
-    for dir, fn in files_loader:
-        new_fn = convert_loader_name(fn)
-        add_file_to_rename_map(rename_map, dir, fn, new_fn)
-
-    # rename filenames that conflict on case-insensitive filesystems
-    # TODO: perform this automatically
-    add_file_to_rename_map(rename_map, os.path.join(root, 'en/cpp/numeric/math'), 'NAN.html', 'NAN.2.html')
-    add_file_to_rename_map(rename_map, os.path.join(root, 'en/c/numeric/math'), 'NAN.html', 'NAN.2.html')
-    return rename_map
-
-def rename_files(rename_map):
-    for dir, old_fn, new_fn in rename_map:
+        seen = dict()
+        for fn in (result.get(s, s) for s in filenames):
+            low = fn.lower()
+            num = seen.setdefault(low, 0)
+            if num > 0:
+                name, ext = os.path.splitext(fn)
+                # add file with its path -> only rename that occurrence
+                result[os.path.join(dir, fn)] = "{}.{}{}".format(name, num + 1, ext)
+            seen[low] += 1
+
+    return result
+
+def rename_files(root, rename_map):
+    for dir, old_fn in ((dir, fn) for dir, _, filenames in os.walk(root) for fn in filenames):
         src_path = os.path.join(dir, old_fn)
-        dst_path = os.path.join(dir, new_fn)
-        print("Renaming '{0}' to \n         '{1}'".format(src_path, dst_path))
-        shutil.move(src_path, dst_path)
+
+        new_fn = rename_map.get(old_fn)
+        if new_fn:
+            # look for case conflict of the renamed file
+            new_path = os.path.join(dir, new_fn)
+            new_fn = rename_map.get(new_path, new_fn)
+        else:
+            # original filename unchanged, look for case conflict
+            new_fn = rename_map.get(src_path)
+        if new_fn:
+            dst_path = os.path.join(dir, new_fn)
+            print("Renaming {0}\n      to {1}".format(src_path, dst_path))
+            shutil.move(src_path, dst_path)
 
 def find_html_files(root):
     # find files that need to be preprocessed
@@ -172,7 +162,7 @@ def is_loader_link(target):
 
 def transform_loader_link(target, file, root):
     # Absolute loader.php links need to be made relative
-    abstarget = os.path.join(root, "common/" + convert_loader_name(target))
+    abstarget = os.path.join(root, "common", convert_loader_name(target))
     return os.path.relpath(abstarget, os.path.dirname(file))
 
 def is_ranges_placeholder(target):
@@ -201,20 +191,33 @@ def transform_ranges_placeholder(target, file, root):
     return os.path.relpath(abstarget, os.path.dirname(file))
 
 def is_external_link(target):
-    if re.match('(ht|f)tps?://', target):
-        return True
-    return False
+    url = urllib.parse.urlparse(target)
+    return url.scheme != '' or url.netloc != ''
+
+def trasform_relative_link(rename_map, target, file):
+    # urlparse returns (scheme, host, path, params, query, fragment)
+    _, _, path, params, _, fragment = urllib.parse.urlparse(target)
+    assert params == ''
+
+    path = urllib.parse.unquote(path)
+    path = path.replace('../../upload.cppreference.com/mwiki/','../common/')
+    path = path.replace('../mwiki/','../common/')
+
+    dir, fn = os.path.split(path)
+    new_fn = rename_map.get(fn)
+    if new_fn:
+        # look for case conflict of the renamed file
+        abstarget = os.path.normpath(os.path.join(os.path.dirname(file), dir, new_fn))
+        new_fn = rename_map.get(abstarget, new_fn)
+    else:
+        # original filename unchanged, look for case conflict
+        abstarget = os.path.normpath(os.path.join(os.path.dirname(file), path))
+        new_fn = rename_map.get(abstarget)
+    if new_fn:
+        path = os.path.join(dir, new_fn)
 
-def trasform_relative_link(rename_map, target):
-    target = urllib.parse.unquote(target)
-    for _, fn, new_fn in rename_map:
-        target = target.replace(fn, new_fn)
-    target = target.replace('../../upload.cppreference.com/mwiki/','../common/')
-    target = target.replace('../mwiki/','../common/')
-    target = re.sub(r'(\.php|\.css)\?.*', r'\1', target)
-    target = urllib.parse.quote(target)
-    target = target.replace('%23', '#')
-    return target
+    path = urllib.parse.quote(path)
+    return urllib.parse.urlunparse(('', '', path, params, '', fragment))
 
 # Transforms a link in the given file according to rename map.
 # target is the link to transform.
@@ -230,7 +233,7 @@ def transform_link(rename_map, target, file, root):
     if is_external_link(target):
         return target
 
-    return trasform_relative_link(rename_map, target)
+    return trasform_relative_link(rename_map, target, file)
 
 def has_class(el, *classes_to_check):
     value = el.get('class')

preprocess.py

@@ -38,8 +38,8 @@ def main():
 
     preprocess.rearrange_archive(root)
 
-    rename_map = preprocess.find_files_to_be_renamed(root)
-    preprocess.rename_files(rename_map)
+    rename_map = preprocess.build_rename_map(root)
+    preprocess.rename_files(root, rename_map)
 
     # clean the html files
     file_list = preprocess.find_html_files(root)