Merge remote-tracking branch 'upstream/master'

* upstream/master:
  Gadgets/Coliru: Acquire Ace editor resources via https
  Headers: Fix definition of basic_ifstream
  Add Doxygen web tag file to HTML book archive
  Preprocess: rename fix_relative_link() to tranform_link()
  Preprocess: split fix_relative_link()
  Preprocess: fix edge case handling in has_class()
  Tests: add tests for has_class()
  Tests: add tests for convert_loader_name()
  Add script to run tests
  Preproces: move reusable components to separate folder
  Transform: Rename incorrectly named class
  Transform: Allow XML data to be transformed directly
  Transform: Move reusable components to separate folder
  Transform: use argparse for argument parsing
  Transform: Move code in transform scripts to main()

Author: PeterFeicht

commands/preprocess.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+
+#   Copyright (C) 2011, 2012  Povilas Kanapickas <povilas@radix.lt>
+#
+#   This file is part of cppreference-doc
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see http://www.gnu.org/licenses/.
+
+from datetime import datetime
+import fnmatch
+from lxml import etree
+import re
+import os
+import sys
+import shutil
+import urllib.parse
+from xml_utils import xml_escape, xml_unescape
+
+def rmtree_if_exists(dir):
+    if os.path.isdir(dir):
+        shutil.rmtree(dir)
+
+def move_dir_contents_to_dir(srcdir, dstdir):
+    for fn in os.listdir(srcdir):
+        shutil.move(os.path.join(srcdir, fn),
+                    os.path.join(dstdir, fn))
+
+def rearrange_archive(root):
+    # rearrange the archive. {root} here is output/reference
+
+    # before
+    # {root}/en.cppreference.com/w/ : html
+    # {root}/en.cppreference.com/mwiki/ : data
+    # {root}/en.cppreference.com/ : data
+    # ... (other languages)
+    # {root}/upload.cppreference.com/mwiki/ : data
+
+    # after
+    # {root}/common/ : all common data
+    # {root}/en/ : html for en
+    # ... (other languages)
+
+    data_path = os.path.join(root, 'common')
+    rmtree_if_exists(data_path)
+    shutil.move(os.path.join(root, 'upload.cppreference.com/mwiki'), data_path)
+    shutil.rmtree(os.path.join(root, 'upload.cppreference.com'))
+
+    for lang in ["en"]:
+        path = os.path.join(root, lang + ".cppreference.com/")
+        src_html_path = path + "w/"
+        src_data_path = path + "mwiki/"
+        html_path = os.path.join(root, lang)
+
+        if os.path.isdir(src_html_path):
+            shutil.move(src_html_path, html_path)
+
+        if os.path.isdir(src_data_path):
+            # the skin files should be the same for all languages thus we
+            # can merge everything
+            move_dir_contents_to_dir(src_data_path, data_path)
+
+        # also copy the custom fonts
+        shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed60.ttf'), data_path)
+        shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed75.ttf'), data_path)
+        # and the favicon
+        shutil.copy(os.path.join(path, 'favicon.ico'), data_path)
+
+        # remove what's left
+        shutil.rmtree(path)
+
+    # remove the XML source file
+    for fn in fnmatch.filter(os.listdir(root), 'cppreference-export*.xml'):
+        os.remove(os.path.join(root, fn))
+
+def add_file_to_rename_map(rename_map, dir, fn, new_fn):
+    path = os.path.join(dir, fn)
+    if not os.path.isfile(path):
+        print("ERROR: Not renaming '{0}' because path does not exist".format(path))
+        return
+    rename_map.append((dir, fn, new_fn))
+
+# Converts complex URL to resources supplied by MediaWiki loader to a simplified name
+def convert_loader_name(fn):
+    if re.search("modules=site&only=scripts", fn):
+        return "site_scripts.js"
+    elif re.search("modules=site&only=styles", fn):
+        return "site_modules.css"
+    elif re.search("modules=skins.*&only=scripts", fn):
+        return "skin_scripts.js"
+    elif re.search("modules=startup&only=scripts", fn):
+        return "startup_scripts.js"
+    elif re.search("modules=.*ext.*&only=styles", fn):
+        return "ext.css"
+    else:
+        raise Exception('Loader file {0} does not match any known files'.format(fn))
+
+def find_files_to_be_renamed(root):
+    # Returns a rename map: array of tuples each of which contain three strings:
+    # the directory the file resides in, the source and destination filenames.
+
+    # The rename map specifies files to be renamed in order to support them on
+    # windows filesystems which don't support certain characters in file names
+    rename_map = []
+
+    files_rename = []           # general files to be renamed
+    files_loader = []           # files served by load.php. These should map to
+                                # consistent and short file names because we
+                                # modify some of them later in the pipeline
+
+    for dir, dirnames, filenames in os.walk(root):
+        filenames_loader = set(fnmatch.filter(filenames, 'load.php[?]*'))
+        # match any filenames with '?"*' characters
+        filenames_rename = set(fnmatch.filter(filenames, '*[?"*]*'))
+
+        # don't process load.php files in general rename handler
+        filenames_rename -= filenames_loader
+
+        for fn in filenames_loader:
+            files_loader.append((dir, fn))
+        for fn in filenames_rename:
+            files_rename.append((dir, fn))
+
+    for dir,orig_fn in files_rename:
+        fn = orig_fn
+        fn = re.sub('\?.*', '', fn)
+        fn = re.sub('"', '_q_', fn)
+        fn = re.sub('\*', '_star_', fn)
+        add_file_to_rename_map(rename_map, dir, orig_fn, fn)
+
+    # map loader names to more recognizable names
+    for dir,fn in files_loader:
+        new_fn = convert_loader_name(fn)
+        add_file_to_rename_map(rename_map, dir, fn, new_fn)
+
+    # rename filenames that conflict on case-insensitive filesystems
+    # TODO: perform this automatically
+    add_file_to_rename_map(rename_map, os.path.join(root, 'en/cpp/numeric/math'), 'NAN.html', 'NAN.2.html')
+    add_file_to_rename_map(rename_map, os.path.join(root, 'en/c/numeric/math'), 'NAN.html', 'NAN.2.html')
+    return rename_map
+
+def rename_files(rename_map):
+    for dir, old_fn, new_fn in rename_map:
+        src_path = os.path.join(dir, old_fn)
+        dst_path = os.path.join(dir, new_fn)
+        print("Renaming '{0}' to \n         '{1}'".format(src_path, dst_path))
+        shutil.move(src_path, dst_path)
+
+def find_html_files(root):
+    # find files that need to be preprocessed
+    html_files = []
+    for dir, dirnames, filenames in os.walk(root):
+        for filename in fnmatch.filter(filenames, '*.html'):
+            html_files.append(os.path.join(dir, filename))
+    return html_files
+
+def is_loader_link(target):
+    if re.match('https?://[a-z]+\.cppreference\.com/mwiki/load\.php', target):
+        return True
+    return False
+
+def transform_loader_link(target, file, root):
+    # Absolute loader.php links need to be made relative
+    abstarget = os.path.join(root, "common/" + convert_loader_name(target))
+    return os.path.relpath(abstarget, os.path.dirname(file))
+
+def is_external_link(target):
+    if re.match('(ht|f)tps?://', target):
+        return True
+    return False
+
+def trasform_relative_link(rename_map, target):
+    target = urllib.parse.unquote(target)
+    for dir,fn,new_fn in rename_map:
+        target = target.replace(fn, new_fn)
+    target = target.replace('../../upload.cppreference.com/mwiki/','../common/')
+    target = target.replace('../mwiki/','../common/')
+    target = re.sub('(\.php|\.css)\?.*', '\\1', target)
+    target = urllib.parse.quote(target)
+    target = target.replace('%23', '#')
+    return target
+
+# Transforms a link in the given file according to rename map.
+# target is the link to transform.
+# file is the path of the file the link came from.
+# root is the path to the root of the archive.
+def transform_link(rename_map, target, file, root):
+    if is_loader_link(target):
+        return transform_loader_link(target, file, root)
+
+    if is_external_link(target):
+        return target
+
+    return trasform_relative_link(rename_map, target)
+
+def has_class(el, classes_to_check):
+    value = el.get('class')
+    if value is None:
+        return False
+    classes = value.split(' ')
+    for cl in classes_to_check:
+        if cl != '' and cl in classes:
+            return True
+    return False
+
+def preprocess_html_file(root, fn, rename_map):
+    parser = etree.HTMLParser()
+    html = etree.parse(fn, parser)
+
+    # remove non-printable elements
+    for el in html.xpath('//*'):
+        if has_class(el, ['noprint', 'editsection']) and el.get('id') != 'cpp-footer-base':
+            el.getparent().remove(el)
+        if el.get('id') == 'toc':
+            el.getparent().remove(el)
+
+    # remove external links to unused resources
+    for el in html.xpath('/html/head/link'):
+        if el.get('rel') in [ 'alternate', 'search', 'edit', 'EditURI' ]:
+            el.getparent().remove(el)
+        elif el.get('rel') == 'shortcut icon':
+            (head, tail) = os.path.split(el.get('href'))
+            el.set('href', os.path.join(head, 'common', tail))
+
+    # remove Google Analytics scripts
+    for el in html.xpath('/html/body/script'):
+        if el.get('src') is not None and 'google-analytics.com/ga.js' in el.get('src'):
+            el.getparent().remove(el)
+        elif el.text is not None and ('google-analytics.com/ga.js' in el.text or 'pageTracker' in el.text):
+            el.getparent().remove(el)
+
+    # make custom footer
+    footer = html.xpath('//*[@id=\'footer\']')[0]
+    for child in footer.getchildren():
+        id = child.get('id')
+        if id == 'cpp-navigation':
+            items = child.find('ul')
+            items.clear()
+
+            link = etree.SubElement(etree.SubElement(items, 'li'), 'a')
+            url = re.sub('(..)/(.*)\\.html', 'http://\\1.cppreference.com/w/\\2', os.path.relpath(fn, root))
+            url = re.sub('(.*)/index', '\\1/', url)
+            link.set('href', url)
+            link.text = 'Online version'
+
+            li = etree.SubElement(items, 'li')
+            mtime = datetime.fromtimestamp(os.stat(fn).st_mtime);
+            li.text = f"Offline version retrieved {mtime.isoformat(sep=' ', timespec='minutes')}."
+        elif id == 'footer-info':
+            pass
+        else:
+            footer.remove(child)
+
+    # apply changes to links caused by file renames
+    for el in html.xpath('//*[@src or @href]'):
+        if el.get('src') is not None:
+            el.set('src', transform_link(rename_map, el.get('src'), fn, root))
+        elif el.get('href') is not None:
+            el.set('href', transform_link(rename_map, el.get('href'), fn, root))
+
+    for err in parser.error_log:
+        print("HTML WARN: {0}".format(err))
+
+    html.write(fn, encoding='utf-8', method='html')
+
+def preprocess_css_file(fn):
+    f = open(fn, "r", encoding='utf-8')
+    text = f.read()
+    f.close()
+
+    # note that query string is not used in css files
+
+    text = text.replace('../DejaVuSansMonoCondensed60.ttf', 'DejaVuSansMonoCondensed60.ttf')
+    text = text.replace('../DejaVuSansMonoCondensed75.ttf', 'DejaVuSansMonoCondensed75.ttf')
+
+    # QT Help viewer doesn't understand nth-child
+    text = text.replace('nth-child(1)', 'first-child')
+
+    f = open(fn, "w", encoding='utf-8')
+    f.write(text)
+    f.close()

gadgets/coliru_compiler.js

@@ -545,10 +545,10 @@ window.jump_to_error = function(node, lineno) {
 var editors = [];
 
 $.when(
-    get_script_cached('http://d1n0x3qji82z53.cloudfront.net/src-min-noconflict/ace.js')
+    get_script_cached('https://d1n0x3qji82z53.cloudfront.net/src-min-noconflict/ace.js')
 ).done(function() {
     $.when(
-        get_script_cached('http://d1n0x3qji82z53.cloudfront.net/src-min-noconflict/mode-c_cpp.js'),
+        get_script_cached('https://d1n0x3qji82z53.cloudfront.net/src-min-noconflict/mode-c_cpp.js'),
         $.Deferred(function(deferred) {
             $(deferred.resolve);
         })

headers/fstream

@@ -101,7 +101,7 @@ typedef basic_fstream<wchar_t> wfstream;
 template <
     class CharT,
     class Traits = std::char_traits<CharT>
-    > class basic_ifstream : public std::basic_ostream<CharT, Traits> {
+    > class basic_ifstream : public std::basic_istream<CharT, Traits> {
 public:
     basic_ifstream();
     explicit basic_ifstream(const char* filename,