Auto merge of #24169 - Manishearth:rollup, r=Manishearth

- Successful merges: #24132, #24139, #24147, #24148, #24150, #24166
- Failed merges:

AUTHORS.txt

@@ -796,7 +796,7 @@ Tamir Duberstein <tamird@squareup.com>
 Taras Shpot <mrshpot@gmail.com>
 Taylor Hutchison <seanthutchison@gmail.com>
 Ted Horst <ted.horst@earthlink.net>
-Tero Hänninen <tejohann@kapsi.fi>
+Tero Hänninen <lgvz@users.noreply.github.com>
 Thad Guidry <thadguidry@gmail.com>
 Thiago Carvalho <thiago.carvalho@westwing.de>
 Thiago Pontes <email@thiago.me>

src/compiletest/runtest.rs

@@ -382,8 +382,7 @@ fn run_debuginfo_gdb_test(config: &Config, props: &TestProps, testfile: &Path) {
 
             // write debugger script
             let mut script_str = String::with_capacity(2048);
-            let charset = if cfg!(target_os = "bitrig") { "auto" } else { "UTF-8" };
-            script_str.push_str(&format!("set charset {}\n", charset));
+            script_str.push_str(&format!("set charset {}\n", charset()));
             script_str.push_str(&format!("file {}\n", exe_file.to_str().unwrap()));
             script_str.push_str("target remote :5039\n");
             script_str.push_str(&format!("set solib-search-path \
@@ -517,8 +516,7 @@ fn run_debuginfo_gdb_test(config: &Config, props: &TestProps, testfile: &Path) {
                                                        .to_string();
             // write debugger script
             let mut script_str = String::with_capacity(2048);
-            let charset = if cfg!(target_os = "bitrig") { "auto" } else { "UTF-8" };
-            script_str.push_str(&format!("set charset {}\n", charset));
+            script_str.push_str(&format!("set charset {}\n", charset()));
             script_str.push_str("show version\n");
 
             match config.gdb_version {
@@ -1791,3 +1789,11 @@ fn run_codegen_test(config: &Config, props: &TestProps,
                      (base_lines as f64) / (clang_lines as f64),
                      0.001);
 }
+
+fn charset() -> &'static str {
+    if cfg!(any(target_os = "bitrig", target_os = "freebsd")) {
+        "auto"
+    } else {
+        "UTF-8"
+    }
+}

src/doc/intro.md

@@ -23,7 +23,7 @@ and safety.
 # Tools
 
 Getting started on a new Rust project is incredibly easy, thanks to Rust's
-package manager, [Cargo](http://crates.io).
+package manager, [Cargo](https://crates.io/).
 
 To start a new project with Cargo, use `cargo new`:
 
@@ -149,8 +149,8 @@ Enough about tools, let's talk code!
 Rust's defining feature is "memory safety without garbage collection". Let's
 take a moment to talk about what that means. *Memory safety* means that the
 programming language eliminates certain kinds of bugs, such as [buffer
-overflows](http://en.wikipedia.org/wiki/Buffer_overflow) and [dangling
-pointers](http://en.wikipedia.org/wiki/Dangling_pointer). These problems occur
+overflows](https://en.wikipedia.org/wiki/Buffer_overflow) and [dangling
+pointers](https://en.wikipedia.org/wiki/Dangling_pointer). These problems occur
 when you have unrestricted access to memory. As an example, here's some Ruby
 code:
 

src/doc/trpl/hello-world.md

@@ -149,10 +149,10 @@ If you come from a dynamically typed language like Ruby, Python, or JavaScript,
 you may not be used to these two steps being separate. Rust is an
 *ahead-of-time compiled language*, which means that you can compile a
 program, give it to someone else, and they don't need to have Rust installed.
-If you give someone a `.rb` or `.py` or `.js` file, they need to have
-Ruby/Python/JavaScript installed, but you just need one command to both compile
-and run your program. Everything is a tradeoff in language design, and Rust has
-made its choice.
+If you give someone a `.rb` or `.py` or `.js` file, they need to have a
+Ruby/Python/JavaScript implementation installed, but you just need one command
+to both compile and run your program. Everything is a tradeoff in language design,
+and Rust has made its choice.
 
 Congratulations! You have officially written a Rust program. That makes you a
 Rust programmer! Welcome.

src/etc/unicode.py

@@ -12,8 +12,11 @@
 
 # This script uses the following Unicode tables:
 # - DerivedCoreProperties.txt
+# - DerivedNormalizationProps.txt
 # - EastAsianWidth.txt
+# - auxiliary/GraphemeBreakProperty.txt
 # - PropList.txt
+# - ReadMe.txt
 # - Scripts.txt
 # - UnicodeData.txt
 #
@@ -51,41 +54,20 @@
     'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
 }
 
-
-# Grapheme cluster data
-# taken from UAX29, http://www.unicode.org/reports/tr29/
-# these code points are excluded from the Control category
-# NOTE: CR and LF are also technically excluded, but for
-# the sake of convenience we leave them in the Control group
-# and manually check them in the appropriate place. This is
-# still compliant with the implementation requirements.
-grapheme_control_exceptions = set([0x200c, 0x200d])
-
-# the Regional_Indicator category
-grapheme_regional_indicator = [(0x1f1e6, 0x1f1ff)]
-
-# "The following ... are specifically excluded" from the SpacingMark category
-# http://www.unicode.org/reports/tr29/#SpacingMark
-grapheme_spacingmark_exceptions = [(0x102b, 0x102c), (0x1038, 0x1038),
-    (0x1062, 0x1064), (0x1067, 0x106d), (0x1083, 0x1083), (0x1087, 0x108c),
-    (0x108f, 0x108f), (0x109a, 0x109c), (0x19b0, 0x19b4), (0x19b8, 0x19b9),
-    (0x19bb, 0x19c0), (0x19c8, 0x19c9), (0x1a61, 0x1a61), (0x1a63, 0x1a64),
-    (0xaa7b, 0xaa7b), (0xaa7d, 0xaa7d)]
-
-# these are included in the SpacingMark category
-grapheme_spacingmark_extra = set([0xe33, 0xeb3])
+# these are the surrogate codepoints, which are not valid rust characters
+surrogate_codepoints = (0xd800, 0xdfff)
 
 def fetch(f):
-    if not os.path.exists(f):
+    if not os.path.exists(os.path.basename(f)):
         os.system("curl -O http://www.unicode.org/Public/UNIDATA/%s"
                   % f)
 
-    if not os.path.exists(f):
+    if not os.path.exists(os.path.basename(f)):
         sys.stderr.write("cannot load %s" % f)
         exit(1)
 
 def is_surrogate(n):
-    return 0xD800 <= n <= 0xDFFF
+    return surrogate_codepoints[0] <= n <= surrogate_codepoints[1]
 
 def load_unicode_data(f):
     fetch(f)
@@ -228,7 +210,7 @@ def load_properties(f, interestingprops):
     re1 = re.compile("^([0-9A-F]+) +; (\w+)")
     re2 = re.compile("^([0-9A-F]+)\.\.([0-9A-F]+) +; (\w+)")
 
-    for line in fileinput.input(f):
+    for line in fileinput.input(os.path.basename(f)):
         prop = None
         d_lo = 0
         d_hi = 0
@@ -623,20 +605,14 @@ def optimize_width_table(wtable):
         (canon_decomp, compat_decomp, gencats, combines,
                 lowerupper, upperlower) = load_unicode_data("UnicodeData.txt")
         want_derived = ["XID_Start", "XID_Continue", "Alphabetic", "Lowercase", "Uppercase"]
-        other_derived = ["Default_Ignorable_Code_Point", "Grapheme_Extend"]
+        other_derived = ["Default_Ignorable_Code_Point"]
         derived = load_properties("DerivedCoreProperties.txt", want_derived + other_derived)
         scripts = load_properties("Scripts.txt", [])
         props = load_properties("PropList.txt",
                 ["White_Space", "Join_Control", "Noncharacter_Code_Point"])
         norm_props = load_properties("DerivedNormalizationProps.txt",
                      ["Full_Composition_Exclusion"])
 
-        # grapheme cluster category from DerivedCoreProperties
-        # the rest are defined below
-        grapheme_cats = {}
-        grapheme_cats["Extend"] = derived["Grapheme_Extend"]
-        del(derived["Grapheme_Extend"])
-
         # bsearch_range_table is used in all the property modules below
         emit_bsearch_range_table(rf)
 
@@ -691,34 +667,24 @@ def optimize_width_table(wtable):
 
         ### grapheme cluster module
         # from http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Break_Property_Values
-        # Hangul syllable categories
-        want_hangul = ["L", "V", "T", "LV", "LVT"]
-        grapheme_cats.update(load_properties("HangulSyllableType.txt", want_hangul))
+        grapheme_cats = load_properties("auxiliary/GraphemeBreakProperty.txt", [])
 
         # Control
+        #  Note 1:
         # This category also includes Cs (surrogate codepoints), but Rust's `char`s are
         # Unicode Scalar Values only, and surrogates are thus invalid `char`s.
-        grapheme_cats["Control"] = set()
-        for cat in ["Zl", "Zp", "Cc", "Cf"]:
-            grapheme_cats["Control"] |= set(ungroup_cat(gencats[cat]))
+        # Thus, we have to remove Cs from the Control category
+        #  Note 2:
+        # 0x0a and 0x0d (CR and LF) are not in the Control category for Graphemes.
+        # However, the Graphemes iterator treats these as a special case, so they
+        # should be included in grapheme_cats["Control"] for our implementation.
         grapheme_cats["Control"] = group_cat(list(
-            grapheme_cats["Control"]
-            - grapheme_control_exceptions
-            | (set(ungroup_cat(gencats["Cn"]))
-               & set(ungroup_cat(derived["Default_Ignorable_Code_Point"])))))
-
-        # Regional Indicator
-        grapheme_cats["RegionalIndicator"] = grapheme_regional_indicator
-
-        # Prepend - "Currently there are no characters with this value"
-        # (from UAX#29, Unicode 7.0)
-
-        # SpacingMark
-        grapheme_cats["SpacingMark"] = group_cat(list(
-            set(ungroup_cat(gencats["Mc"]))
-            - set(ungroup_cat(grapheme_cats["Extend"]))
-            | grapheme_spacingmark_extra
-            - set(ungroup_cat(grapheme_spacingmark_exceptions))))
+            (set(ungroup_cat(grapheme_cats["Control"]))
+             | set(ungroup_cat(grapheme_cats["CR"]))
+             | set(ungroup_cat(grapheme_cats["LF"])))
+            - set(ungroup_cat([surrogate_codepoints]))))
+        del(grapheme_cats["CR"])
+        del(grapheme_cats["LF"])
 
         grapheme_table = []
         for cat in grapheme_cats: