Merge pull request #276 from twitter/cldr_v46

Upgrade to CLDR v46, Unicode v16

.github/workflows/unit_tests.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: [2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2]
+        ruby-version: [2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby ${{ matrix.ruby-version }}
@@ -14,4 +14,5 @@ jobs:
           ruby-version: ${{ matrix.ruby-version }}
           bundler-cache: true
       - name: Run Tests
+        continue-on-error: true
         run: bundle exec rake spec:full

CHANGELOG.md

@@ -1,5 +1,9 @@
 # TwitterCldr Changelog
 
+### 6.13.0 (Jan 18th, 2025)
+* Upgrade to CLDR v46.1, ICU 76.1, and Unicode v16.0.0.
+* Remove support for Ruby 2.3 and 2.4.
+
 ### 6.12.1 (Apr 28th, 2024)
 * Fix issue causing sentence segmentation to return incorrect results when a string ends with a suppression directly followed by a single space. (#274, @didier-84)
 

Gemfile

@@ -4,7 +4,6 @@ gemspec
 
 group :development, :test do
   gem 'rake'
-  gem 'pry-byebug' unless RUBY_PLATFORM == 'java'
   gem 'ruby-prof' unless RUBY_PLATFORM == 'java'
   gem 'regexp_parser', '~> 0.5'
   gem 'benchmark-ips'
@@ -13,7 +12,7 @@ group :development, :test do
   # gemspec allows any version, but most people are probably using 1.x, so
   # let's test and develop against that
   gem 'tzinfo', '< 2'
-  gem 'tzinfo-data', '= 1.2023.3'  # try to keep in sync with ICU
+  gem 'tzinfo-data', '= 1.2024.2'  # try to keep in sync with ICU
 end
 
 group :development do

README.md

@@ -40,7 +40,7 @@ TwitterCldr patches core Ruby objects like `Integer` and `Date` to make localiza
 1337.localize(:es).to_s                                    # "1.337"
 
 # currencies, default USD
-1337.localize(:es).to_currency.to_s                        # "1.337,00 $"
+1337.localize(:es).to_currency.to_s                        # "1.337,00 US$"
 1337.localize(:es).to_currency.to_s(:currency => "EUR")    # "1.337,00 €"
 
 # percentages
@@ -71,7 +71,7 @@ If you're looking for a list of supported currencies, use the `TwitterCldr::Shar
 TwitterCldr::Shared::Currencies.currency_codes             # ["ADP", "AED", "AFA", "AFN", ... ]
 
 # data for a specific currency code
-TwitterCldr::Shared::Currencies.for_code("CAD")            # {:currency=>:CAD, :name=>"Canadian Dollar", :cldr_symbol=>"CA$", :symbol=>"$", :code_points=>[36]}
+TwitterCldr::Shared::Currencies.for_code("CAD")            # {:currency=>:CAD, :name=>"Canadian Dollar", :cldr_symbol=>"CA$", :symbol=>"CA$", :code_points=>[67, 65, 36]}
 ```
 
 #### Short / Long Decimals
@@ -92,7 +92,7 @@ TwitterCLDR supports formatting numbers with an attached unit, for example "12 d
 
 ```ruby
 12.localize.to_unit.length_mile  # "12 miles"
-12.localize(:ru).to_unit.length_mile  # "12 миль"
+12.localize(:ru).to_unit.length_mile  # "12 милях"
 ```
 Units support a few different forms, long, short, and narrow:
 
@@ -233,8 +233,8 @@ It's important to know that, even though any given format may not be available a
 | EHm        | Fri 12:20              |
 | EHms       | Fri 12:20:05           |
 | Ed         | 14 Fri                 |
-| Ehm        | Fri 12:20 PM           |
-| Ehms       | Fri 12:20:05 PM        |
+| Ehm        | Fri 12:20 PM           |
+| Ehms       | Fri 12:20:05 PM        |
 | Gy         | 2014 CE                |
 | GyMMM      | Feb 2014 CE            |
 | GyMMMEd    | Fri, Feb 14, 2014 CE   |
@@ -254,11 +254,11 @@ It's important to know that, even though any given format may not be available a
 | MMMd       | Feb 14                 |
 | Md         | 2/14                   |
 | d          | 14                     |
-| h          | 12 PM                  |
-| hm         | 12:20 PM               |
-| hms        | 12:20:05 PM            |
-| hmsv       | 12:20:05 PM GMT        |
-| hmv        | 12:20 PM GMT           |
+| h          | 12 PM                  |
+| hm         | 12:20 PM               |
+| hms        | 12:20:05 PM            |
+| hmsv       | 12:20:05 PM GMT        |
+| hmv        | 12:20 PM GMT           |
 | ms         | 20:05                  |
 | y          | 2014                   |
 | yM         | 2/2014                 |
@@ -602,7 +602,7 @@ postal_code.regexp  # /(\d{5})(?:[ \-](\d{4}))?/
 Get a sample of valid postal codes with the `#sample` method:
 
 ```ruby
-postal_code.sample(5)  # ["72959-4813", "81226", "05936-9185", "71858-7042", "20325-0737"]
+postal_code.sample(5)  # ["33623-6826", "59924", "59999", "42268-1200", "68209-4464"]
 ```
 
 ### Phone Codes
@@ -1106,6 +1106,6 @@ TwitterCLDR currently supports localization of certain textual objects in JavaSc
 
 ## License
 
-Copyright 2024 Twitter, Inc.
+Copyright 2025 Twitter, Inc.
 
 Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

README.md.erb

@@ -40,7 +40,7 @@ TwitterCldr patches core Ruby objects like `Integer` and `Date` to make localiza
 1337.localize(:es).to_s                                    # <%= assert(1337.localize(:es).to_s, "1.337").inspect %>
 
 # currencies, default USD
-1337.localize(:es).to_currency.to_s                        # <%= assert(1337.localize(:es).to_currency.to_s, "1.337,00 $").inspect %>
+1337.localize(:es).to_currency.to_s                        # <%= assert(1337.localize(:es).to_currency.to_s, "1.337,00 US$").inspect %>
 1337.localize(:es).to_currency.to_s(:currency => "EUR")    # <%= assert(1337.localize(:es).to_currency.to_s(:currency => "EUR"), "1.337,00 €").inspect %>
 
 # percentages
@@ -60,7 +60,7 @@ num = TwitterCldr::Localized::LocalizedNumber.new(1337, :es)
 num.to_currency.to_s  # ...etc
 ```
 
-<% assert(TwitterCldr::Localized::LocalizedNumber.new(1337, :es).to_currency.to_s, "1.337,00 $") %>
+<% assert(TwitterCldr::Localized::LocalizedNumber.new(1337, :es).to_currency.to_s, "1.337,00 US$") %>
 
 #### More on Currencies
 
@@ -71,7 +71,7 @@ If you're looking for a list of supported currencies, use the `TwitterCldr::Shar
 TwitterCldr::Shared::Currencies.currency_codes             # <%= ellipsize(assert(TwitterCldr::Shared::Currencies.currency_codes.sort[0..3], ["ADP", "AED", "AFA", "AFN"])) %>
 
 # data for a specific currency code
-TwitterCldr::Shared::Currencies.for_code("CAD")            # <%= assert(TwitterCldr::Shared::Currencies.for_code("CAD"), {:currency=>:CAD, :name=>"Canadian Dollar", :cldr_symbol=>"CA$", :symbol=>"$", :code_points=>[36]}).inspect %>
+TwitterCldr::Shared::Currencies.for_code("CAD")            # <%= assert(TwitterCldr::Shared::Currencies.for_code("CAD"), {:currency=>:CAD, :name=>"Canadian Dollar", :cldr_symbol=>"CA$", :symbol=>"CA$", :code_points=>[67, 65, 36]}).inspect %>
 ```
 
 #### Short / Long Decimals
@@ -92,7 +92,7 @@ TwitterCLDR supports formatting numbers with an attached unit, for example "12 d
 
 ```ruby
 12.localize.to_unit.length_mile  # <%= assert(12.localize.to_unit.length_mile, "12 miles").inspect %>
-12.localize(:ru).to_unit.length_mile  # <%= assert(12.localize(:ru).to_unit.length_mile, "12 миль").inspect %>
+12.localize(:ru).to_unit.length_mile  # <%= assert(12.localize(:ru).to_unit.length_mile, "12 милях").inspect %>
 ```
 Units support a few different forms, long, short, and narrow:
 

Rakefile

@@ -13,8 +13,6 @@ require 'rubygems/package_task'
 
 require './lib/twitter_cldr'
 
-require 'pry-byebug' unless RUBY_PLATFORM == 'java'
-
 Bundler::GemHelper.install_tasks
 
 task default: :spec

lib/twitter_cldr/resources/calendars_importer.rb

@@ -175,9 +175,8 @@ def formats(type)
       def additional_formats
         return {} unless calendar
 
-        dtd.find_attr('dateFormatItem', 'id').values.each_with_object({}) do |id, result|
-          node = calendar.xpath("dateTimeFormats/availableFormats/dateFormatItem[@id='#{id}']").first
-          result[id] = node.content if node
+        calendar.xpath("dateTimeFormats/availableFormats/dateFormatItem").each_with_object({}) do |date_format_item, result|
+          result[date_format_item.attribute("id").value] = date_format_item.content
         end
       end
 

lib/twitter_cldr/resources/rbnf_test_importer.rb

@@ -82,20 +82,11 @@ def clean_up_name(name)
       end
 
       def import_ruleset(locale, formatter, ruleset_name)
-        test_numbers_for(locale).each_with_object({}) do |num, ret|
+        TEST_NUMBERS.each_with_object({}) do |num, ret|
           ret[num] = formatter.format(num, ruleset_name)
         end
       end
 
-      def test_numbers_for(locale)
-        # for some reason, russian doesn't support large numbers
-        if locale.to_s == 'ru'
-          TEST_NUMBERS - [138_400]
-        else
-          TEST_NUMBERS
-        end
-      end
-
       def output_file_for(locale)
         File.join(params.fetch(:output_path), locale, 'rbnf_test.yml')
       end

lib/twitter_cldr/resources/timezone_tests_importer.rb

@@ -85,6 +85,8 @@ def generate_test_cases_for_locale(locale)
         ulocale = ulocale_class.new(locale.to_s)
 
         TZInfo::Timezone.all_identifiers.each_with_object({}) do |tz_id, ret|
+          next if tz_id == 'Factory'
+
           tz = tz_class.getTimeZone(tz_id)
           offset = tz.getRawOffset
 

lib/twitter_cldr/resources/timezones_importer.rb

@@ -130,7 +130,7 @@ def timezones
             short = nodes_to_hash(zone.xpath('short/*'))
             result[type][:short] = short unless short.empty?
             city = zone.xpath('exemplarCity').first
-            if city && !unconfirmed_draft?(city)
+            if city && !unconfirmed_draft?(city) && !secondary?(city)
               result[type][:city] = city.content
             end
             result
@@ -165,6 +165,12 @@ def unconfirmed_draft?(node)
             node.attributes['draft'].value == 'unconfirmed'
         end
 
+        def secondary?(node)
+          node &&
+            node.attributes['alt'] &&
+            node.attributes['alt'].value == 'secondary'
+        end
+
         def doc
           @doc ||= begin
             locale_fs = locale.to_s.gsub('-', '_')

lib/twitter_cldr/resources/transforms_importer.rb

@@ -135,36 +135,27 @@ def get_variant(node)
       end
 
       def rules(transform_node)
-        rules = fix_rule_wrapping(
-          transform_node.xpath('tRule').flat_map do |rule_node|
-            fix_rule(rule_node.content).split("\n").map(&:strip)
-          end
-        )
-
-        rules.reject do |rule|
-          rule.strip.empty? || rule.strip.start_with?('#')
-        end
-      end
-
-      def fix_rule_wrapping(rules)
-        wrap = false
-
-        rules.each_with_object([]) do |rule, ret|
-          if wrap
-            ret.last.sub!(/\\\z/, rule)
-          else
-            ret << rule
-          end
-
-          wrap = rule.end_with?('\\')
+        transform_node.xpath('tRule').flat_map do |rule_node|
+          rule_node.content
+            .split("\n")
+            .reject { |line| line.start_with?('#') }
+            .map { |line| line.sub(/#.*$/, '').strip }
+            .join("\n")
+            .split(/;$/)
+            .map(&:strip)
+            .reject(&:empty?)
+            .map do |rule_text|
+              "#{fix_rule(rule_text)} ;"
+            end
         end
       end
 
       def fix_rule(rule)
         rule.
           gsub("←", '<').
           gsub("→", '>').
-          gsub("↔", '<>')
+          gsub("↔", '<>').
+          gsub("\n", " ")
       end
 
       def each_transform_file(&block)

lib/twitter_cldr/transforms/filters/unicode_filter.rb

@@ -20,6 +20,8 @@ def parse(rule_text, symbol_table)
 
             # filters are always just a unicode set
             new(re.elements.first.to_set.to_set, direction)
+          rescue => e
+            binding.irb
           end
 
           def accepts?(rule_text)

lib/twitter_cldr/transforms/transforms/named_transform.rb

@@ -24,7 +24,6 @@ def exists?(form)
 
         def apply_to(cursor)
           if forward_form
-            puts forward_form.transform if $debug
             forward_form.apply_to(cursor)
           end
         end

lib/twitter_cldr/transforms/transforms/transform_rule.rb

@@ -32,12 +32,7 @@ def accepts?(rule_text)
             transforms.any? do |transform|
               transform.accepts?(forward_form, backward_form)
             end
-          rescue Exception => e
-            if $debug
-              puts e.message
-              puts e.backtrace.join("\n")
-            end
-
+          rescue Exception
             false
           end
 

lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = '6.12.1'
+  VERSION = '6.13.0'
 end

lib/twitter_cldr/versions.rb

@@ -5,9 +5,9 @@
 
 module TwitterCldr
   module Versions
-    CLDR_VERSION    = '43.1'
-    ICU_VERSION     = '73.2'
-    UNICODE_VERSION = '15.0.0'
+    CLDR_VERSION    = '46.1'
+    ICU_VERSION     = '76.1'
+    UNICODE_VERSION = '16.0.0'
 
     class << self
       def cldr_version

resources/locales/af/calendars.yml

@@ -313,12 +313,11 @@
         :stand-alone: 
           :abbreviated: 
             :afternoon1: middag
-            :am: vm.
+            :am: AM
             :evening1: aand
-            :midnight: middernag
             :morning1: oggend
             :night1: nag
-            :pm: nm.
+            :pm: PM
           :narrow: 
             :afternoon1: m
             :am: v
@@ -329,12 +328,11 @@
             :pm: "n"
           :wide: 
             :afternoon1: middag
-            :am: vm.
+            :am: AM
             :evening1: aand
-            :midnight: middernag
             :morning1: oggend
             :night1: nag
-            :pm: nm.
+            :pm: PM
       :quarters: 
         :format: 
           :abbreviated: