Support custom encodings on Windows through GNU libiconv

.github/workflows/win.yml

@@ -15,7 +15,7 @@ jobs:
           make deps
       - name: Cross-compile Crystal
         run: |
-          LLVM_TARGETS=X86 bin/crystal build --cross-compile --target x86_64-pc-windows-msvc src/compiler/crystal.cr -Dwithout_playground
+          LLVM_TARGETS=X86 bin/crystal build --cross-compile --target x86_64-pc-windows-msvc src/compiler/crystal.cr -Dwithout_playground -Dwithout_iconv
 
       - name: Upload Crystal object file
         uses: actions/upload-artifact@v2
@@ -42,6 +42,7 @@ jobs:
         with:
           path: | # openssl and llvm take much longer to build so they are cached separately
             libs/pcre.lib
+            libs/iconv.lib
             libs/gc.lib
             libs/z.lib
             libs/mpir.lib
@@ -81,6 +82,19 @@ jobs:
         run: |
           cmake . -DBUILD_SHARED_LIBS=OFF -DPCRE_SUPPORT_UNICODE_PROPERTIES=ON -DPCRE_SUPPORT_JIT=ON -DPCRE_STATIC_RUNTIME=ON -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF
           cmake --build . --config Release
+      - name: Download libiconv
+        if: steps.cache-libs.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: pffang/libiconv-for-Windows
+          ref: 9b7aba8da6e125ef33912fa4412779279f204003 # master @ 2021-08-24
+          path: libiconv
+      - name: Build libiconv
+        if: steps.cache-libs.outputs.cache-hit != 'true'
+        working-directory: ./libiconv
+        run: |
+          sed -i 's/MultiThreadedDLL/MultiThreaded/' libiconv.vcxproj
+          MSBuild.exe /p:Platform=x64 /p:Configuration=ReleaseStatic libiconv.vcxproj
       - name: Download zlib
         if: steps.cache-libs.outputs.cache-hit != 'true'
         run: |
@@ -138,6 +152,7 @@ jobs:
         run: |
           mkdir libs
           mv pcre/Release/pcre.lib libs/
+          mv libiconv/lib64/libiconvStatic.lib libs/iconv.lib
           mv bdwgc/Release/gc.lib libs/
           mv zlib/Release/zlibstatic.lib libs/z.lib
           mv mpir/lib/x64/Release/mpir.lib libs/

spec/std/file_spec.cr

@@ -1122,47 +1122,48 @@ describe "File" do
     end
   end
 
-  # TODO: these specs don't compile on win32 because iconv isn't implemented
-  describe "encoding" do
-    pending_win32 "writes with encoding" do
-      with_tempfile("encoding-write.txt") do |path|
-        File.write(path, "hello", encoding: "UCS-2LE")
-        File.read(path).to_slice.should eq("hello".encode("UCS-2LE"))
+  {% unless flag?(:without_iconv) %}
+    describe "encoding" do
+      it "writes with encoding" do
+        with_tempfile("encoding-write.txt") do |path|
+          File.write(path, "hello", encoding: "UCS-2LE")
+          File.read(path).to_slice.should eq("hello".encode("UCS-2LE"))
+        end
       end
-    end
 
-    pending_win32 "reads with encoding" do
-      with_tempfile("encoding-read.txt") do |path|
-        File.write(path, "hello", encoding: "UCS-2LE")
-        File.read(path, encoding: "UCS-2LE").should eq("hello")
+      it "reads with encoding" do
+        with_tempfile("encoding-read.txt") do |path|
+          File.write(path, "hello", encoding: "UCS-2LE")
+          File.read(path, encoding: "UCS-2LE").should eq("hello")
+        end
       end
-    end
 
-    pending_win32 "opens with encoding" do
-      with_tempfile("encoding-open.txt") do |path|
-        File.write(path, "hello", encoding: "UCS-2LE")
-        File.open(path, encoding: "UCS-2LE") do |file|
-          file.gets_to_end.should eq("hello")
+      it "opens with encoding" do
+        with_tempfile("encoding-open.txt") do |path|
+          File.write(path, "hello", encoding: "UCS-2LE")
+          File.open(path, encoding: "UCS-2LE") do |file|
+            file.gets_to_end.should eq("hello")
+          end
         end
       end
-    end
 
-    pending_win32 "does each line with encoding" do
-      with_tempfile("encoding-each_line.txt") do |path|
-        File.write(path, "hello", encoding: "UCS-2LE")
-        File.each_line(path, encoding: "UCS-2LE") do |line|
-          line.should eq("hello")
+      it "does each line with encoding" do
+        with_tempfile("encoding-each_line.txt") do |path|
+          File.write(path, "hello", encoding: "UCS-2LE")
+          File.each_line(path, encoding: "UCS-2LE") do |line|
+            line.should eq("hello")
+          end
         end
       end
-    end
 
-    pending_win32 "reads lines with encoding" do
-      with_tempfile("encoding-read_lines.txt") do |path|
-        File.write(path, "hello", encoding: "UCS-2LE")
-        File.read_lines(path, encoding: "UCS-2LE").should eq(["hello"])
+      it "reads lines with encoding" do
+        with_tempfile("encoding-read_lines.txt") do |path|
+          File.write(path, "hello", encoding: "UCS-2LE")
+          File.read_lines(path, encoding: "UCS-2LE").should eq(["hello"])
+        end
       end
     end
-  end
+  {% end %}
 
   describe "closed stream" do
     it "raises if writing on a closed stream" do

spec/std/io/buffered_spec.cr

@@ -440,69 +440,71 @@ describe "IO::Buffered" do
     io.read_char.should eq('b')
   end
 
-  pending_win32 describe: "encoding" do
-    describe "decode" do
-      it "gets_to_end" do
-        str = "Hello world" * 200
-        base_io = IO::Memory.new(str.encode("UCS-2LE"))
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("UCS-2LE")
-        io.gets_to_end.should eq(str)
-      end
+  {% unless flag?(:without_iconv) %}
+    describe "encoding" do
+      describe "decode" do
+        it "gets_to_end" do
+          str = "Hello world" * 200
+          base_io = IO::Memory.new(str.encode("UCS-2LE"))
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("UCS-2LE")
+          io.gets_to_end.should eq(str)
+        end
 
-      it "gets" do
-        str = "Hello world\nFoo\nBar\n" + ("1234567890" * 1000)
-        base_io = IO::Memory.new(str.encode("UCS-2LE"))
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("UCS-2LE")
-        io.gets.should eq("Hello world")
-        io.gets.should eq("Foo")
-        io.gets.should eq("Bar")
-      end
+        it "gets" do
+          str = "Hello world\nFoo\nBar\n" + ("1234567890" * 1000)
+          base_io = IO::Memory.new(str.encode("UCS-2LE"))
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("UCS-2LE")
+          io.gets.should eq("Hello world")
+          io.gets.should eq("Foo")
+          io.gets.should eq("Bar")
+        end
 
-      it "gets with chomp = false" do
-        str = "Hello world\nFoo\nBar\n" + ("1234567890" * 1000)
-        base_io = IO::Memory.new(str.encode("UCS-2LE"))
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("UCS-2LE")
-        io.gets(chomp: false).should eq("Hello world\n")
-        io.gets(chomp: false).should eq("Foo\n")
-        io.gets(chomp: false).should eq("Bar\n")
-      end
+        it "gets with chomp = false" do
+          str = "Hello world\nFoo\nBar\n" + ("1234567890" * 1000)
+          base_io = IO::Memory.new(str.encode("UCS-2LE"))
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("UCS-2LE")
+          io.gets(chomp: false).should eq("Hello world\n")
+          io.gets(chomp: false).should eq("Foo\n")
+          io.gets(chomp: false).should eq("Bar\n")
+        end
 
-      it "gets big string" do
-        str = "Hello\nWorld\n" * 10_000
-        base_io = IO::Memory.new(str.encode("UCS-2LE"))
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("UCS-2LE")
-        10_000.times do |i|
-          io.gets(chomp: false).should eq("Hello\n")
-          io.gets(chomp: false).should eq("World\n")
+        it "gets big string" do
+          str = "Hello\nWorld\n" * 10_000
+          base_io = IO::Memory.new(str.encode("UCS-2LE"))
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("UCS-2LE")
+          10_000.times do |i|
+            io.gets(chomp: false).should eq("Hello\n")
+            io.gets(chomp: false).should eq("World\n")
+          end
         end
-      end
 
-      it "gets big EUC-JP string" do
-        str = ("好我是人\n" * 1000).encode("EUC-JP")
-        base_io = IO::Memory.new(str)
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("EUC-JP")
-        1000.times do
-          io.gets(chomp: false).should eq("好我是人\n")
+        it "gets big EUC-JP string" do
+          str = ("好我是人\n" * 1000).encode("EUC-JP")
+          base_io = IO::Memory.new(str)
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("EUC-JP")
+          1000.times do
+            io.gets(chomp: false).should eq("好我是人\n")
+          end
         end
-      end
 
-      it "reads char" do
-        str = "x\nHello world" + ("1234567890" * 1000)
-        base_io = IO::Memory.new(str.encode("UCS-2LE"))
-        io = BufferedWrapper.new(base_io)
-        io.set_encoding("UCS-2LE")
-        io.gets(chomp: false).should eq("x\n")
-        str = str[2..-1]
-        str.each_char do |char|
-          io.read_char.should eq(char)
+        it "reads char" do
+          str = "x\nHello world" + ("1234567890" * 1000)
+          base_io = IO::Memory.new(str.encode("UCS-2LE"))
+          io = BufferedWrapper.new(base_io)
+          io.set_encoding("UCS-2LE")
+          io.gets(chomp: false).should eq("x\n")
+          str = str[2..-1]
+          str.each_char do |char|
+            io.read_char.should eq(char)
+          end
+          io.read_char.should be_nil
         end
-        io.read_char.should be_nil
       end
     end
-  end
+  {% end %}
 end