Merge pull request #13 from whatsthecraic/non-text-bytes-in-the-header

Non text bytes in the header
RelationalAI · Nov 22, 2023 · 40498f3 · 40498f3 · Drvi · Nov 25, 2023
2 parents a7837ec + 1bde0d6
commit 40498f3
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 2 deletions.
diff --git a/src/init_parsing.jl b/src/init_parsing.jl
@@ -97,7 +97,14 @@ function process_header_and_schema_and_finish_row_skip!(
                 elseif !Parsers.ok(code)
                     throw(HeaderParsingError("Error parsing header for column $i at $(lines_skipped_total+1):$(pos) (row:pos)."))
                 else
-                    push!(parsing_ctx.header, Symbol(strip(Parsers.getstring(row_bytes, val, options.e))))
+                    identifier_s = strip(Parsers.getstring(row_bytes, val, options.e))
+                    try
+                        push!(parsing_ctx.header, Symbol(identifier_s))
+                    catch
+                        # defensively truncate identifier_s to 2k characters in case something is very cursed
+                        throw(HeaderParsingError("Error parsing header for column $i ('$(first(identifier_s, 2000))') at " *
+                            "$(lines_skipped_total+1):$pos (row:pos): presence of invalid non text bytes in the CSV snippet"))
+                    end
                 end
                 pos += tlen
             end
@@ -166,7 +173,14 @@ function process_header_and_schema_and_finish_row_skip!(
             elseif !Parsers.ok(code)
                 throw(HeaderParsingError("Error parsing header for column $i at $(lines_skipped_total+1):$pos (row:pos)."))
             else
-                push!(parsing_ctx.header, Symbol(strip(Parsers.getstring(row_bytes, val, options.e))))
+                identifier_s = strip(Parsers.getstring(row_bytes, val, options.e))
+                try
+                    push!(parsing_ctx.header, Symbol(identifier_s))
+                catch
+                    # defensively truncate identifier_s to 2k characters in case something is very cursed
+                    throw(HeaderParsingError("Error parsing header for column $i ('$(first(identifier_s, 2000))') at " *
+                        "$(lines_skipped_total+1):$pos (row:pos): presence of invalid non text bytes in the CSV snippet"))
+                end
             end
             pos += tlen
             i += 1

diff --git a/test/exception_handling.jl b/test/exception_handling.jl
@@ -228,6 +228,14 @@ end
         header=true,
     )
 
+    @test_throws ChunkedCSV.HeaderParsingError("Error parsing header for column 1 ('a\0') at 1:1 (row:pos): presence of invalid non text bytes in the CSV snippet") parse_file(IOBuffer("""
+        a\0,b
+        1,2
+        """),
+        [Int,Int],
+        header=true,
+    )
+
     @test_throws ArgumentError("Provided header and schema names don't match. In schema, not in header: [:q]. In header, not in schema: [:a, :b, :c]") parse_file(IOBuffer("""
         a,b,c
         1,2,3