add escape quotes in spark read csv

RADAR-base · Jun 10, 2024 · 561f8db · 561f8db
1 parent 4b6f56d
commit 561f8db
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/radarpipeline/io/reader.py b/radarpipeline/io/reader.py
@@ -220,6 +220,8 @@ def _read_variable_data_files(
                     df = self.spark.read.load(
                         file_dict[column_hash],
                         format="csv",
+                        quote="\"",
+                        escape="\"",
                         header=True,
                         schema=schema.get_schema_by_hash(column_hash),
                         enforceSchema="false",
@@ -230,6 +232,8 @@ def _read_variable_data_files(
                     df = self.spark.read.load(
                         file_dict[column_hash],
                         format="csv",
+                        quote="\"",
+                        escape="\"",
                         header=True,
                         inferSchema="true",
                         encoding=constants.ENCODING,