Merge pull request #35 from factly/fix/decimals-in-samples

Add inferschema on read_csv to infer correct datatype

app/utils/dataframes.py

@@ -49,7 +49,7 @@ async def get_dataframe_honouring_encoding_async(
         pl.DataFrame: polars Dataframe object
     """
     try:
-        df = pl.read_csv(source, null_values="NA", infer_schema_length=0)
+        df = pl.read_csv(source, null_values="NA", infer_schema_length=None)
     except (UnicodeDecodeError, pl_exc.ComputeError) as err:
         logger.warning(f"File encoding is not default: {err}")
         logger.warning("Trying to read file with proper encoding")
@@ -59,7 +59,7 @@ async def get_dataframe_honouring_encoding_async(
             source,
             null_values="NA",
             encoding=encoding,
-            infer_schema_length=0,
+            infer_schema_length=None,
         )
     return df
 
@@ -78,7 +78,7 @@ def get_dataframe_honouring_encoding(
         pl.DataFrame: polars Dataframe object
     """
     try:
-        df = pl.read_csv(source, null_values="NA", infer_schema_length=0)
+        df = pl.read_csv(source, null_values="NA", infer_schema_length=None)
     except (UnicodeDecodeError, pl_exc.ComputeError) as err:
         logger.error(f"Could not interpret File encoding : {err}")
         encoding = get_encoding(obj=source, is_object=is_object)
@@ -87,7 +87,7 @@ def get_dataframe_honouring_encoding(
             source,
             null_values="NA",
             encoding=encoding,
-            infer_schema_length=0,
+            infer_schema_length=None,
         )
     return df
 

app/utils/profile_segments.py

@@ -6,6 +6,7 @@
 from numpy import bool_
 from pandas import DataFrame
 from pydantic import parse_obj_as
+from ydata_profiling import ProfileReport
 
 from app.core.config import Settings
 from app.models.analysis import Analysis
@@ -38,7 +39,9 @@ def json_conversion_objects(obj):
 
 
 class ProfileSegments:
-    def __init__(self, pandas_profile, columns=None, round_to=3):
+    def __init__(
+        self, pandas_profile: ProfileReport, columns=None, round_to=3
+    ):
         """
         Pass pandas profile of a dataset as argument
         """
@@ -93,6 +96,9 @@ def package(self) -> Dict:
     def samples(self) -> List[Sample]:
         # get samples
         samples = self.profile_description.sample
+        import logging
+
+        logging.error(samples)
         for sample in samples:
             sample.data = sample.data.round(decimals=self.round_to).to_json()
         # * 'head' and 'tail' are returned as dataset sample